fix(s3): align encrypted checksums and multipart completion (#7025)

* fix(s3): align encrypted checksum handling

* test(s3): align multipart SSE-C completion

* fix(ecstore): scope startup helper to tests

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
-sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2
+sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
+sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4

.config/e2e-nightly-selection.txt

@@ -1 +1 @@
-sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105
+sha256=51da41c54167602f2bd6c45921b39a44562bf3cfcdf468d992bb992c62cad7fd

.config/make/coverage.mak

@@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow,
 	@mkdir -p target/llvm-cov
 	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
 	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
+	$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json

.config/make/lint-fmt.mak

@@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 .PHONY: test-wiring-check
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
-	python3 ./scripts/check_test_wiring.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -35,13 +35,14 @@ script-tests: ## Run shell script tests
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/test_fuzz_runner.sh
+	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
-	python3 ./scripts/check_test_wiring.py --self-test
-	python3 ./scripts/check_security_coverage.py --self-test
-	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-	python3 ./scripts/s3-tests/test_report_compat.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
-	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
 
 .PHONY: test

.config/nextest.toml

@@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
 
 # --- default profile (local): serialize the flaky groups, never retry --------
 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.default.scripts]]
@@ -89,6 +89,29 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests. Every test in the module builds a
+# real-disk (TempDir-backed) hermetic erasure set and drives MiB-scale writes
+# plus deep-scan heal — the same load-sensitive cross-disk IO shape as the
+# crash_consistency scenarios above. Under a heavily parallel run a single
+# disk's IO can fail while write quorum still holds, which flips per-disk
+# readback and aggregate-outcome assertions nondeterministically (different
+# tests each round; all pass standalone). Preventive serialization only, no
+# retries. The matching ci-profile override is after [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair. Both carry
+# #[serial(metadata_cache_invalidation_probe)] — a no-op across nextest's
+# process boundary — and assert get_object_metadata_cache generation
+# semantics on a 4-disk hermetic set, the same load-sensitive shape that
+# forced the transition matrix tests into this group. Preventive
+# serialization only, no retries. The matching ci-profile override is after
+# [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # The production-handler relocation regression builds an isolated 8-disk,
 # 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
 # from overlapping the ecstore commit fixtures above.
@@ -183,7 +206,7 @@ fail-fast = false
 path = "junit.xml"
 
 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.ci.scripts]]
@@ -250,6 +273,20 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests under the ci profile too (see the
+# matching default-profile override near the top). Not a quarantine: no
+# retries, just serialized real-disk heal IO.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair under the ci
+# profile too (see the matching default-profile override near the top). Not a
+# quarantine: no retries.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # Match the default-profile embedded test isolation without quarantining or
 # retrying failures in CI.
 [[profile.ci.overrides]]
@@ -440,7 +477,7 @@ path = "junit.xml"
 [profile.e2e-nightly]
 default-filter = """
     package(e2e_test)
-    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
 """
 fail-fast = false
 
@@ -496,7 +533,7 @@ path = "junit.xml"
 default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
-    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
     & !test(/^replication_extension_test::/)
 """
 fail-fast = false

.github/workflows/audit.yml

@@ -24,8 +24,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
@@ -37,8 +40,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
     # Daily, not weekly. This schedule exists to catch RustSec advisories
@@ -146,6 +152,12 @@ jobs:
       - name: Check performance A/B workflow trust boundary
         run: ./scripts/security/check_performance_ab_workflow.sh
 
+      - name: Check tier evidence workflow isolation
+        run: ./scripts/security/check_tier_artifact_workflow.sh
+
+      - name: Check package version contract
+        run: ./scripts/test_package_versions.sh
+
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -244,7 +244,7 @@ jobs:
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 150
+    timeout-minutes: 180
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Release binaries ship without dial9 telemetry and therefore do not need
@@ -408,9 +408,9 @@ jobs:
 
           if [[ "${{ matrix.cross }}" == "true" ]]; then
             # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           fi
 
       - name: Create release package

.github/workflows/e2e-upgrade.yml

@@ -49,8 +49,20 @@ env:
   UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
 
 jobs:
-  direct-upgrade:
-    name: Direct upgrade from rc.2
+  upgrade:
+    name: ${{ matrix.name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Direct upgrade from rc.2
+            cache_key: e2e-direct-upgrade
+            test: direct_upgrade_from_rc2_preserves_object_contracts
+            artifact: direct-upgrade
+          - name: Mixed-version rolling upgrade from rc.2
+            cache_key: e2e-mixed-version-upgrade
+            test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
+            artifact: mixed-version-upgrade
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -64,7 +76,7 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: e2e-direct-upgrade
+          cache-shared-key: ${{ matrix.cache_key }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: "false"
 
@@ -89,17 +101,17 @@ jobs:
           cargo build --locked -p rustfs --bin rustfs
           : > target/debug/rustfs.features
 
-      - name: Run direct-upgrade compatibility test
+      - name: Run upgrade compatibility test
         run: |
           cargo test --locked -p e2e_test \
-            upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \
+            "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: direct-upgrade-server-logs-${{ github.run_number }}
+          name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs
           if-no-files-found: warn
           retention-days: 14

.github/workflows/package.yml

@@ -21,10 +21,10 @@
 # - workflow_run: automatically package after "Build and Release" completes
 #   for a release tag (the mac/windows/linux binaries are already uploaded
 #   to the GitHub release before packaging starts)
-# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
+# - workflow_dispatch: manual fallback with a release tag and/or exact build run ID
 #
 # Flow:
-# 1. Resolve the triggering Build workflow run for the release tag
+# 1. Resolve and validate the selected Build workflow run and source identity
 # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
 # 3. Build DEB packages for amd64 and arm64
 # 4. Build RPM packages for x86_64 and aarch64
@@ -51,7 +51,7 @@ on:
         required: false
         type: string
       build_run_id:
-        description: "Build workflow run ID (overrides tag lookup)"
+        description: "Build workflow run ID (when combined with tag, both must identify the same release commit)"
         required: false
         type: string
 
@@ -82,6 +82,9 @@ jobs:
       version: ${{ steps.resolve.outputs.version }}
       build_type: ${{ steps.resolve.outputs.build_type }}
       build_run_id: ${{ steps.resolve.outputs.build_run_id }}
+      build_run_number: ${{ steps.resolve.outputs.build_run_number }}
+      head_sha: ${{ steps.resolve.outputs.head_sha }}
+      dev_sequence: ${{ steps.resolve.outputs.dev_sequence }}
       tag: ${{ steps.resolve.outputs.tag }}
     steps:
       - name: Resolve build run
@@ -89,90 +92,129 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
+          EVENT_NAME: ${{ github.event_name }}
+          REPOSITORY: ${{ github.repository }}
           INPUT_TAG: ${{ github.event.inputs.tag }}
           INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
         run: |
           set -euo pipefail
 
-          # Determine tag
-          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            TAG="${HEAD_BRANCH}"
-          elif [[ -n "$INPUT_TAG" ]]; then
-            TAG="$INPUT_TAG"
+          fail() {
+            echo "❌ $1" >&2
+            exit 1
+          }
+
+          TAG=""
+          BUILD_RUN_ID=""
+          case "$EVENT_NAME" in
+            workflow_run)
+              TAG="$HEAD_BRANCH"
+              BUILD_RUN_ID="$WORKFLOW_RUN_ID"
+              ;;
+            workflow_dispatch)
+              TAG="$INPUT_TAG"
+              BUILD_RUN_ID="$INPUT_RUN_ID"
+              ;;
+            *) fail "unsupported event: $EVENT_NAME" ;;
+          esac
+
+          # Validate and classify tags before using them in API paths or logs.
+          semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)'
+          prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)'
+          if [[ -n "$TAG" ]]; then
+            if [[ "$TAG" =~ ^${semver_core}-${prerelease_id}-preview\.(0|[1-9][0-9]*)$ ]]; then
+              BUILD_TYPE=preview
+            elif [[ "$TAG" =~ ^${semver_core}-${prerelease_id}$ ]]; then
+              BUILD_TYPE=prerelease
+            elif [[ "$TAG" =~ ^${semver_core}$ ]]; then
+              BUILD_TYPE=release
+            else
+              fail "tag is not a supported strict package version"
+            fi
           else
-            TAG=""
+            BUILD_TYPE=development
           fi
 
-          echo "Tag: ${TAG:-<none>}"
-
-          # Determine build run ID
-          BUILD_RUN_ID=""
-
-          if [[ -n "$INPUT_RUN_ID" ]]; then
-            # Explicit run ID takes priority
-            BUILD_RUN_ID="$INPUT_RUN_ID"
-            echo "Using explicit build run ID: $BUILD_RUN_ID"
-
-          elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            # Use the Build and Release run that triggered this workflow
-            BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
-            echo "Using triggering workflow run: $BUILD_RUN_ID"
-
+          if [[ -n "$BUILD_RUN_ID" ]]; then
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "build run ID must be a positive decimal integer"
+            echo "Using selected build run: $BUILD_RUN_ID"
           elif [[ -n "$TAG" ]]; then
-            # Find the build run that produced this tag
             echo "Looking for build run for tag: $TAG"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
 
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              # Tag might not be a branch; try event=push with head_branch matching
-              BUILD_RUN_ID=$(gh api \
-                "repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
-                --jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
-            fi
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful build run found for tag: $TAG"
-              exit 1
+            if [[ -z "$BUILD_RUN_ID" ]]; then
+              BUILD_RUN_ID=$(gh api --method GET \
+                "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+                -f event=push -f status=success -F per_page=100 2>/dev/null |
+                jq -r --arg tag "$TAG" \
+                  '[.workflow_runs[] | select(.head_branch == $tag)][0].id // empty' || true)
             fi
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful build run found for tag"
             echo "Found build run: $BUILD_RUN_ID"
-
           else
-            # No tag — latest successful main build
             echo "No tag specified, looking for latest main build"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful main build found"
-              exit 1
-            fi
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch=main -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful main build found"
             echo "Latest main build: $BUILD_RUN_ID"
           fi
 
-          # Determine version and build type
+          # Fetch once and use the same immutable run metadata for identity,
+          # ordering, workflow provenance, and release-channel validation.
+          RUN_JSON=$(gh api "repos/${REPOSITORY}/actions/runs/${BUILD_RUN_ID}") ||
+            fail "cannot read selected build run"
+          RUN_ID=$(jq -r '.id // empty' <<<"$RUN_JSON")
+          RUN_NUMBER=$(jq -r '.run_number // empty' <<<"$RUN_JSON")
+          RUN_STATUS=$(jq -r '.status // empty' <<<"$RUN_JSON")
+          RUN_CONCLUSION=$(jq -r '.conclusion // empty' <<<"$RUN_JSON")
+          RUN_PATH=$(jq -r '.path // empty' <<<"$RUN_JSON")
+          HEAD_SHA=$(jq -r '.head_sha // empty' <<<"$RUN_JSON")
+          RUN_HEAD_BRANCH=$(jq -r '.head_branch // empty' <<<"$RUN_JSON")
+
+          [[ "$RUN_ID" == "$BUILD_RUN_ID" ]] || fail "run metadata ID mismatch"
+          [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] || fail "build run number must be a positive decimal integer"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]] || fail "selected build run is not successful"
+          [[ "$RUN_PATH" == .github/workflows/build.yml ]] || fail "selected run is not Build and Release"
+          [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "selected build run has an invalid head SHA"
+          [[ "$RUN_HEAD_BRANCH" != *$'\n'* && -n "$RUN_HEAD_BRANCH" ]] || fail "selected build run has an invalid head branch"
+
           if [[ -n "$TAG" ]]; then
+            [[ "$RUN_HEAD_BRANCH" == "$TAG" ]] || fail "tag and build run head branch do not match"
+
+            TAG_REF_JSON=$(gh api "repos/${REPOSITORY}/git/ref/tags/${TAG}") ||
+              fail "cannot resolve release tag ref"
+            TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_REF_JSON")
+            TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_REF_JSON")
+            depth=0
+            while [[ "$TAG_OBJECT_TYPE" == tag && $depth -lt 5 ]]; do
+              TAG_OBJECT_JSON=$(gh api "repos/${REPOSITORY}/git/tags/${TAG_OBJECT_SHA}") ||
+                fail "cannot peel annotated release tag"
+              TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_OBJECT_JSON")
+              TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_OBJECT_JSON")
+              depth=$((depth + 1))
+            done
+            [[ "$TAG_OBJECT_TYPE" == commit && "$TAG_OBJECT_SHA" =~ ^[0-9a-f]{40}$ ]] ||
+              fail "release tag does not resolve to a commit"
+            [[ "$TAG_OBJECT_SHA" == "$HEAD_SHA" ]] || fail "release tag commit and build run head SHA do not match"
             VERSION="$TAG"
-            if [[ "$TAG" == *"-preview"* ]]; then
-              BUILD_TYPE="preview"
-            elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
-              BUILD_TYPE="prerelease"
-            else
-              BUILD_TYPE="release"
-            fi
+            DEV_SEQUENCE=""
           else
-            SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
-              --jq '.head_sha' 2>/dev/null | head -c 7)
-            VERSION="dev-${SHORT_SHA}"
-            BUILD_TYPE="development"
+            VERSION="dev-${HEAD_SHA}"
+            DEV_SEQUENCE="$RUN_NUMBER"
           fi
 
           {
             echo "version=$VERSION"
             echo "build_type=$BUILD_TYPE"
             echo "build_run_id=$BUILD_RUN_ID"
+            echo "build_run_number=$RUN_NUMBER"
+            echo "head_sha=$HEAD_SHA"
+            echo "dev_sequence=$DEV_SEQUENCE"
             echo "tag=${TAG}"
           } >> "$GITHUB_OUTPUT"
 
@@ -180,6 +222,7 @@ jobs:
           echo "  Version: $VERSION"
           echo "  Build type: $BUILD_TYPE"
           echo "  Build run ID: $BUILD_RUN_ID"
+          echo "  Build run number: $RUN_NUMBER"
 
   # Build DEB and RPM packages for each architecture
   package:
@@ -206,6 +249,22 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Normalize package metadata
+        id: versions
+        shell: bash
+        env:
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
+          DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+        run: |
+          set -euo pipefail
+
+          normalized=$(./scripts/release/package_versions.sh \
+            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
+          printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
+
       - name: Download binary artifact from build run
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
@@ -245,18 +304,16 @@ jobs:
       - name: Build DEB package
         id: deb
         shell: bash
+        env:
+          DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          DEB_FILE: ${{ steps.versions.outputs.deb_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          DEB_ARCH="${{ matrix.deb_arch }}"
-          # DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
-          # Use a variable for ~ to prevent tilde expansion by bash
-          TILDE='~'
-          DEB_VERSION="${VERSION/-/$TILDE}"
-          PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
+          PKG_DIR="${DEB_FILE%.deb}"
 
-          echo "Building DEB: ${PKG_DIR}.deb"
+          echo "Building DEB: ${DEB_FILE}"
 
           mkdir -p "${PKG_DIR}/DEBIAN"
           mkdir -p "${PKG_DIR}/usr/bin"
@@ -333,9 +390,12 @@ jobs:
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
           cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
 
-          fakeroot dpkg-deb --build "${PKG_DIR}"
+          fakeroot dpkg-deb --build "${PKG_DIR}" "$DEB_FILE"
 
-          DEB_FILE="${PKG_DIR}.deb"
+          [[ $(dpkg-deb -f "$DEB_FILE" Package) == rustfs ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Version) == "$DEB_VERSION" ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Architecture) == "$DEB_ARCH" ]]
+          dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -tf - | grep -Fx './usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
@@ -343,16 +403,19 @@ jobs:
       - name: Build RPM package
         id: rpm
         shell: bash
+        env:
+          RPM_VERSION: ${{ steps.versions.outputs.rpm_version }}
+          RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+          RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          RPM_ARCH="${{ matrix.rpm_arch }}"
-
           echo "Building RPM for ${RPM_ARCH}"
 
-          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
+          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
           sudo gem install fpm
+          ./scripts/test_package_versions.sh --require-package-managers
 
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
@@ -367,8 +430,10 @@ jobs:
 
           fpm -s dir -t rpm \
             --name rustfs \
-            --version "$VERSION" \
+            --version "$RPM_VERSION" \
+            --iteration "$RPM_RELEASE" \
             --architecture "$RPM_ARCH" \
+            --package "$RPM_FILE" \
             --depends "glibc >= 2.31" \
             --maintainer "RustFS Team <support@rustfs.com>" \
             --description "High-performance distributed object storage" \
@@ -410,13 +475,15 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
-            RPM_FILE="${RPM_FILE#./}"
-          if [[ -z "$RPM_FILE" ]]; then
+          if [[ ! -f "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
+          RPM_METADATA=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$RPM_FILE")
+          EXPECTED_METADATA=$(printf 'rustfs\n%s\n%s\n%s' "$RPM_VERSION" "$RPM_RELEASE" "$RPM_ARCH")
+          [[ "$RPM_METADATA" == "$EXPECTED_METADATA" ]]
+          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
@@ -438,6 +505,9 @@ jobs:
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
           R2_BUCKET: ${{ secrets.R2_BUCKET }}
           AWS_EC2_METADATA_DISABLED: true
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
@@ -455,7 +525,6 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
@@ -465,9 +534,6 @@ jobs:
 
           echo "📤 Uploading to $R2_PATH"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           for f in "$DEB_FILE" "$RPM_FILE"; do
             if [[ -n "$f" && -f "$f" ]]; then
               echo "Uploading: $f"
@@ -493,14 +559,13 @@ jobs:
         if: needs.resolve.outputs.tag != ''
         env:
           GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.resolve.outputs.tag }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
 
-          TAG="${{ needs.resolve.outputs.tag }}"
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           # Upload the packages, then refresh the release checksums so the new
           # assets are covered, matching the binary release flow.
           for f in "$DEB_FILE" "$RPM_FILE"; do
@@ -552,14 +617,19 @@ jobs:
     steps:
       - name: Print summary
         shell: bash
+        env:
+          SUMMARY_VERSION: ${{ needs.resolve.outputs.version }}
+          SUMMARY_BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SUMMARY_BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          SUMMARY_PACKAGE_STATUS: ${{ needs.package.result }}
         run: |
           {
             echo "## 📦 Package Summary"
             echo ""
             echo "| Item | Value |"
             echo "|------|-------|"
-            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
-            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
-            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
-            echo "| Package Status | ${{ needs.package.result }} |"
+            echo "| Version | \`${SUMMARY_VERSION}\` |"
+            echo "| Build Type | ${SUMMARY_BUILD_TYPE} |"
+            echo "| Build Run | #${SUMMARY_BUILD_RUN_ID} |"
+            echo "| Package Status | ${SUMMARY_PACKAGE_STATUS} |"
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-functional-chain.yml

@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Functional chain driver: runs the nine functional suites in a fixed order
+# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security, with
+# performance on its own runner in parallel) and guarantees the chain keeps
+# moving even when individual suites fail.
+#
+# Each suite workflow can still be dispatched standalone (workflow_dispatch);
+# only chain-triggered runs forward to the next suite via repository_dispatch,
+# so a standalone run never drags the rest of the chain behind it.
+#
+# Why not workflow_run chaining: GitHub does not guarantee delivery of
+# workflow_run events (they are fire-and-forget), and the head-SHA filter made
+# newly added suites (storage) unable to trigger at all. Explicit
+# repository_dispatch handoffs are verifiable and re-drivable.
+
+name: RustFS Functional Chain
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    # Entry point: start the chain after the nightly build completes. The
+    # build's own conclusion does not gate the chain; each suite reports its
+    # own result to rustfs/backlog and the dashboard.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  start-chain:
+    name: Start functional chain (upgrade first)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    steps:
+      - name: Dispatch first suite (upgrade)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-upgrade' \
+            -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -23,6 +23,11 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the storage suite finishes. Heal runs
+    # exactly once per chain; the pool expansion workflow no longer embeds
+    # its own heal pass.
+    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -49,19 +54,33 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 480
-    # Manual-only standalone run. Nightly chain already runs heal in
-    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
-    if: ${{ github.event_name == 'workflow_dispatch' }}
+    # Standalone manual run, or one link of the nightly functional chain
+    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -116,8 +135,8 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
+            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
+            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
             --log-file /tmp/rustfs-heal-test.log
 
       - name: Generate report
@@ -175,155 +194,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'heal'
+          SUITE_LABEL: 'Heal'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-heal-report.md'
+          LOG_FILE: '/tmp/rustfs-heal-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -354,6 +282,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Pool expansion)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-pool' \
+              -F 'client_payload[from_suite]=heal'; then
+              echo "dispatched next suite Pool expansion (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after heal (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-pool`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-kms-test.yml

@@ -23,10 +23,9 @@ on:
         description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after S3 compatibility test completes.
-    workflows: ["RustFS S3 Compatibility Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the S3 compatibility suite finishes.
+    types: [rustfs-chain-kms]
 
 permissions:
   contents: read
@@ -52,16 +51,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -240,105 +250,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'kms'
+          SUITE_LABEL: 'KMS'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-kms-report.md'
+          LOG_FILE: '/tmp/rustfs-kms.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -369,6 +338,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Tier)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-tier' \
+              -F 'client_payload[from_suite]=kms'; then
+              echo "dispatched next suite Tier (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after kms (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-tier`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-performance-test.yml

@@ -48,10 +48,10 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Run after the nightly build completes; the nightly deb is what the test installs.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
+    types: [rustfs-chain-performance]
 
 permissions:
   contents: read
@@ -84,19 +84,33 @@ env:
 jobs:
   performance-test:
     runs-on: pf-testing
+    # Requirement: a failing benchmark must not fail the workflow;
+    # failures are filed to rustfs/backlog.
+    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -212,6 +226,65 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'performance'
+          SUITE_LABEL: 'Performance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-perf-report.md'
+          LOG_FILE: '/tmp/rustfs-perf-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
       - name: Upload test logs & results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion / Heal Test
+name: RustFS Pool Expansion Test
 
 on:
   workflow_dispatch:
@@ -33,14 +33,6 @@ on:
         description: 'Run the pool decommission step (3-pool topology only)'
         type: boolean
         default: true
-      stop_node_gb:
-        description: 'Heal: stop the outage node when surviving nodes reach N GiB'
-        required: false
-        default: '15'
-      warp_stop_gb:
-        description: 'Heal: stop warp when surviving nodes reach N GiB'
-        required: false
-        default: '40'
       cleanup_before:
         description: 'Reset the nodes before the test (DESTROYS existing data/config)'
         type: boolean
@@ -49,17 +41,16 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the heal suite finishes.
+    types: [rustfs-chain-pool]
 
 permissions:
   contents: read
 
-# Only one test run at a time: every job mutates the same shared test
+# Only one test run at a time: the job mutates the same shared test
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
-# other. Jobs inside a run are chained with needs to serialize them.
+# other.
 concurrency:
   group: rustfs-shared-functional-tests
   cancel-in-progress: false
@@ -80,330 +71,16 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  heal-test:
-    name: Heal test
-    runs-on: smoke-testing
-    timeout-minutes: 480
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        if: ${{ inputs.cleanup_before != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Install RustFS package & start cluster
-        run: |
-          ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Preflight checks
-        run: |
-          ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Run heal test (write -> outage -> heal -> verify)
-        id: test
-        run: |
-          ARGS=(--steps "3,4,5,6,7" -y \
-            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          {
-            echo "# RustFS heal test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-          SUITE: heal
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          fi
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-heal-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-heal-test.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
-
-      - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS heal test failed"
-          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
-          echo "See the uploaded log artifact for details."
-
-  # Pool expansion runs after heal regardless of heal outcome.
+  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
+  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 360
-    needs: heal-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run') }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -411,14 +88,25 @@ jobs:
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Initialize pool test artifacts
         run: |
@@ -810,155 +498,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'pool'
+          SUITE_LABEL: 'Pool expansion'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
+          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -997,6 +594,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Security)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-security' \
+              -F 'client_payload[from_suite]=pool'; then
+              echo "dispatched next suite Security (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after pool (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-security`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-s3-compat-test.yml

@@ -11,10 +11,9 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Run after upgrade compatibility completes; the nightly deb is what the test installs.
-    workflows: ["RustFS Upgrade Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the upgrade suite finishes.
+    types: [rustfs-chain-s3]
 
 permissions:
   contents: read
@@ -40,16 +39,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -220,155 +230,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 's3'
+          SUITE_LABEL: 'S3 compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+          LOG_FILE: '/tmp/rustfs-s3-compat.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -399,6 +318,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: KMS)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-kms' \
+              -F 'client_payload[from_suite]=s3'; then
+              echo "dispatched next suite KMS (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after s3 (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-kms`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-security-test.yml

@@ -46,10 +46,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs last in the functional chain, after pool/heal, on the shared VMs.
-    workflows: ["RustFS Pool Expansion / Heal Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the pool expansion suite finishes (last link).
+    types: [rustfs-chain-security]
 
 permissions:
   contents: read
@@ -77,16 +76,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Checkout repository (for the OIDC live gate script)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -193,155 +203,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'security'
+          SUITE_LABEL: 'Security'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-security-report.md'
+          LOG_FILE: ''
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()

.github/workflows/rustfs-storage-test.yml

@@ -20,10 +20,9 @@ on:
           - single-multi
           - multi-multi
         default: all
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the tier suite finishes.
+    types: [rustfs-chain-storage]
 
 permissions:
   contents: read
@@ -49,16 +48,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -235,156 +245,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'storage', label: 'Storage Engine' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'storage'
+          SUITE_LABEL: 'Storage engine'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-storage-report.md'
+          LOG_FILE: '/tmp/rustfs-storage.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -415,6 +333,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Heal)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-heal' \
+              -F 'client_payload[from_suite]=storage'; then
+              echo "dispatched next suite Heal (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after storage (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-heal`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-tier-test.yml

@@ -11,10 +11,18 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after KMS test completes.
-    workflows: ["RustFS KMS Test"]
-    types: [completed]
+      rc_sha256:
+        description: 'Optional SHA-256 for the preinstalled rc binary; mismatch is an infrastructure failure.'
+        required: false
+        type: string
+      force_case_failure:
+        description: 'Diagnostic only: rewrite single-single/TIER-101 to FAIL after execution to verify artifact and final-gate behavior.'
+        required: false
+        default: false
+        type: boolean
+  repository_dispatch:
+    # Chain handoff: dispatched when the KMS suite finishes.
+    types: [rustfs-chain-tier]
 
 permissions:
   contents: read
@@ -33,23 +41,50 @@ env:
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  RUSTFS_EXPECTED_RC_SHA256: ${{ inputs.rc_sha256 || vars.RUSTFS_TIER_RC_SHA256 }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
 
 jobs:
   tier-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      - name: Initialize run evidence directory
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then
+            echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2
+            exit 1
+          fi
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -110,13 +145,25 @@ jobs:
         id: test
         continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-tier.log
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
         run: |
           set -euo pipefail
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
           chmod +x auto-testing/rustfs-tier-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          RC_BIN="$(command -v rc)"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
+          ARGS=(
+            --all-topologies
+            -y
+            --log-file "${LOG_FILE}"
+            --rc-bin "${RC_BIN}"
+            --artifacts-dir "${TIER_ARTIFACTS_DIR}"
+          )
+          if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
+            ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
+          fi
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -126,15 +173,34 @@ jobs:
           fi
           ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
 
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-tier.log
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+      - name: Inject diagnostic case failure
+        if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }}
         run: |
           set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json"
+          test -s "${RESULT_FILE}"
+          TMP_FILE="$(mktemp "${TIER_ARTIFACTS_DIR}/cases/.forced.XXXXXX")"
+          jq '.status = "FAIL" | .case_rc = 97' "${RESULT_FILE}" > "${TMP_FILE}"
+          mv "${TMP_FILE}" "${RESULT_FILE}"
+
+      - name: Generate report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        env:
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          TRIGGER_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
+          REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md"
+          CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
           if [ -n "${PACKAGE_URL}" ]; then
             PACKAGE_SOURCE="${PACKAGE_URL}"
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -142,65 +208,31 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-tier-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          set +e
+          python3 auto-testing/rustfs_tier_report.py \
+            --results-dir "${TIER_ARTIFACTS_DIR}/cases" \
+            --provenance "${TIER_ARTIFACTS_DIR}/provenance.json" \
+            --output "${CASE_TABLE}"
+          CASE_GATE_RC=$?
+          set -e
+          printf '%s\n' "${CASE_GATE_RC}" > "${GATE_RC_FILE}"
+          if [ ! -s "${CASE_TABLE}" ]; then
+            {
+              echo "## Case Summary"
+              echo ""
+              echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
+            } > "${CASE_TABLE}"
+          fi
           {
             echo "# RustFS tier test report"
             echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${TRIGGER_NAME}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${TEST_OUTCOME}"
+            echo "- Structured Gate Exit: ${CASE_GATE_RC}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            cat "${CASE_TABLE}"
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -210,11 +242,11 @@ jobs:
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
           SUITE: tier
         run: |
           set -euo pipefail
@@ -236,115 +268,42 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: Verify required tier evidence
+        id: evidence_verify
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          failed=0
+          for name in \
+            rustfs-tier.log \
+            rustfs-tier-report.md \
+            rustfs-tier-cases.md \
+            rustfs-tier-gate.rc \
+            provenance.json; do
+            if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence is missing or empty: ${name}" >&2
+              failed=1
+            fi
+          done
+          for name in cases logs; do
+            if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence directory is missing: ${name}" >&2
+              failed=1
+            fi
+          done
+          if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then
+            echo "no atomic tier case result was produced" >&2
+            failed=1
           fi
+          [ "${failed}" -eq 0 ]
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-tier-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-tier.log
-            /tmp/rustfs-tier-report.md
-          if-no-files-found: warn
+          name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.TIER_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
@@ -367,6 +326,157 @@ jobs:
             '
           done
 
+      - name: Enforce tier suite result
+        id: gate
+        if: always()
+        env:
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
+        run: |
+          set -euo pipefail
+          failed=0
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${TEST_OUTCOME}" != "success" ]; then
+            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "structured gate result is unavailable because evidence initialization failed" >&2
+          elif [ ! -s "${GATE_RC_FILE}" ]; then
+            echo "structured gate result is missing" >&2
+            failed=1
+          else
+            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
+            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
+              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
+              failed=1
+            fi
+          fi
+          [ "${failed}" -eq 0 ]
+
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'tier'
+          SUITE_LABEL: 'Tier'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
+          GATE_OUTCOME: ${{ steps.gate.outcome }}
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
+          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
+            echo "- Evidence verification: ${VERIFY_OUTCOME}"
+            echo "- Final gate: ${GATE_OUTCOME}"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+              echo "(the run evidence directory was rejected; its contents were not read)"
+            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
+              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
+            elif [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
+      - name: "Continue functional chain (next: Storage engine)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-storage' \
+              -F 'client_payload[from_suite]=tier'; then
+              echo "dispatched next suite Storage engine (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after tier (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-storage`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-upgrade-test.yml

@@ -53,11 +53,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs first in the functional chain: upgrade compatibility gates the
-    # nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
+    types: [rustfs-chain-upgrade]
 
 permissions:
   contents: read
@@ -83,16 +81,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -288,155 +297,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 'upgrade', label: 'Upgrade' },
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('upgrade');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'upgrade'
+          SUITE_LABEL: 'Upgrade compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+          LOG_FILE: '/tmp/rustfs-upgrade.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -468,6 +386,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: S3 compatibility)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${{GH_TOKEN:-}}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-s3' \
+              -F 'client_payload[from_suite]=upgrade'; then
+              echo "dispatched next suite S3 compatibility (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after upgrade (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-s3`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

AGENTS.md

@@ -123,12 +123,13 @@ runtime/build output:
 - Use `make pre-commit` only when its repository-wide fast checks add confidence
   beyond the focused checks.
 
-### Broad or High-Risk Changes
+### Broad Cross-Module Changes
 
-After the required adversarial review, run `make pre-pr` when targeted coverage
-cannot bound the impact, including dependency/toolchain/build-matrix changes,
-unbounded cross-crate APIs, or locking, durability, erasure coding, replication,
-RPC, IAM/KMS/auth, cryptography, on-disk/on-wire, and S3-visible behavior.
+Do not run `make pre-pr` by default before opening a PR. Consider it only when
+the final diff is broad, spans multiple modules, and targeted checks cannot
+bound the impact. Decide dynamically from the affected boundaries and risks;
+otherwise use the scoped formatting, linting, compilation, and test checks
+above.
 
 `make pre-pr` includes `make pre-commit`; never run both for the same unchanged
 diff. Do not repeat a check already covered by a successful umbrella gate.

CLAUDE.md

@@ -15,7 +15,7 @@ cargo check -p <crate>               # fast type-check one crate
 cargo test -p <crate>                # test one crate
 cargo fmt --all                      # format (required before PR)
 make pre-commit                      # fast gate: fmt + arch checks + quick-check (NO clippy/tests)
-make pre-pr                          # full pre-PR gate: fmt + arch checks + clippy + tests
+make pre-pr                          # optional full gate for broad cross-module changes
 make build-docker BUILD_OS=ubuntu22.04
 ```
 

CONTRIBUTING.md

@@ -62,12 +62,20 @@ make test
 # Fast pre-commit gate — see below for exactly what it runs
 make pre-commit
 
-# Full pre-PR gate (pre-commit gates + clippy + tests)
+# Optional full gate for broad cross-module changes (pre-commit + clippy + tests)
 make pre-pr
 ```
 
 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.
 
+> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the
+> security-coverage and scheduled-validation self-tests in `make test`) and import
+> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through
+> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls
+> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a
+> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a
+> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`.
+
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
 > For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
@@ -88,14 +96,16 @@ make pre-pr
 8. `quick-check` — `cargo check --workspace --exclude e2e_test`
 
 **`make pre-commit` does NOT run clippy and does NOT run any tests.**
-A green `make pre-commit` is not enough to open a pull request.
+It does not replace the scoped Clippy and test checks applicable to a change.
 
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
 tests). Complete the applicable multi-role adversarial review described in
-`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
-updating a pull request. This is what CI enforces.
+`AGENTS.md` first. Do not run `make pre-pr` locally by default before opening or
+updating a pull request. Consider it only for a broad change that spans multiple
+modules and whose impact cannot be bounded by targeted checks; decide from the
+affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -114,8 +124,9 @@ Or manually:
 chmod +x .git/hooks/pre-commit
 ```
 
-With or without a hook, the expectation is the same: run `make pre-commit`
-before committing and `make pre-pr` before opening a pull request.
+With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
+applicable scoped checks, and reserve `make pre-pr` for broad cross-module
+changes whose impact cannot be bounded by those checks.
 
 ### 📝 Formatting Configuration
 
@@ -154,7 +165,9 @@ Example output when formatting fails:
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
-6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+6. **Run applicable scoped checks before opening/updating a PR**; consider
+   `make pre-pr` only for broad cross-module changes whose impact cannot be
+   bounded by targeted checks
 7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration

Cargo.lock

@@ -4210,7 +4210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5039,9 +5039,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284"
 
 [[package]]
 name = "hex"
@@ -5668,7 +5668,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6650,10 +6650,11 @@ dependencies = [
 
 [[package]]
 name = "mysql_async"
-version = "0.37.0"
+version = "0.37.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2"
+checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4"
 dependencies = [
+ "arc-swap",
  "bytes",
  "crossbeam-queue",
  "crossbeam-utils",
@@ -6986,9 +6987,9 @@ checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
 
 [[package]]
 name = "nvml-wrapper"
-version = "0.12.1"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
+checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
 dependencies = [
  "bitflags 2.13.1",
  "libloading",
@@ -7000,9 +7001,9 @@ dependencies = [
 
 [[package]]
 name = "nvml-wrapper-sys"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b"
+checksum = "5d2079f4c9b6d2170bfb71c6355734ead6c47da75c179847395c31f9f2f66ede"
 dependencies = [
  "libloading",
 ]
@@ -7013,7 +7014,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.21.7",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -8036,9 +8037,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppmd-rust"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
+checksum = "9e9219bcb9d7aca6b2f63c83cf100cf78bcd619ac46e6ecbd0dd90869a39345d"
 
 [[package]]
 name = "ppv-lite86"
@@ -8240,7 +8241,7 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -8260,7 +8261,7 @@ version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -8610,7 +8611,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -9461,6 +9462,7 @@ dependencies = [
  "rustfs-io-metrics",
  "rustfs-keystone",
  "rustfs-kms",
+ "rustfs-license",
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
@@ -9512,7 +9514,7 @@ dependencies = [
  "tokio-util",
  "tonic",
  "tower",
- "tower-http 0.7.0",
+ "tower-http 0.7.1",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -10063,6 +10065,13 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustfs-license"
+version = "1.0.0-rc.5"
+dependencies = [
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "rustfs-lifecycle"
 version = "1.0.0-rc.5"
@@ -10606,6 +10615,8 @@ dependencies = [
  "rustfs-s3select-api",
  "rustfs-test-utils",
  "s3s",
+ "serde",
+ "serde_json",
  "tokio",
  "tracing",
 ]
@@ -10943,7 +10954,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11016,7 +11027,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11073,7 +11084,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11104,6 +11115,7 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
+ "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
  "serde",
@@ -11127,28 +11139,45 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "s3s-rfc2047"
+version = "0.16.0-alpha.1"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
+dependencies = [
+ "base64-simd",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
+ "base64-simd",
+ "hmac 0.13.0",
  "jiff",
+ "sha1 0.11.0",
+ "smallvec",
  "thiserror 2.0.20",
 ]
 
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
  "arrayvec",
  "base64-simd",
  "hex-simd",
+ "hmac 0.13.0",
  "jiff",
  "nom 8.0.0",
  "serde",
+ "sha2 0.11.0",
  "smallvec",
+ "std-next",
  "thiserror 2.0.20",
+ "zeroize",
 ]
 
 [[package]]
@@ -12040,9 +12069,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "10.0.2"
+version = "11.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
+checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
 dependencies = [
  "async-trait",
  "chrono",
@@ -12233,10 +12262,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -12729,9 +12758,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
 dependencies = [
  "async-compression",
  "bitflags 2.13.1",
@@ -13355,7 +13384,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]

Cargo.toml

@@ -29,6 +29,7 @@ members = [
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
     "crates/keystone", # OpenStack Keystone integration
+    "crates/license", # License and entitlement provider contracts
     "crates/lifecycle", # Lifecycle rule evaluation contracts
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
@@ -71,7 +72,7 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.97.1"
+rust-version = "1.98.0"
 version = "1.0.0-rc.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
@@ -105,6 +106,7 @@ rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
 rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
 rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
+rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
 rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
 rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
 rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
@@ -142,7 +144,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
-mysql_async = { default-features = false, version = "0.37" }
+mysql_async = { default-features = false, version = "0.37.1" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
@@ -174,7 +176,7 @@ tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
 tower = { version = "0.5.3" }
-tower-http = { version = "0.7.0" }
+tower-http = { version = "0.7.1" }
 
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
@@ -232,7 +234,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
 astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
@@ -283,7 +285,7 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.16" }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.12.1"
+nvml-wrapper = "0.13.0"
 parking_lot = "0.12.5"
 path-absolutize = "4.0.1"
 percent-encoding = "2.3.2"
@@ -305,7 +307,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "28e9ebb23dd2fb7d667084f34121b4aa4807a5c6", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -355,7 +357,7 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "10.0.2" }
+suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"

Makefile

@@ -23,6 +23,12 @@ SHELL := $(shell which bash)
 .SHELLFLAGS = -eu -o pipefail -c
 
 DOCKER_CLI ?= docker
+# Python interpreter for the repository's helper scripts. They import tomllib
+# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go
+# through a resolver that picks a new-enough interpreter (or falls back to uv).
+# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via
+# RUSTFS_PYTHON_BIN=<command>.
+RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh
 IMAGE_NAME ?= rustfs:v1.0.0
 CONTAINER_NAME ?= rustfs-dev
 # Docker build configurations

crates/e2e_test/README.md

@@ -169,7 +169,7 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/checksum_upload_test.rs

@@ -22,7 +22,7 @@ mod tests {
     use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
+    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption};
     use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
     use md5::{Digest as Md5Digest, Md5};
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
@@ -260,6 +260,117 @@ mod tests {
         info!("PASSED: HeadObject returns stored SHA256 digest");
     }
 
+    #[tokio::test]
+    async fn test_head_object_returns_sse_s3_checksum() {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server_with_env(
+            vec![],
+            &[
+                ("RUSTFS_SSE_S3_MASTER_KEY", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="),
+                ("RUSTFS_CONSOLE_ENABLE", "false"),
+            ],
+        )
+        .await
+        .expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-sse-s3-checksum-head";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let put = client
+            .put_object()
+            .bucket(bucket)
+            .key("encrypted.txt")
+            .body(ByteStream::from_static(b"encrypted checksum"))
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 PutObject with CRC32 failed");
+        let expected = put.checksum_crc32().expect("PutObject must return CRC32");
+
+        let head = client
+            .head_object()
+            .bucket(bucket)
+            .key("encrypted.txt")
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 HeadObject failed");
+
+        assert_eq!(head.checksum_crc32(), Some(expected));
+
+        client
+            .copy_object()
+            .bucket(bucket)
+            .key("encrypted-copy.txt")
+            .copy_source(format!("{bucket}/encrypted.txt"))
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .send()
+            .await
+            .expect("SSE-S3 CopyObject failed");
+        let copy_head = client
+            .head_object()
+            .bucket(bucket)
+            .key("encrypted-copy.txt")
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 copied HeadObject failed");
+
+        assert_eq!(copy_head.checksum_crc32(), Some(expected));
+
+        let multipart_key = "encrypted-multipart.txt";
+        let create = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 CreateMultipartUpload with CRC32 failed");
+        let upload_id = create.upload_id().expect("CreateMultipartUpload must return an upload ID");
+        let part = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from_static(b"encrypted multipart checksum"))
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 UploadPart with CRC32 failed");
+        let completed_part = CompletedPart::builder()
+            .part_number(1)
+            .e_tag(part.e_tag().expect("UploadPart must return an ETag"))
+            .checksum_crc32(part.checksum_crc32().expect("UploadPart must return CRC32"))
+            .build();
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(upload_id)
+            .multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
+            .send()
+            .await
+            .expect("SSE-S3 CompleteMultipartUpload with CRC32 failed");
+        let expected_multipart = complete.checksum_crc32().expect("CompleteMultipartUpload must return CRC32");
+        let multipart_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 multipart HeadObject failed");
+
+        assert_eq!(multipart_head.checksum_crc32(), Some(expected_multipart));
+    }
+
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]

crates/e2e_test/src/common.rs

@@ -1469,31 +1469,18 @@ impl RustFSTestClusterEnvironment {
     ///   times out, or cluster service readiness times out.
     pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let binary_path = rustfs_binary_path();
+        self.start_with_binary(&binary_path).await
+    }
+
+    /// Start every cluster node with a specific RustFS binary.
+    ///
+    /// Upgrade compatibility tests use this to initialize a cluster with a
+    /// pinned previous release before replacing nodes with the workspace build.
+    pub async fn start_with_binary(&mut self, binary_path: &Path) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let volumes_arg = self.build_volumes_arg();
 
-        for (i, node) in self.nodes.iter_mut().enumerate() {
-            info!("Starting cluster node {} on {}", i, node.address);
-
-            let mut command = Command::new(&binary_path);
-            command
-                .env("RUSTFS_VOLUMES", &volumes_arg)
-                .env("RUSTFS_ADDRESS", &node.address)
-                .env("RUSTFS_ACCESS_KEY", &self.access_key)
-                .env("RUSTFS_SECRET_KEY", &self.secret_key)
-                .env("RUSTFS_CONSOLE_ENABLE", "false")
-                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
-
-            for (key, value) in &self.extra_env {
-                command.env(key, value);
-            }
-            for (key, value) in &self.node_extra_env[i] {
-                command.env(key, value);
-            }
-            capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
-
-            let process = command.current_dir(&node.data_dir).spawn()?;
-
-            node.process = Some(process);
+        for node_idx in 0..self.nodes.len() {
+            self.spawn_node(node_idx, binary_path, &volumes_arg)?;
         }
 
         for (i, node) in self.nodes.iter().enumerate() {
@@ -1509,20 +1496,46 @@ impl RustFSTestClusterEnvironment {
 
     /// Start one node process using the cluster's existing volume layout.
     pub async fn start_node(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        self.start_node_from_binary(node_idx, &binary_path).await
+    }
+
+    /// Start one stopped cluster node with a specific RustFS binary while
+    /// preserving the cluster's volume layout and that node's data directory.
+    pub async fn start_node_from_binary(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let volumes_arg = self.build_volumes_arg();
+        self.spawn_node(node_idx, binary_path, &volumes_arg)?;
+
+        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        self.wait_for_node_service_ready(node_idx).await?;
+        Ok(())
+    }
+
+    fn spawn_node(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+        volumes_arg: &str,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         self.ensure_node_index(node_idx)?;
         if self.nodes[node_idx].process.is_some() {
             return Err(format!("cluster node {node_idx} is already running").into());
         }
+        if !binary_path.is_file() {
+            return Err(format!("RustFS binary does not exist: {}", binary_path.display()).into());
+        }
 
-        let binary_path = rustfs_binary_path();
-        let volumes_arg = self.build_volumes_arg();
         let log_path = self.node_capture_log_paths[node_idx].clone();
         let node = &mut self.nodes[node_idx];
-        info!("Starting cluster node {} on {}", node_idx, node.address);
+        info!("Starting cluster node {} on {} with {}", node_idx, node.address, binary_path.display());
 
-        let mut command = Command::new(&binary_path);
+        let mut command = Command::new(binary_path);
         command
-            .env("RUSTFS_VOLUMES", &volumes_arg)
+            .env("RUSTFS_VOLUMES", volumes_arg)
             .env("RUSTFS_ADDRESS", &node.address)
             .env("RUSTFS_ACCESS_KEY", &self.access_key)
             .env("RUSTFS_SECRET_KEY", &self.secret_key)
@@ -1539,9 +1552,6 @@ impl RustFSTestClusterEnvironment {
 
         let process = command.current_dir(&node.data_dir).spawn()?;
         node.process = Some(process);
-
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
-        self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
 
@@ -1689,6 +1699,51 @@ impl RustFSTestClusterEnvironment {
         process.wait()?;
         Ok(())
     }
+
+    /// Gracefully stop one cluster node and wait for its process to exit.
+    ///
+    /// This is intentionally separate from [`Self::stop_node`]: the latter is
+    /// a hard kill used by crash-recovery tests, while this path lets RustFS
+    /// complete its normal shutdown hooks before a test restarts the node.
+    pub async fn stop_node_gracefully(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.ensure_node_index(node_idx)?;
+
+        #[cfg(unix)]
+        {
+            let Some(process) = self.nodes[node_idx].process.as_ref() else {
+                return Ok(());
+            };
+            let pid = process.id().to_string();
+            let signal_status = Command::new("kill").args(["-TERM", &pid]).status()?;
+            if !signal_status.success() {
+                return Err(format!("failed to send SIGTERM to cluster node {node_idx} (pid {pid})").into());
+            }
+
+            let mut process = self.nodes[node_idx]
+                .process
+                .take()
+                .ok_or_else(|| format!("cluster node {node_idx} process disappeared while stopping"))?;
+            let deadline = std::time::Instant::now() + Duration::from_secs(45);
+            loop {
+                if let Some(status) = process.try_wait()? {
+                    info!("Cluster node {} stopped gracefully with {}", node_idx, status);
+                    return Ok(());
+                }
+                if std::time::Instant::now() >= deadline {
+                    let _ = process.kill();
+                    let _ = process.wait();
+                    return Err(format!("cluster node {node_idx} did not stop gracefully within 45 seconds").into());
+                }
+                sleep(Duration::from_millis(100)).await;
+            }
+        }
+
+        #[cfg(not(unix))]
+        {
+            let _ = node_idx;
+            Err("graceful cluster-node stop is only supported on Unix E2E hosts".into())
+        }
+    }
 }
 
 impl Drop for RustFSTestClusterEnvironment {

crates/e2e_test/src/degraded_listing_availability_test.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression: an object legally committed at degraded write quorum must stay
+//! listable while a *different* drive is offline.
+//!
+//! On a 4-drive EC 2+2 set, a PUT made while one drive is down persists
+//! `xl.meta` on 3 of 4 drives (write quorum). If a different drive later goes
+//! offline before heal converges, a strict latest-listing quorum of 3 can only
+//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
+//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
+//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
+//! product fix relaxes the listing's required object quorum by the number of
+//! set drives the listing could not consult (see
+//! `latest_listing_required_object_quorum` in
+//! `crates/ecstore/src/store/list_objects.rs`).
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestClusterEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use bytes::Bytes;
+    use std::collections::HashSet;
+    use std::error::Error;
+    use std::time::{Duration, Instant};
+    use tracing::info;
+
+    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+    const BUCKET: &str = "degraded-listing-availability";
+    const OBJECT_COUNT: usize = 8;
+    /// Well under the observed heal-convergence gap (~50s in the CI incident),
+    /// so a listing that only completes after heal restores the missing copy
+    /// still fails this deadline on a regressed build.
+    const LISTING_DEADLINE: Duration = Duration::from_secs(25);
+    const GET_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+    const PUT_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+
+    fn object_key(idx: usize) -> String {
+        format!("degraded-object-{idx:02}")
+    }
+
+    async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
+        let mut keys = HashSet::new();
+        let mut continuation_token: Option<String> = None;
+        loop {
+            let response = client
+                .list_objects_v2()
+                .bucket(BUCKET)
+                .set_continuation_token(continuation_token.clone())
+                .send()
+                .await?;
+            keys.extend(
+                response
+                    .contents()
+                    .iter()
+                    .filter_map(|object| object.key().map(str::to_owned)),
+            );
+            match response.next_continuation_token() {
+                Some(token) => continuation_token = Some(token.to_owned()),
+                None => break,
+            }
+        }
+        Ok(keys)
+    }
+
+    /// 4-node single-drive cluster (EC 2+2, write quorum 3):
+    /// 1. Stop node 1 and PUT objects — each commits on nodes {0, 2, 3} only.
+    /// 2. Stop node 3 (a holder drive), then bring node 1 back before heal can
+    ///    recreate the missing copies there.
+    /// 3. Every object still satisfies read quorum (nodes 0 and 2), so GET
+    ///    must succeed AND ListObjectsV2 must report every key well before
+    ///    heal converges.
+    #[tokio::test]
+    async fn degraded_write_remains_listable_while_a_different_drive_is_offline() -> TestResult {
+        init_logging();
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+        // Listing availability must not depend on heal convergence: disable
+        // the background healers so the degraded objects keep their metadata
+        // on exactly 3 of 4 drives for the whole test.
+        cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
+        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
+        cluster.start().await?;
+        cluster.create_test_bucket(BUCKET).await?;
+        let client = cluster.create_s3_client(0)?;
+
+        info!("stopping node 1 so the uploads commit at degraded write quorum (3 of 4)");
+        cluster.stop_node(1)?;
+        // The first writes after a node drops can see transient 503s while the
+        // survivors notice the dead peer; retry briefly (overwrites of the same
+        // unversioned key are idempotent).
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let body = format!("degraded listing payload {idx}");
+            let deadline = Instant::now() + PUT_RETRY_DEADLINE;
+            loop {
+                let request = client
+                    .put_object()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .body(Bytes::from(body.clone()).into());
+                match request.send().await {
+                    Ok(_) => break,
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded PUT for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded PUT for {key} failed: {error}").into()),
+                }
+            }
+        }
+
+        info!("stopping node 3 (holds a copy) and restoring node 1 (holds none)");
+        cluster.stop_node(3)?;
+        cluster.start_node(1).await?;
+
+        // The first requests after a node drops can see transient 503s while
+        // the survivors notice the dead peer; retry briefly before asserting.
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let deadline = Instant::now() + GET_RETRY_DEADLINE;
+            let body = loop {
+                match client.get_object().bucket(BUCKET).key(&key).send().await {
+                    Ok(response) => break response.body.collect().await?.into_bytes(),
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded GET for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded object {key} failed read quorum GET: {error}").into()),
+                }
+            };
+            assert!(!body.is_empty(), "degraded object {key} should read back at read quorum");
+        }
+
+        let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
+        let deadline = Instant::now() + LISTING_DEADLINE;
+        let listed = loop {
+            let listed = match list_all_keys(&client).await {
+                Ok(keys) => keys,
+                Err(error) if Instant::now() < deadline => {
+                    info!("retrying degraded listing: {error}");
+                    tokio::time::sleep(Duration::from_millis(500)).await;
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            if expected.is_subset(&listed) {
+                break listed;
+            }
+            assert!(
+                Instant::now() < deadline,
+                "objects readable at read quorum stayed missing from ListObjectsV2 for {LISTING_DEADLINE:?}: \
+                 missing={:?} listed={listed:?}",
+                expected.difference(&listed).collect::<Vec<_>>(),
+            );
+            tokio::time::sleep(Duration::from_millis(500)).await;
+        };
+        info!(listed = listed.len(), "degraded objects are listable while node 3 is offline");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -560,18 +560,12 @@ async fn test_multipart_encryption_type(
         .set_parts(Some(completed_parts))
         .build();
 
-    let mut complete_request = s3_client
+    let complete_request = s3_client
         .complete_multipart_upload()
         .bucket(bucket)
         .key(object_key)
         .upload_id(upload_id)
         .multipart_upload(completed_multipart_upload);
-    if matches!(encryption_type, EncryptionType::SSEC) {
-        complete_request = complete_request
-            .sse_customer_algorithm("AES256")
-            .sse_customer_key(sse_c_key.as_ref().unwrap())
-            .sse_customer_key_md5(sse_c_md5.as_ref().unwrap());
-    }
     let _complete_output = complete_request.send().await?;
 
     // Download and verify

crates/e2e_test/src/lib.rs

@@ -348,6 +348,11 @@ mod delete_regression_test;
 #[cfg(test)]
 mod listing_regression_test;
 
+// Cluster regression: objects committed at degraded write quorum must stay
+// listable while a different drive is offline (CI run 33478999853).
+#[cfg(test)]
+mod degraded_listing_availability_test;
+
 // P1 regression: bucket statistics accuracy (rustfs#5615, #5008, #5116, #5055, #3898, #1012)
 #[cfg(test)]
 mod bucket_stats_regression_test;

crates/e2e_test/src/replacement_privileged_e2e_test.rs

@@ -40,10 +40,10 @@ mod tests {
 
     const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E";
     const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E_IN_NAMESPACE";
+    const LOG_DIR_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_LOG_DIR";
     const TARGET_NODE: usize = 1;
     const TARGET_DRIVE: usize = 0;
     const MOUNT_SIZE: &str = "size=128m,mode=0700";
-    const ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS: u64 = 180;
     const REPLACEMENT_RECOVERY_DIR: &str = ".rustfs.sys/buckets/ahm-replacement";
     const REPLACEMENT_INTENT_SUFFIX: &str = "_ahm_replacement_intent.json";
     const REPLACEMENT_COMPLETION_PROOF_SUFFIX: &str = "_ahm_replacement_completion_proof.json";
@@ -142,6 +142,23 @@ mod tests {
             run_command("dmsetup", &["resume", &self.dm_name])
         }
 
+        fn verify_raw_io_is_unavailable(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let mapper = format!("/dev/mapper/{}", self.dm_name);
+            let output = Command::new("dd")
+                .env("LC_ALL", "C")
+                .arg(format!("if={mapper}"))
+                .args(["of=/dev/null", "bs=4096", "count=1", "iflag=direct", "status=none"])
+                .output()?;
+            if output.status.success() {
+                return Err(format!("dm-error target unexpectedly allowed a raw read from {mapper}").into());
+            }
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            if !stderr.contains("Input/output error") {
+                return Err(format!("raw read from dm-error target failed unexpectedly: {stderr}").into());
+            }
+            Ok(())
+        }
+
         fn restore_available(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
             let sectors = run_command_stdout("blockdev", &["--getsz", &self.loop_device])?;
             let linear_table = format!("0 {sectors} linear {} 0", self.loop_device);
@@ -194,6 +211,72 @@ mod tests {
         }
     }
 
+    struct ZramBlockMount {
+        target: PathBuf,
+        device: String,
+        mounted: bool,
+    }
+
+    impl ZramBlockMount {
+        fn reserve(target: &Path) -> Result<Self, Box<dyn Error + Send + Sync>> {
+            if !Path::new("/dev/zram-control").exists() {
+                run_command("modprobe", &["zram"])?;
+            }
+            let device = run_command_stdout("zramctl", &["--find", "--size", "256M"])?;
+            if device.is_empty() {
+                return Err("zramctl --find --size returned an empty device".into());
+            }
+
+            Ok(Self {
+                target: target.to_path_buf(),
+                device,
+                mounted: false,
+            })
+        }
+
+        fn mount_target(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let result = (|| {
+                run_command("mkfs.ext4", &["-F", &self.device])?;
+                let target_arg = path_to_string(&self.target, "zram replacement mount target")?;
+                run_command("mount", &[&self.device, &target_arg])
+            })();
+            if let Err(error) = result {
+                let _ = self.cleanup();
+                return Err(error);
+            }
+            self.mounted = true;
+            Ok(())
+        }
+
+        fn cleanup(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let mut first_error: Option<Box<dyn Error + Send + Sync>> = None;
+            if self.mounted {
+                if let Err(error) = detach_mount(&self.target) {
+                    first_error.get_or_insert(error);
+                } else {
+                    self.mounted = false;
+                }
+            }
+            if !self.device.is_empty() {
+                if let Err(error) = run_command("zramctl", &["--reset", &self.device]) {
+                    first_error.get_or_insert(error);
+                } else {
+                    self.device.clear();
+                }
+            }
+            if let Some(error) = first_error {
+                return Err(error);
+            }
+            Ok(())
+        }
+    }
+
+    impl Drop for ZramBlockMount {
+        fn drop(&mut self) {
+            let _ = self.cleanup();
+        }
+    }
+
     fn checked_command_output(program: &str, args: &[&str]) -> Result<std::process::Output, Box<dyn Error + Send + Sync>> {
         let output = Command::new(program).args(args).output()?;
         if output.status.success() {
@@ -298,6 +381,18 @@ mod tests {
         Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN; unshare exited with status {status}").into())
     }
 
+    fn replacement_node_log_path(
+        cluster_temp_dir: &str,
+        parity: usize,
+        node_index: usize,
+    ) -> Result<PathBuf, Box<dyn Error + Send + Sync>> {
+        let log_dir = std::env::var_os(LOG_DIR_ENV)
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from(cluster_temp_dir));
+        fs::create_dir_all(&log_dir)?;
+        Ok(log_dir.join(format!("replacement-ec{parity}-node{node_index}-{}.log", std::process::id())))
+    }
+
     fn payload(len: usize, seed: u8) -> Vec<u8> {
         let mut next = seed;
         (0..len)
@@ -472,8 +567,20 @@ mod tests {
             if let Some(version_id) = &version.version_id {
                 request = request.version_id(version_id);
             }
-            let response = request.send().await?;
-            let body = response.body.collect().await?.into_bytes();
+            let response = request.send().await.map_err(|error| {
+                format!("body GET failed for {}/{}@{:?}: {error}", version.bucket, version.key, version.version_id)
+            })?;
+            let body = response
+                .body
+                .collect()
+                .await
+                .map_err(|error| {
+                    format!(
+                        "body stream failed for {}/{}@{:?}: {error}",
+                        version.bucket, version.key, version.version_id
+                    )
+                })?
+                .into_bytes();
             assert_eq!(
                 sha256_hex(&body),
                 *expected_sha256,
@@ -582,81 +689,6 @@ mod tests {
         Ok(())
     }
 
-    fn log_tail(log: &str) -> String {
-        let mut lines = log.lines().rev().take(80).collect::<Vec<_>>();
-        lines.reverse();
-        lines.join("\n")
-    }
-
-    fn log_len(path: &Path) -> Result<u64, Box<dyn Error + Send + Sync>> {
-        match fs::metadata(path) {
-            Ok(metadata) => Ok(metadata.len()),
-            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0),
-            Err(error) => Err(format!("failed to stat target node log {path:?}: {error}").into()),
-        }
-    }
-
-    fn log_from_offset(path: &Path, offset: u64) -> Result<String, Box<dyn Error + Send + Sync>> {
-        let log = match fs::read(path) {
-            Ok(log) => log,
-            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
-            Err(error) => return Err(format!("failed to read target node log {path:?}: {error}").into()),
-        };
-        let start = usize::try_from(offset).unwrap_or(usize::MAX).min(log.len());
-        Ok(String::from_utf8_lossy(&log[start..]).into_owned())
-    }
-
-    fn live_disk_loss_scan_completed(log: &str, target_disk: &Path) -> bool {
-        let target = target_disk.to_string_lossy();
-        let mut saw_live_loss = false;
-        for line in log.lines() {
-            if line.contains("Heal auto-scan disk inspection failed")
-                && line.contains("check_failed")
-                && line.contains(target.as_ref())
-            {
-                saw_live_loss = true;
-                continue;
-            }
-            if saw_live_loss && (line.contains("Heal auto disk scanner idle") || line.contains("Heal auto-scan cycle completed"))
-            {
-                return true;
-            }
-        }
-        false
-    }
-
-    fn live_disk_loss_scan_completed_from_path(
-        log_path: &Path,
-        start_offset: u64,
-        target_disk: &Path,
-    ) -> Result<bool, Box<dyn Error + Send + Sync>> {
-        Ok(live_disk_loss_scan_completed(&log_from_offset(log_path, start_offset)?, target_disk))
-    }
-
-    async fn wait_for_live_disk_loss_observation(
-        log_path: &Path,
-        target_disk: &Path,
-        start_offset: u64,
-        timeout_secs: u64,
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let deadline = Instant::now() + Duration::from_secs(timeout_secs);
-        let mut tick = interval(Duration::from_secs(1));
-        loop {
-            if live_disk_loss_scan_completed_from_path(log_path, start_offset, target_disk)? {
-                return Ok(());
-            }
-            if Instant::now() >= deadline {
-                let log = log_from_offset(log_path, start_offset)?;
-                return Err(format!(
-                    "scanner did not finish a live target-loss scan for {target_disk:?} within {timeout_secs}s; log tail:\n{}",
-                    log_tail(&log)
-                )
-                .into());
-            }
-            tick.tick().await;
-        }
-    }
-
     fn cluster_status_is_definitive(status: &serde_json::Value) -> Result<bool, Box<dyn Error + Send + Sync>> {
         status["cluster"]["definitive"]
             .as_bool()
@@ -707,6 +739,13 @@ mod tests {
             .collect()
     }
 
+    fn is_transient_recovery_version_absence(error: &(dyn Error + 'static)) -> bool {
+        matches!(
+            error.downcast_ref::<rustfs_filemeta::Error>(),
+            Some(rustfs_filemeta::Error::FileVersionNotFound)
+        )
+    }
+
     fn incomplete_versions(
         target_disk: &Path,
         versions: &[BaselineVersion],
@@ -714,7 +753,21 @@ mod tests {
         let mut missing = BTreeSet::new();
         for version in versions {
             let actual =
-                census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref())?;
+                match census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref()) {
+                    Ok(actual) => actual,
+                    // During replacement recovery, xl.meta may arrive before this
+                    // particular historical version. The generic census helper
+                    // correctly reports that as an error; this progress poll must
+                    // instead wait for the version to be restored.
+                    Err(error) if is_transient_recovery_version_absence(error.as_ref()) => {
+                        missing.insert(format!(
+                            "{}/{}@{:?}: version metadata not yet present on replacement",
+                            version.bucket, version.key, version.version_id
+                        ));
+                        continue;
+                    }
+                    Err(error) => return Err(error),
+                };
             if !actual.matches_manifest(&version.expected) {
                 missing.insert(format!("{}/{}@{:?}: {actual:?}", version.bucket, version.key, version.version_id));
             }
@@ -822,13 +875,15 @@ mod tests {
 
         let mut mount_ns = MountNamespaceGuard::new()?;
         let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(3, 4)).await?;
-        let target_log_path = PathBuf::from(&cluster.temp_dir).join(format!("replacement-node{TARGET_NODE}.log"));
-        cluster.set_node_capture_log_path(TARGET_NODE, target_log_path.to_string_lossy())?;
+        for node_index in 0..cluster.nodes.len() {
+            let node_log_path = replacement_node_log_path(&cluster.temp_dir, parity, node_index)?;
+            cluster.set_node_capture_log_path(node_index, node_log_path.to_string_lossy())?;
+        }
         let target_disk = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dirs[TARGET_DRIVE]);
-        // Each drive below is an independent tmpfs mount, so this privileged
-        // path must exercise the production distinct-device/readiness fences.
+        // The blank target uses a temporary zram block device, so the
+        // replacement readiness fence sees no root or sibling alias.
         cluster.extra_env.retain(|(key, _)| key != "RUSTFS_UNSAFE_BYPASS_DISK_CHECK");
-        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-faultable-images");
+        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-block-images");
         let mut target_mount = None;
         for (node_index, node) in cluster.nodes.iter().enumerate() {
             for (drive_index, drive) in node.data_dirs.iter().enumerate() {
@@ -845,6 +900,7 @@ mod tests {
             }
         }
         let mut target_mount = target_mount.ok_or("target drive was not mounted with the faultable block fixture")?;
+        let mut replacement_mount = ZramBlockMount::reserve(&target_disk)?;
 
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
         cluster.set_env("RUSTFS_SCANNER_ENABLED", "true");
@@ -852,28 +908,34 @@ mod tests {
         cluster.set_env("RUSTFS_SCANNER_CYCLE", "1");
         cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
         cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
-        cluster.set_node_env(TARGET_NODE, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
+        for node_index in 0..cluster.nodes.len() {
+            cluster.set_node_env(node_index, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
+        }
         cluster.start().await?;
 
         let clients = cluster.create_all_clients()?;
-        let versions = seed_baseline(&clients[0], &target_disk).await?;
-        verify_bodies(&clients[0], &versions).await?;
+        let versions = seed_baseline(&clients[0], &target_disk)
+            .await
+            .map_err(|error| format!("pre-fault baseline seeding failed: {error}"))?;
+        verify_bodies(&clients[0], &versions)
+            .await
+            .map_err(|error| format!("pre-fault body verification failed: {error}"))?;
 
-        let live_loss_log_offset = log_len(&target_log_path)?;
-        target_mount.make_unavailable()?;
-        wait_for_live_disk_loss_observation(
-            &target_log_path,
-            &target_disk,
-            live_loss_log_offset,
-            ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS,
-        )
-        .await?;
-        assert_no_replacement_status_records(&cluster, &target_disk).await?;
-        assert_no_replacement_admission_artifacts(&cluster, &target_disk)?;
+        target_mount
+            .make_unavailable()
+            .map_err(|error| format!("failed to install the dm-error target: {error}"))?;
+        target_mount
+            .verify_raw_io_is_unavailable()
+            .map_err(|error| format!("dm-error target was not proven by a direct raw read: {error}"))?;
+        assert_no_replacement_status_records(&cluster, &target_disk)
+            .await
+            .map_err(|error| format!("live-fault replacement status check failed: {error}"))?;
+        assert_no_replacement_admission_artifacts(&cluster, &target_disk)
+            .map_err(|error| format!("live-fault replacement artifact check failed: {error}"))?;
 
-        cluster.stop_node(TARGET_NODE)?;
+        cluster.stop_node_gracefully(TARGET_NODE).await?;
         target_mount.cleanup()?;
-        mount_ns.mount_tmpfs(&target_disk, &format!("rustfs-e2e-p{parity}-replacement"))?;
+        replacement_mount.mount_target()?;
         let missing_before_restart = incomplete_versions(&target_disk, &versions)?;
         assert_eq!(
             missing_before_restart.len(),
@@ -882,46 +944,26 @@ mod tests {
         );
         cluster.start_node(TARGET_NODE).await?;
 
-        wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
-        verify_bodies(&clients[0], &versions).await?;
+        let recovery_result = async {
+            wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
+            verify_bodies(&clients[0], &versions).await
+        }
+        .await;
+        let stop_result = cluster.stop_node_gracefully(TARGET_NODE).await;
+        let replacement_cleanup_result = replacement_mount.cleanup();
 
-        Ok(())
-    }
+        if let Err(error) = recovery_result {
+            if let Err(stop_error) = stop_result {
+                info!(%stop_error, "replacement target stop failed while preserving recovery failure");
+            }
+            if let Err(cleanup_error) = replacement_cleanup_result {
+                info!(%cleanup_error, "replacement zram cleanup failed while preserving recovery failure");
+            }
+            return Err(error);
+        }
+        stop_result?;
+        replacement_cleanup_result?;
 
-    #[test]
-    fn live_loss_barrier_requires_scanner_failure_after_log_offset() -> Result<(), Box<dyn Error + Send + Sync>> {
-        let target = Path::new("/mnt/target");
-        assert!(live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto-scan cycle completed",
-            target
-        ));
-        assert!(live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "Heal auto disk scanner idle\nHeal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "event=disk_health_check_failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/other disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        let path = std::env::temp_dir().join(format!("rustfs-replacement-scan-{}.log", std::process::id()));
-        let stale =
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
-        fs::write(&path, stale)?;
-        let offset = log_len(&path)?;
-        assert!(!live_disk_loss_scan_completed_from_path(&path, offset, target)?);
-        let fresh =
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
-        fs::write(&path, format!("{stale}{fresh}"))?;
-        assert!(live_disk_loss_scan_completed_from_path(&path, offset, target)?);
-        fs::remove_file(path)?;
         Ok(())
     }
 
@@ -954,6 +996,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn recovery_census_only_treats_missing_version_as_transient() {
+        let missing_version: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileVersionNotFound);
+        let missing_file: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileNotFound);
+
+        assert!(is_transient_recovery_version_absence(missing_version.as_ref()));
+        assert!(!is_transient_recovery_version_absence(missing_file.as_ref()));
+    }
+
     #[tokio::test]
     async fn completion_poll_samples_census_before_status() {
         let order = std::rc::Rc::new(std::cell::RefCell::new(Vec::new()));

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,14 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
 };
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::task::JoinSet;
+use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
@@ -28,6 +31,14 @@ const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
 const VERSIONED_BUCKET: &str = "upgrade-versioned-data";
+const MIXED_BUCKET: &str = "upgrade-mixed-version-data";
+const MIXED_NODE_COUNT: usize = 4;
+const MULTIPART_WORKERS: usize = 16;
+const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
+// Peers keep a restarted node's drive in Suspect/Returning for roughly
+// probe_interval (2s) x success_threshold (3) after it comes back; 30s
+// comfortably covers that window plus CI scheduling jitter.
+const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
@@ -103,6 +114,132 @@ async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<
     Ok(())
 }
 
+fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("mixed-upgrade-node-{node_idx}.log"));
+        cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
+    let mut tasks = JoinSet::new();
+    for worker in 0..MULTIPART_WORKERS {
+        let client = clients[worker % clients.len()].clone();
+        let phase = phase.to_string();
+        tasks.spawn(async move {
+            let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
+            for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
+                let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
+                let part = vec![u8::try_from(worker)?; 64 * 1024];
+                write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
+                keys.push(key);
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
+        });
+    }
+
+    let mut keys = Vec::with_capacity(MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER);
+    while let Some(result) = tasks.join_next().await {
+        keys.extend(result??);
+    }
+    Ok(keys)
+}
+
+/// Assert that `client` eventually lists exactly `expected` objects under
+/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
+///
+/// A single-snapshot assertion here is racy by construction: each phase both
+/// writes and lists within seconds of a node restart. While a peer still holds
+/// the restarted node's drive in Suspect/Returning, strict-quorum listing
+/// consults only the remaining three drives and drops any object that was
+/// itself legally written at write quorum (3/4 drives) during an earlier
+/// node's identical post-restart window — its xl.meta is then visible on only
+/// two of the three consulted drives, below the required object quorum of
+/// three. GET still succeeds for such objects; only the listing under-counts
+/// until drive health converges. A genuine upgrade data-loss regression still
+/// fails after the deadline.
+async fn wait_for_phase_listing(client: &Client, phase: &str, expected: usize, context: &str) -> TestResult {
+    let deadline = Instant::now() + LISTING_CONVERGENCE_TIMEOUT;
+    loop {
+        let listed = client
+            .list_objects_v2()
+            .bucket(MIXED_BUCKET)
+            .prefix(format!("{phase}/"))
+            .send()
+            .await?;
+        let count = listed.contents().len();
+        if count == expected {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!(
+                "{context}: listing under {phase}/ returned {count} of {expected} objects even after {}s of post-restart convergence",
+                LISTING_CONVERGENCE_TIMEOUT.as_secs()
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn exercise_mixed_cluster(
+    cluster: &RustFSTestClusterEnvironment,
+    phase: &str,
+    current_node: usize,
+    previous_node: usize,
+) -> TestResult {
+    let clients = cluster.create_all_clients()?;
+    let current_client = &clients[current_node];
+    let previous_client = &clients[previous_node];
+
+    let current_key = format!("{phase}/written-by-current");
+    let current_body = format!("{phase}: current RustFS build").into_bytes();
+    current_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&current_key)
+        .body(ByteStream::from(current_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(previous_client, MIXED_BUCKET, &current_key, None).await?.1, current_body);
+
+    let previous_key = format!("{phase}/written-by-previous");
+    let previous_body = format!("{phase}: previous RustFS release").into_bytes();
+    previous_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&previous_key)
+        .body(ByteStream::from(previous_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
+
+    let multipart_keys = write_multipart_load(&clients, phase).await?;
+    let expected_count = multipart_keys.len() + 2;
+    for (label, client) in [("current", current_client), ("previous", previous_client)] {
+        wait_for_phase_listing(
+            client,
+            phase,
+            expected_count,
+            &format!("the {label} RustFS version must stream the complete mixed-version listing"),
+        )
+        .await?;
+    }
+
+    let last_multipart_key = format!("{phase}/multipart/{:02}/{:02}", MULTIPART_WORKERS - 1, MULTIPART_UPLOADS_PER_WORKER - 1);
+    assert_eq!(
+        read_object(previous_client, MIXED_BUCKET, &last_multipart_key, None).await?.1,
+        vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -252,3 +389,43 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
 
     Ok(())
 }
+
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let current_binary = rustfs_binary_path();
+    let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
+    cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn");
+    configure_cluster_logs(&mut cluster)?;
+    cluster.start_with_binary(&previous_binary).await?;
+    cluster.create_test_bucket(MIXED_BUCKET).await?;
+
+    cluster.stop_node(0)?;
+    cluster.start_node_from_binary(0, &current_binary).await?;
+    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+
+    for node_idx in [1, 2] {
+        cluster.stop_node(node_idx)?;
+        cluster.start_node_from_binary(node_idx, &current_binary).await?;
+    }
+    exercise_mixed_cluster(&cluster, "one-previous-node", 0, 3).await?;
+
+    cluster.stop_node(3)?;
+    cluster.start_node_from_binary(3, &current_binary).await?;
+
+    for (node_idx, client) in cluster.create_all_clients()?.iter().enumerate() {
+        for phase in ["one-current-node", "one-previous-node"] {
+            wait_for_phase_listing(
+                client,
+                phase,
+                MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2,
+                &format!("node {node_idx}: the homogeneous current cluster must preserve every object"),
+            )
+            .await?;
+        }
+    }
+
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -75,6 +75,10 @@ pub mod bucket {
                 delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
                 inspect_transition_transaction_for_operator,
             };
+            #[cfg(feature = "test-util")]
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionTransactionRecoveryStats, recover_transition_transaction_records,
+            };
         }
 
         pub mod evaluator {
@@ -99,6 +103,17 @@ pub mod bucket {
             pub use crate::bucket::lifecycle::tier_delete_journal::{
                 persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
             };
+
+            #[cfg(feature = "test-util")]
+            pub mod test_util {
+                /// Model a single-node, all-v6 fleet after its capability probe has completed.
+                ///
+                /// Call this only once while constructing an isolated test store, before any
+                /// tier-delete journal permit or background worker can be active.
+                pub fn install_all_v6_fleet_capability_proof() {
+                    crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+                }
+            }
         }
 
         pub mod tier_last_day_stats {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -485,6 +485,19 @@ impl BucketTargetSys {
         mutex
     }
 
+    /// Snapshot the heartbeat-tracked health of `url`'s endpoint.
+    ///
+    /// Returns `None` when the heartbeat has never seen the endpoint. Unlike
+    /// [`Self::is_offline`] this deliberately does not call `init_hc`: a caller
+    /// that only reports metrics must not create health entries as a side
+    /// effect, or merely rendering a status page would mark an unknown peer
+    /// online.
+    pub async fn endpoint_health(&self, url: &Url) -> Option<EpHealth> {
+        let key = endpoint_health_key(url);
+        let health_map = self.h_mutex.read().await;
+        health_map.get(&key).cloned()
+    }
+
     pub async fn is_offline(&self, url: &Url) -> bool {
         let key = endpoint_health_key(url);
         {

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
     DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
 };
 use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
-use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
+use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
 use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
 use crate::bucket::object_lock::ObjectLockApi;
 use crate::bucket::versioning::VersioningApi as _;
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
 use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
 use crate::error::Error;
 use crate::error::StorageError;
-use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
+use crate::error::{
+    is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
+    is_network_or_host_down,
+};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
 use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
 use crate::services::tier::{
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
     }
 }
 
-async fn delete_free_version_remote_object(
+async fn acquire_free_version_tier_lease(
     oi: &ObjectInfo,
     tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
-) -> Result<(), std::io::Error> {
+) -> Result<(TierOperationLease, bool), std::io::Error> {
     let version_id_exact = validate_transition_remote_version(oi)?;
     let identity = tier_destination_id_from_metadata(&oi.user_defined)?
         .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
-    delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+    let lease =
+        TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
+            .await
+            .map_err(std::io::Error::other)?;
+    Ok((lease, version_id_exact))
+}
+
+async fn delete_free_version_remote_object_with_lease(
+    oi: &ObjectInfo,
+    lease: &TierOperationLease,
+    version_id_exact: bool,
+) -> Result<(), std::io::Error> {
+    delete_object_from_remote_tier_with_lease_idempotent(
         &oi.transitioned_object.name,
         &oi.transitioned_object.version_id,
-        &oi.transitioned_object.tier,
-        identity,
-        tier_config_mgr,
+        lease,
         version_id_exact,
     )
     .await?;
     Ok(())
 }
 
+fn free_version_physical_topology_generation(api: &ECStore) -> String {
+    let mut hasher = Sha256::new();
+    for pool in &api.pools {
+        hasher.update(pool.pool_idx.to_be_bytes());
+        hasher.update(pool.disk_set.len().to_be_bytes());
+        for set in &pool.disk_set {
+            hasher.update(set.set_index.to_be_bytes());
+        }
+    }
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
+    if candidate.transitioned_object.tier != expected.transitioned_object.tier
+        || candidate.transitioned_object.name != expected.transitioned_object.name
+    {
+        return Ok(false);
+    }
+    let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
+    let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
+    if candidate_identity != expected_identity {
+        return Ok(false);
+    }
+    if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+        || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+    {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version remote version state is unknown",
+        ));
+    }
+    Ok(candidate.transition_version_state == expected.transition_version_state
+        && candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
+}
+
+async fn scan_exact_free_version_targets(
+    api: &ECStore,
+    oi: &ObjectInfo,
+    local_object: &str,
+) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
+    let mut targets = Vec::new();
+    for pool in &api.pools {
+        for set in &pool.disk_set {
+            let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
+                Ok(Some(versions)) => versions,
+                Ok(None) => continue,
+                Err(err) if is_err_strict_volume_not_found(&err) => continue,
+                Err(err) => return Err(std::io::Error::other(err)),
+            };
+            for version in versions.versions.iter().chain(versions.free_versions.iter()) {
+                let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
+                if free_version_remote_tuple_matches(&candidate, oi)? {
+                    if candidate.transitioned_object.free_version {
+                        // Data movement can leave the same remote tuple in
+                        // several physical pools. Ordinary deletion assigns a
+                        // fresh local free-version UUID to each copy, but all
+                        // of those markers own the same idempotent remote
+                        // DELETE. Consume them together while holding every
+                        // physical object lock; treating their local UUIDs as
+                        // conflicting would strand cleanup forever.
+                        let mut actual = version.clone();
+                        actual.name = local_object.to_string();
+                        targets.push((Arc::clone(set), actual));
+                    } else {
+                        return Err(std::io::Error::new(
+                            std::io::ErrorKind::WouldBlock,
+                            "a live transitioned source still references the free-version remote tuple",
+                        ));
+                    }
+                }
+            }
+        }
+    }
+    Ok(targets)
+}
+
+fn free_version_cleanup_fences_current(
+    topology_generation: &str,
+    api: &ECStore,
+    bucket_guard: &rustfs_lock::NamespaceLockGuard,
+    object_guards: &[crate::store::ObjectLockDiagGuard],
+    lease: &TierOperationLease,
+    cancel: &CancellationToken,
+    deadline: tokio::time::Instant,
+) -> bool {
+    !cancel.is_cancelled()
+        && tokio::time::Instant::now() < deadline
+        && !bucket_guard.is_lock_lost()
+        && object_guards.iter().all(|guard| !guard.is_lock_lost())
+        && lease.is_current_generation()
+        && free_version_physical_topology_generation(api) == topology_generation
+}
+
+async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
+    const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
+
+    let topology_generation = free_version_physical_topology_generation(&api);
+    let bucket_guard = api
+        .acquire_bucket_lifecycle_read_lock(&oi.bucket)
+        .await
+        .map_err(std::io::Error::other)?;
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
+    let local_object = encode_dir_object(&oi.name);
+    let object_guards = api
+        .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
+        .await
+        .map_err(std::io::Error::other)?;
+    let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if targets.is_empty() {
+        return Ok(false);
+    }
+
+    let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence is invalid before remote delete",
+        ));
+    }
+    tokio::select! {
+        _ = cancel.cancelled() => {
+            return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
+        }
+        result = tokio::time::timeout_at(
+            deadline,
+            delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
+        ) => {
+            result
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
+        }
+    }
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        // Remote DELETE is idempotent, but a changed fence makes the local
+        // outcome ambiguous. Keep every marker for a fully fenced retry.
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence changed after remote delete",
+        ));
+    }
+
+    let mut first_error = None;
+    for (set, actual) in &targets {
+        let mut delete_request = FileInfo {
+            name: local_object.clone(),
+            version_id: actual.version_id,
+            ..Default::default()
+        };
+        delete_request.set_tier_free_version();
+        if let Err(err) = set
+            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
+            .await
+            && first_error.is_none()
+        {
+            first_error = Some(std::io::Error::other(err));
+        }
+    }
+    let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if !remaining.is_empty() {
+        return Err(first_error.unwrap_or_else(|| {
+            std::io::Error::new(
+                std::io::ErrorKind::WouldBlock,
+                "tier free-version cleanup remained on at least one physical set",
+            )
+        }));
+    }
+    if let Some(err) = first_error {
+        return Err(err);
+    }
+    Ok(true)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn delete_free_version_remote_object(
+    oi: &ObjectInfo,
+    tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
+) -> Result<(), std::io::Error> {
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
+}
+
 #[allow(
     dead_code,
     reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -618,8 +813,11 @@ where
     F: FnOnce() -> Fut,
     Fut: std::future::Future<Output = T>,
 {
-    delete_free_version_remote_object(oi, tier_config_mgr).await?;
-    Ok(delete_local().await)
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
+    let result = delete_local().await;
+    drop(lease);
+    Ok(result)
 }
 
 struct NewerNoncurrentTask {
@@ -690,6 +888,10 @@ impl ExpiryState {
         usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
     }
 
+    pub fn active_tasks(&self) -> usize {
+        usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
+    }
+
     fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
         let queued = wrkr.try_send(Some(task)).is_ok();
         if queued {
@@ -826,7 +1028,7 @@ impl ExpiryState {
     }
 
     pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let expiry_state = api.ctx.expiry_state();
         if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
             return;
         }
@@ -867,7 +1069,7 @@ impl ExpiryState {
         stats: Arc<ExpiryStats>,
         recovery_notify: Arc<Notify>,
     ) {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
             static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
             FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
         });
@@ -968,119 +1170,33 @@ impl ExpiryState {
                     else if v.as_any().is::<FreeVersionTask>() {
                         let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
                         let oi = v.0.clone();
-                        if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
-                            recovery_notify.notify_one();
-                            debug!(
-                                bucket = %oi.bucket,
-                                object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
-                                error = ?err,
-                                event = EVENT_LIFECYCLE_WORKER_STATE,
-                                component = LOG_COMPONENT_ECSTORE,
-                                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                reason = "remote_tier_delete_failed",
-                                "Lifecycle worker skipped remote tier delete"
-                            );
-                            continue;
-                        }
-
-                        let local_object = encode_dir_object(&oi.name);
-                        let mut fi = FileInfo {
-                            name: local_object.clone(),
-                            version_id: oi.version_id,
-                            ..Default::default()
-                        };
-                        // This removes an existing internal cleanup marker. Keeping
-                        // `deleted` false makes duplicate tasks return not-found
-                        // instead of creating an ordinary delete marker.
-                        fi.set_tier_free_version();
-
-                        let mut deleted_locally = false;
-                        for pool in &api.pools {
-                            let set = pool.get_disks_by_key(&local_object);
-                            let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
-                                Ok(lock) => lock,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        pool_index = pool.pool_idx,
-                                        set_index = set.set_index,
-                                        error = ?err,
-                                        reason = "local_free_version_lock_failed",
-                                        "Lifecycle worker failed to create local free-version cleanup lock"
-                                    );
-                                    continue;
-                                }
-                            };
-                            let _object_lock_guard =
-                                match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
-                                    Ok(guard) => guard,
-                                    Err(err) => {
-                                        recovery_notify.notify_one();
-                                        debug!(
-                                            event = EVENT_LIFECYCLE_WORKER_STATE,
-                                            component = LOG_COMPONENT_ECSTORE,
-                                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                            bucket = %oi.bucket,
-                                            object = %oi.name,
-                                            pool_index = pool.pool_idx,
-                                            set_index = set.set_index,
-                                            error = ?err,
-                                            reason = "local_free_version_lock_failed",
-                                            "Lifecycle worker failed to acquire local free-version cleanup lock"
-                                        );
-                                        continue;
-                                    }
-                                };
-                            match set
-                                .delete_object_version(&oi.bucket, &local_object, &fi, false)
-                                .await
-                            {
-                                Ok(()) => {
-                                    deleted_locally = true;
-                                    break;
-                                }
-                                Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        remote_object = %oi.transitioned_object.name,
-                                        remote_version_id = %oi.transitioned_object.version_id,
-                                        tier = %oi.transitioned_object.tier,
-                                        error = ?err,
-                                        reason = "local_free_version_delete_failed",
-                                        "Lifecycle worker failed local free-version cleanup"
-                                    );
-                                    break;
-                                }
-                            }
-                        }
-
-                        if !deleted_locally {
-                            debug!(
+                        match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
+                            Ok(true) => {}
+                            Ok(false) => debug!(
                                 event = EVENT_LIFECYCLE_WORKER_STATE,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
                                 bucket = %oi.bucket,
                                 object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
                                 reason = "local_free_version_missing",
-                                "Lifecycle worker could not find transitioned free version locally"
-                            );
+                                "Lifecycle worker found that the exact free-version was already absent"
+                            ),
+                            Err(err) => {
+                                recovery_notify.notify_one();
+                                debug!(
+                                    event = EVENT_LIFECYCLE_WORKER_STATE,
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                    bucket = %oi.bucket,
+                                    object = %oi.name,
+                                    remote_object = %oi.transitioned_object.name,
+                                    remote_version_id = %oi.transitioned_object.version_id,
+                                    tier = %oi.transitioned_object.tier,
+                                    error = ?err,
+                                    reason = "free_version_exact_cleanup_deferred",
+                                    "Lifecycle worker retained the exact free-version for a fenced retry"
+                                );
+                            }
                         }
                     }
                     else {
@@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer(
     RecoveredFreeVersionEnqueueObserverGuard
 }
 
-pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
-    let expiry_state = runtime_sources::expiry_state_handle();
+pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
+    let expiry_state = api.ctx.expiry_state();
     let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
 
     #[cfg(test)]
@@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
     }
 
     Some(tokio::spawn(async move {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
+        let expiry_state = api.ctx.expiry_state();
         run_tier_free_version_recovery_loop(
             cancel_token,
             expiry_state,
@@ -6229,9 +6345,18 @@ mod tests {
             rustfs_utils::crypto::hex(old_identity),
         );
         oi.user_defined = Arc::new(metadata.clone());
+        let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
         delete_free_version_remote_object_then(&oi, &manager, {
             let local_delete_calls = Arc::clone(&local_delete_calls);
+            let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
+            let manager = manager.clone();
             move || async move {
+                assert_eq!(
+                    crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+                    1,
+                    "the identity-bound tier lease must span the exact local marker delete"
+                );
+                lease_observed_during_local_delete.store(true, Ordering::Relaxed);
                 local_delete_calls.fetch_add(1, Ordering::Relaxed);
             }
         })
@@ -6239,6 +6364,12 @@ mod tests {
         .expect("matching destination identity should allow idempotent remote cleanup");
         assert_eq!(old_backend.remove_count().await, 1);
         assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
+        assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
+        assert_eq!(
+            crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+            0,
+            "the tier lease should be released after the local marker delete completes"
+        );
 
         let mut single_prefix_metadata = HashMap::new();
         single_prefix_metadata.insert(
@@ -6472,6 +6603,7 @@ mod tests {
         let state = ExpiryState::new();
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6480,6 +6612,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         let err = state
@@ -6620,6 +6753,7 @@ mod tests {
         let state = ExpiryState::new_with_unconsumed_worker_channel(1);
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6628,6 +6762,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         state
@@ -6759,7 +6894,7 @@ mod tests {
         };
 
         assert!(
-            super::enqueue_recovered_free_version(oi).await,
+            super::enqueue_recovered_free_version(&ecstore, oi).await,
             "the resized production worker queue should accept the task"
         );
         stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -6875,12 +7010,12 @@ mod tests {
             .await
             .expect("free-version task should reach the worker");
         tokio::time::timeout(StdDuration::from_secs(30), async {
-            while remote_backend.remove_count().await == 0 {
+            while stats.active_tasks() == 0 {
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("worker should complete remote cleanup before taking the local lock");
+        .expect("worker should mark the cleanup task active before the lock assertion");
         let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
             while stats.active_tasks() != 0 {
                 tokio::task::yield_now().await;
@@ -6889,7 +7024,12 @@ mod tests {
         .await;
         assert!(
             completed_while_locked.is_err(),
-            "local cleanup must wait while a competing object writer owns the namespace lock"
+            "the cleanup task must wait while a competing object writer owns the namespace lock"
+        );
+        assert_eq!(
+            remote_backend.remove_count().await,
+            0,
+            "the remote tuple must not be deleted before the all-physical namespace fence is acquired"
         );
         for disk_path in &disk_paths {
             assert!(
@@ -6900,6 +7040,13 @@ mod tests {
         }
 
         drop(object_lock_guard);
+        tokio::time::timeout(StdDuration::from_secs(30), async {
+            while remote_backend.remove_count().await == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("worker should delete the remote tuple after acquiring the released namespace fence");
         tx.send(None).await.expect("worker stop signal should be delivered");
         worker.await.expect("free-version worker should stop cleanly");
 
@@ -6995,6 +7142,7 @@ mod tests {
             .next()
             .expect("seeded free version should be recoverable");
         let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
+        let ordinary_marker_mod_time = OffsetDateTime::now_utc();
 
         for disk_path in &disk_paths {
             let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7017,7 +7165,7 @@ mod tests {
                     name: object.to_string(),
                     version_id: Some(stale_version_id),
                     deleted: true,
-                    mod_time: Some(OffsetDateTime::now_utc()),
+                    mod_time: Some(ordinary_marker_mod_time),
                     ..Default::default()
                 })
                 .expect("same-ID ordinary marker should replace the stale free version");
@@ -7031,6 +7179,13 @@ mod tests {
             .expect("same-ID ordinary marker metadata should be written");
         }
 
+        assert!(
+            !super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
+                .await
+                .expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
+            "the stale free-version task must not report local cleanup"
+        );
+
         let state = ExpiryState::new();
         let (stats, recovery_notify) = {
             let state = state.read().await;
@@ -11522,7 +11677,7 @@ mod tests {
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
+    async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11530,6 +11685,7 @@ mod tests {
             .expect("mock tier lease should be available")
             .backend_identity();
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "legacy-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -11538,25 +11694,28 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Unknown,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
+        crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
+            .await
+            .expect("legacy unknown journal should remain byte-compatible and persistable");
         let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
             .await
-            .expect_err("unknown journal state must fail before backend IO");
+            .expect_err("legacy unknown journal must be quarantined before backend IO");
 
-        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
         assert_eq!(backend.remove_count().await, 0);
     }
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_deletes_confirmed_exact_provider_token() {
+    async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
             .await
             .expect("mock tier lease should be available");
-        let identity = lease.backend_identity();
         backend
             .set_put_remote_version(Some("provider-version-token".to_string()))
             .await;
@@ -11570,34 +11729,30 @@ mod tests {
             .expect("confirmed remote candidate should be seeded");
         backend.set_remove_failure(true);
         backend.set_reject_non_empty_remote_versions(true);
-        let je = Jentry {
-            obj_name: "remote/object".to_string(),
-            version_id: "provider-version-token".to_string(),
-            tier_name: "WARM".to_string(),
-            backend_identity: Some(identity),
-            version_id_exact: true,
-            version_state: rustfs_filemeta::TransitionVersionState::Exact,
-            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
-            source: None,
-        };
-
-        crate::set_disk::cleanup_rejected_transition_upload_durably(
+        let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
             &lease,
-            &je.obj_name,
-            &je.version_id,
+            "remote/object",
+            "provider-version-token",
             true,
             Some(ecstore.clone()),
         )
         .await
-        .expect("failed immediate cleanup should remain durable in the journal");
-        assert!(backend.contains(&je.obj_name).await);
+        .expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
+        assert_eq!(err.kind(), std::io::ErrorKind::Other);
+        assert!(backend.contains("remote/object").await);
 
         backend.set_remove_failure(false);
-        crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
-            .await
-            .expect("identity-bound exact journal must retry confirmed candidate cleanup");
+        crate::set_disk::cleanup_rejected_transition_upload_durably(
+            &lease,
+            "remote/object",
+            "provider-version-token",
+            true,
+            Some(ecstore),
+        )
+        .await
+        .expect("the transaction retry must delete the same confirmed candidate");
 
-        assert!(!backend.contains(&je.obj_name).await);
+        assert!(!backend.contains("remote/object").await);
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(
             backend.remove_versions().await,
@@ -11760,11 +11915,14 @@ mod tests {
         };
         let mut recovery_rx = recovery_rx.lock().await;
         assert!(
-            super::enqueue_recovered_free_version(ObjectInfo {
-                bucket: "prefill".to_string(),
-                name: "prefill".to_string(),
-                ..Default::default()
-            })
+            super::enqueue_recovered_free_version(
+                &ecstore,
+                ObjectInfo {
+                    bucket: "prefill".to_string(),
+                    name: "prefill".to_string(),
+                    ..Default::default()
+                },
+            )
             .await,
             "the production recovery queue should accept its first task"
         );

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum DurableIlmRecordKind {
     TierDeleteJournal,
+    TierDeleteDispatchManifest,
     TransitionTransaction,
     ManualTransitionJob,
     ManualTransitionScope,
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
     max_record_size: 64 * 1024,
     kind: DurableIlmRecordKind::TierDeleteJournal,
 };
+pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-journal-v6",
+    prefix: "ilm/tier-delete-journal-v6/",
+    max_record_size: 64 * 1024,
+    kind: DurableIlmRecordKind::TierDeleteJournal,
+};
+pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-dispatch-manifest",
+    prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
+    max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
+    kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
+};
 pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
     name: "transition-transaction",
     prefix: "ilm/transition-transactions/records",
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
+    TIER_DELETE_JOURNAL_V6_NAMESPACE,
+    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
     TRANSITION_TRANSACTION_NAMESPACE,
     MANUAL_TRANSITION_JOB_NAMESPACE,
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         content_sha256: String,
         identity_sha256: String,
         committed: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        dispatch_identity_sha256: Option<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        state: Option<super::tier_sweeper::TierDeleteJournalState>,
+    },
+    TierDeleteDispatchManifest {
+        content_sha256: String,
+        identity_sha256: String,
+        state: tier_delete_journal::TierDeleteDispatchManifestState,
     },
     TransitionTransaction {
         content_sha256: String,
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
     pub(crate) fn content_sha256(&self) -> &str {
         match self {
             Self::TierDeleteJournal { content_sha256, .. }
+            | Self::TierDeleteDispatchManifest { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
     }
 
     pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
+        for checkpoint in [self, next] {
+            if let Self::TierDeleteJournal {
+                committed,
+                dispatch_identity_sha256,
+                state,
+                ..
+            } = checkpoint
+                && (state.is_some() != dispatch_identity_sha256.is_some()
+                    || state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
+            {
+                return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
+            }
+        }
         if self == next {
             if let Self::ManualTransitionJob {
                 progress,
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
         let valid = match (self, next) {
             (
                 Self::TierDeleteJournal {
+                    content_sha256: previous_content,
                     identity_sha256: previous_identity,
                     committed: previous_committed,
+                    dispatch_identity_sha256: previous_dispatch_identity,
+                    state: previous_state,
                     ..
                 },
                 Self::TierDeleteJournal {
+                    content_sha256: next_content,
                     identity_sha256: next_identity,
                     committed: next_committed,
+                    dispatch_identity_sha256: next_dispatch_identity,
+                    state: next_state,
                     ..
                 },
             ) => {
+                use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+                let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
+                    (Some(previous), Some(next)) => previous == next,
+                    (None, None) => true,
+                    // Old receipts did not record the v6 dispatch binding. A
+                    // byte-identical observation may adopt the stronger proof,
+                    // but an in-flight mutation must fail closed instead of
+                    // guessing which operation owned the journal.
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                let state_is_monotonic = match (previous_state, next_state) {
+                    (Some(previous), Some(next)) => {
+                        previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
+                    }
+                    (None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
+            }
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: previous_state,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: next_identity,
+                    state: next_state,
+                    ..
+                },
+            ) => {
+                use tier_delete_journal::TierDeleteDispatchManifestState::{
+                    Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
+                };
                 previous_identity == next_identity
-                    && (previous_committed == next_committed || (!previous_committed && *next_committed))
+                    && matches!(
+                        (previous_state, next_state),
+                        (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
+                    )
             }
             (
                 Self::TransitionTransaction {
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
             Err(Error::other("durable ILM record generation is not a monotonic successor"))
         }
     }
+
+    /// Whether `self` is an older generation of the same immutable record
+    /// that can reach `terminal` through one or more valid state transitions.
+    /// This is deliberately broader than `validate_successor`, which remains
+    /// adjacent-only for receipt advancement. Terminal cleanup uses this only
+    /// after the exact terminal ETag and terminal receipt were committed, to
+    /// purge older object versions exposed by that deletion.
+    pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
+        if self == terminal || self.validate_successor(terminal).is_ok() {
+            return true;
+        }
+        match (self, terminal) {
+            (
+                Self::TierDeleteJournal {
+                    identity_sha256: previous_identity,
+                    dispatch_identity_sha256: previous_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
+                    ..
+                },
+                Self::TierDeleteJournal {
+                    identity_sha256: terminal_identity,
+                    dispatch_identity_sha256: terminal_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
+                    ..
+                },
+            ) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: terminal_identity,
+                    state:
+                        tier_delete_journal::TierDeleteDispatchManifestState::Aborted
+                        | tier_delete_journal::TierDeleteDispatchManifestState::Completed,
+                    ..
+                },
+            ) => previous_identity == terminal_identity,
+            _ => false,
+        }
+    }
 }
 
 fn transition_state_distance(
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
             if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
                 return Err(Error::other("tier delete journal content does not match its path"));
             }
-            let operation_id = path
+            let legacy_operation_id = path
                 .strip_prefix(namespace.prefix)
                 .and_then(|suffix| suffix.strip_suffix(".json"))
                 .ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
+            // Legacy v1-v5 paths already expose a 64-hex operation id and
+            // must remain receipt-compatible. V6 uses an operation-scoped
+            // nested path, so derive a fixed, path-unique receipt id instead
+            // of embedding slashes in the receipt locator.
+            let operation_id = if entry.persisted_version == 6 {
+                hex_sha256(path.as_bytes(), ToOwned::to_owned)
+            } else {
+                legacy_operation_id.to_string()
+            };
             let identity_sha256 = checkpoint_hash(&(
                 &entry.obj_name,
                 &entry.version_id,
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 entry.version_state,
                 &entry.source,
             ))?;
+            let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
             (
                 "operation_id",
-                operation_id.to_string(),
+                operation_id,
                 DurableIlmRecordCheckpoint::TierDeleteJournal {
                     content_sha256,
                     identity_sha256,
                     committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
+                    dispatch_identity_sha256,
+                    state: (entry.persisted_version == 6).then_some(entry.state),
+                },
+            )
+        }
+        DurableIlmRecordKind::TierDeleteDispatchManifest => {
+            let (operation_id, identity_sha256, state) =
+                tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
+            (
+                "operation_id",
+                hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
+                DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
+                    content_sha256,
+                    identity_sha256,
+                    state,
                 },
             )
         }
@@ -956,6 +1108,87 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
+        use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
+
+        let operation_id = Uuid::new_v4();
+        let checkpoint = |state| {
+            let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
+            let namespace = classify_durable_ilm_record(&path)
+                .expect("dispatch manifest namespace should classify")
+                .expect("dispatch manifest should be durable");
+            assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
+            validate_durable_ilm_record(&path, &data)
+                .expect("dispatch manifest should validate")
+                .checkpoint
+        };
+
+        let preparing = checkpoint(Preparing);
+        let authorized = checkpoint(DispatchAuthorized);
+        let completed = checkpoint(Completed);
+        let aborting = checkpoint(Aborting);
+        let aborted = checkpoint(Aborted);
+
+        preparing
+            .validate_successor(&authorized)
+            .expect("Preparing may become DispatchAuthorized");
+        authorized
+            .validate_successor(&completed)
+            .expect("DispatchAuthorized may become Completed");
+        preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
+        aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
+        assert!(authorized.validate_successor(&aborting).is_err());
+        assert!(completed.validate_successor(&authorized).is_err());
+        assert!(aborted.validate_successor(&preparing).is_err());
+    }
+
+    #[test]
+    fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
+        use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+        let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: content.repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: state == Some(Committed),
+            dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
+            state,
+        };
+        let prepared = checkpoint("a", Some("d"), Some(Prepared));
+        let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
+        let committed = checkpoint("c", Some("d"), Some(Committed));
+        prepared
+            .validate_successor(&dispatched)
+            .expect("Prepared may advance to Dispatched");
+        dispatched
+            .validate_successor(&committed)
+            .expect("Dispatched may advance to Committed");
+        assert!(prepared.validate_successor(&committed).is_err());
+        assert!(dispatched.validate_successor(&prepared).is_err());
+
+        let rebound = checkpoint("b", Some("e"), Some(Dispatched));
+        assert!(dispatched.validate_successor(&rebound).is_err());
+
+        let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
+            "kind": "tier_delete_journal",
+            "content_sha256": "a".repeat(64),
+            "identity_sha256": "i".repeat(64),
+            "committed": false
+        }))
+        .expect("legacy tier-delete checkpoint should remain decodable");
+        legacy
+            .validate_successor(&prepared)
+            .expect("byte-identical legacy receipt may adopt the stronger v6 proof");
+        let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: "z".repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: false,
+            dispatch_identity_sha256: None,
+            state: None,
+        };
+        assert!(changed_legacy.validate_successor(&prepared).is_err());
+    }
+
     #[test]
     fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
         let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -34,6 +34,6 @@ pub mod tier_sweeper;
 pub mod transition_transaction;
 
 pub(crate) use durable_namespace::{
-    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
-    validate_durable_ilm_record,
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
+    classify_durable_ilm_record, validate_durable_ilm_record,
 };

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
         return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
     }
 
-    let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
+    let page =
+        list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
     let mut stats = FreeVersionRecoveryStats {
         scanned: 0,
         enqueued: 0,
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
             return Err(tier_free_version_recovery_cancelled());
         }
         retry_cursor.visit(&oi);
-        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
+        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
             let (bucket_marker, object_marker) = retry_cursor.retry_markers();
             stats.truncated = true;
             stats.next_bucket_marker = bucket_marker;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -255,6 +255,7 @@ impl ObjSweeper {
         }
         if del_tier {
             return Some(Jentry {
+                persisted_version: 0,
                 obj_name: self.remote_object.clone(),
                 version_id: self.transition_version_id.clone(),
                 tier_name: self.transition_tier.clone(),
@@ -266,6 +267,7 @@ impl ObjSweeper {
                 version_state: self.transition_version_state,
                 state: TierDeleteJournalState::Committed,
                 source: None,
+                dispatch: None,
             });
         }
         None
@@ -298,9 +300,19 @@ impl ObjSweeper {
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 pub(crate) enum TierDeleteJournalState {
     Prepared,
+    Dispatched,
     Committed,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct TierDeleteDispatchBinding {
+    pub(crate) operation_id: Uuid,
+    pub(crate) manifest_object: String,
+    pub(crate) journal_set_sha256: String,
+    pub(crate) topology_generation: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TierDeleteSourceIdentity {
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
 #[derive(Debug, Clone)]
 #[allow(unused_assignments)]
 pub struct Jentry {
+    /// On-disk format version when decoded. Newly constructed entries use 0;
+    /// the encoder chooses their format from the durable ownership fields.
+    /// Recovery uses this value to quarantine v1-v5 without rewriting them.
+    pub(crate) persisted_version: u8,
     pub(crate) obj_name: String,
     pub(crate) version_id: String,
     pub(crate) tier_name: String,
@@ -350,6 +366,23 @@ pub struct Jentry {
     pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
     pub(crate) state: TierDeleteJournalState,
     pub(crate) source: Option<TierDeleteSourceIdentity>,
+    pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
+}
+
+impl Jentry {
+    /// Whether this prepared transaction is eligible to become the sole
+    /// cleanup owner for its transitioned source. The caller may use this to
+    /// decide whether to persist it, but must not set `skip_free_version`
+    /// until persistence succeeds.
+    pub(crate) fn can_replace_tier_free_version(&self) -> bool {
+        self.state == TierDeleteJournalState::Prepared
+            && self.backend_identity.is_some()
+            && self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
+            && self
+                .source
+                .as_ref()
+                .is_some_and(TierDeleteSourceIdentity::has_stable_identity)
+    }
 }
 
 impl ExpiryOp for Jentry {
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
     }
 
     Some(Jentry {
+        persisted_version: 0,
         obj_name: transitioned.name.clone(),
         version_id: transitioned.version_id.clone(),
         tier_name: transitioned.tier.clone(),
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
         version_state: transition_version_state,
         state: TierDeleteJournalState::Committed,
         source: None,
+        dispatch: None,
     })
 }
 
@@ -673,17 +708,73 @@ mod test {
     use rustfs_s3_client::signer_error::invalid_utf8_header_error;
 
     use super::{
-        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
-        RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
-        delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
-        is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
-        should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
+        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
+        RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
+        delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
+        delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
+        is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
+        transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
     };
     use crate::storage_api_contracts::lifecycle::TransitionedObject;
     use rustfs_filemeta::TransitionVersionState;
     use std::io::{Error, ErrorKind};
     use std::time::{Duration, Instant};
 
+    fn stable_prepared_journal() -> Jentry {
+        Jentry {
+            persisted_version: 0,
+            obj_name: "remote/object".to_string(),
+            version_id: "remote-version".to_string(),
+            tier_name: "WARM".to_string(),
+            backend_identity: Some([7; 32]),
+            version_id_exact: true,
+            version_state: TransitionVersionState::Exact,
+            state: TierDeleteJournalState::Prepared,
+            source: Some(TierDeleteSourceIdentity {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: Some(uuid::Uuid::new_v4().to_string()),
+                versioned: true,
+                version_suspended: false,
+                data_dir: None,
+                etag: None,
+                mod_time: None,
+            }),
+            dispatch: None,
+        }
+    }
+
+    #[test]
+    fn only_stable_prepared_journal_can_replace_tier_free_version() {
+        let stable = stable_prepared_journal();
+        assert!(stable.can_replace_tier_free_version());
+
+        let mut committed = stable.clone();
+        committed.state = TierDeleteJournalState::Committed;
+        assert!(!committed.can_replace_tier_free_version());
+
+        let mut unbound = stable.clone();
+        unbound.backend_identity = None;
+        assert!(!unbound.can_replace_tier_free_version());
+
+        let mut unknown = stable.clone();
+        unknown.version_state = TransitionVersionState::Unknown;
+        assert!(!unknown.can_replace_tier_free_version());
+
+        let mut unstable = stable;
+        unstable.source = Some(TierDeleteSourceIdentity {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+            versioned: false,
+            version_suspended: false,
+            data_dir: None,
+            etag: Some("etag-only".to_string()),
+            mod_time: None,
+        });
+        assert!(!unstable.can_replace_tier_free_version());
+    }
+
     #[test]
     fn signer_header_error_detection_matches_utf8_failures() {
         let err = Error::new(

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1136,6 +1136,16 @@ pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Res
     bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
 }
 
+pub(crate) async fn has_authoritative_never_versioned_state_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<bool> {
+    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
+
+    bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
+}
+
 pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -22,6 +22,7 @@ pub(crate) use rustfs_replication::{
     delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
     delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
     is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
-    resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
+    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
+    single_part_replica_etag_mismatch, target_delete_version_id,
 };

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -32,8 +32,9 @@ use super::replication_object_decision_boundary::{
     MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
     delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
     is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
-    should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
+    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
+    target_delete_version_id,
 };
 use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
 use super::replication_resync_boundary::ResyncStatusType;
@@ -88,6 +89,7 @@ use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, LazyLock, Mutex as StdMutex};
+use std::time::Instant;
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 use tokio::io::AsyncRead;
@@ -118,6 +120,7 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_
 const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
 const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
 const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
+const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
 const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
 
 #[allow(
@@ -190,11 +193,19 @@ fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &Obj
 
 const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
 
-/// Targets that already produced a version-identity-drift warning this
-/// process lifetime, by ARN. Deduping is advisory only (the metric still
-/// counts every drifting PUT), so a reconfigured target re-warning only
-/// after a restart is acceptable.
-static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
+/// How long a target stays quiet after reporting version-identity drift.
+///
+/// This used to be a plain "once per ARN per process": one line ever, which on
+/// a long-lived server meant the single most important diagnostic for a
+/// non-converging generic S3 target scrolled away hours before anyone looked
+/// (rustfs#6822). Re-arming on an interval keeps the log bounded while leaving
+/// the condition discoverable in any recent window.
+const VERSION_IDENTITY_DRIFT_LOG_INTERVAL: TokioDuration = TokioDuration::from_secs(600);
+
+/// When each target last reported version-identity drift, by ARN. Throttling is
+/// advisory only — the metric still counts every drifting PUT.
+static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashMap<String, Instant>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
 
 /// Version purges the peer denied under object lock (#6850). A RustFS peer
 /// with the replicated-purge GOVERNANCE exemption
@@ -322,20 +333,39 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
         return;
     }
     counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
+    if !version_identity_drift_log_due(&tgt_client.arn, Instant::now()) {
+        return;
+    }
+    // `error`, not `warn`: the target silently refuses the addressing scheme
+    // every version-addressed delete and heal on it depends on, so replication
+    // to it can never converge. At `warn` this sat below `DEFAULT_LOG_LEVEL`
+    // and no default deployment ever saw the one line that explains why a
+    // purged version is still on the target (rustfs#6822).
+    error!(
+        event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+        arn = %tgt_client.arn,
+        endpoint = %tgt_client.endpoint,
+        sent_version_id = %source_version_id,
+        assigned_version_id = assigned_version_id.unwrap_or("<none>"),
+        "Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
+    );
+}
+
+/// Whether this ARN's version-identity drift is due to be logged again at
+/// `now`, re-arming the throttle when it is. Split out from the audit so the
+/// interval policy is testable without a target client.
+fn version_identity_drift_log_due(arn: &str, now: Instant) -> bool {
     let mut warned = VERSION_IDENTITY_WARNED_ARNS
         .lock()
         .unwrap_or_else(|poisoned| poisoned.into_inner());
-    if warned.insert(tgt_client.arn.clone()) {
-        warn!(
-            event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-            arn = %tgt_client.arn,
-            endpoint = %tgt_client.endpoint,
-            sent_version_id = %source_version_id,
-            assigned_version_id = assigned_version_id.unwrap_or("<none>"),
-            "Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
-        );
+    match warned.get(arn) {
+        Some(last) if now.duration_since(*last) < VERSION_IDENTITY_DRIFT_LOG_INTERVAL => false,
+        _ => {
+            warned.insert(arn.to_string(), now);
+            true
+        }
     }
 }
 
@@ -2050,10 +2080,13 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
         }
     }
 
+    let delete_version_id = dobj.delete_object.version_id.map(|v| v.to_string());
+    note_replication_terminal_failure(&bucket, &dobj.delete_object.object_name, delete_version_id.as_deref(), &rinfos);
+
     let mut drs = get_replication_state(
         &rinfos,
         &dobj.delete_object.replication_state.clone().unwrap_or_default(),
-        dobj.delete_object.version_id.map(|v| v.to_string()),
+        delete_version_id,
     );
     if replication_status != prev_status {
         drs.replication_timestamp = Some(OffsetDateTime::now_utc());
@@ -3023,8 +3056,11 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
         }
     }
 
+    let version_id = roi.version_id.map(|v| v.to_string());
+    note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos);
+
     let previous_state = roi.replication_state.clone().unwrap_or_default();
-    let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
+    let merged_state = get_replication_state(&rinfos, &previous_state, version_id);
     let replication_status = merged_state.composite_replication_status();
     let new_replication_internal = merged_state.replication_status_internal.clone();
     let mut object_info = roi.to_object_info();
@@ -3101,6 +3137,61 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
     (merged_state, state_persisted)
 }
 
+/// Emit the operator-visible record of a replication attempt that ended FAILED.
+///
+/// Every per-branch failure log in this module is deliberately quieter than
+/// `error`: most of them sit on the replication hot path and fire once per
+/// object *per ARN*, so a target that stays unreachable would flood the log
+/// from inside the transfer loop. That left a hole customers fell into
+/// (rustfs#6825): `DEFAULT_LOG_LEVEL` is `error`, so on a stock deployment a
+/// failed object produced no line at all, and an operator staring at a replica
+/// that never arrived had nothing to correlate — the same trap already
+/// documented for the GET path in
+/// `crates/e2e_test/src/get_stream_failure_observability_test.rs`.
+///
+/// This is the one place that knows an object reached a *terminal* FAILED state
+/// for a target, so this is where the guaranteed-visible line belongs. It is
+/// bounded by the number of objects that actually fail rather than by attempts
+/// inside a transfer, and it carries the target's own error so a remote
+/// rejection is diagnosable without the operator first having to lower the
+/// global log level and reproduce.
+fn note_replication_terminal_failure(bucket: &str, object: &str, version_id: Option<&str>, rinfos: &ReplicatedInfos) {
+    for target in rinfos.targets.iter() {
+        if target.is_empty() {
+            continue;
+        }
+        let replication_failed = target.replication_status == ReplicationStatusType::Failed;
+        let purge_failed = target.version_purge_status == VersionPurgeStatusType::Failed;
+        if !replication_failed && !purge_failed {
+            continue;
+        }
+
+        error!(
+            event = EVENT_REPLICATION_OBJECT_FAILED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            bucket = %bucket,
+            object = %object,
+            version_id = version_id.unwrap_or("-"),
+            arn = %target.arn,
+            endpoint = %target.endpoint,
+            op_type = %target.op_type,
+            size = target.size,
+            replication_status = %target.replication_status.as_str(),
+            version_purge_status = %target.version_purge_status.as_str(),
+            // The target's error can carry a signed URL or an echoed auth
+            // header, so it goes through the same redaction as the persisted
+            // resync detail rather than straight into the log.
+            error = %target
+                .error
+                .as_deref()
+                .and_then(sanitize_resync_error_detail)
+                .unwrap_or_else(|| "<none>".to_string()),
+            "Replication failed for object"
+        );
+    }
+}
+
 fn unavailable_object_target_info(roi: &ReplicateObjectInfo, arn: &str) -> ReplicatedTargetInfo {
     ReplicatedTargetInfo {
         arn: arn.to_string(),
@@ -3400,6 +3491,33 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
+        if let Some(reason) = replication_single_put_size_error(is_multipart, transfer_size) {
+            drop(gr);
+            rinfo.replication_status = ReplicationStatusType::Failed;
+            rinfo.error = Some(reason.clone());
+            warn!(
+                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                target_bucket = %tgt_client.bucket,
+                arn = %tgt_client.arn,
+                object = %object,
+                operation = "put_object",
+                transfer_size = transfer_size,
+                error = %reason,
+                "Replication target operation failed"
+            );
+            send_local_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.to_string(),
+                bucket_name: bucket.clone(),
+                object: object_info,
+                user_agent: "Internal: [Replication]".to_string(),
+                ..Default::default()
+            });
+            return rinfo;
+        }
+
         if let Some(err) = if is_multipart {
             drop(gr);
             let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -4068,6 +4186,14 @@ async fn replicate_all_payload_to_target<S: ReplicationObjectIO>(
     ctx: ReplicateAllPayloadContext<'_, S>,
     mut gr: GetObjectReader,
 ) -> Option<std::io::Error> {
+    // Fail before streaming a body the target is required to reject: an S3
+    // PutObject caps at 5 GiB, and this route is chosen by the source object's
+    // storage shape rather than its size (rustfs#6825).
+    if let Some(reason) = replication_single_put_size_error(ctx.is_multipart, ctx.transfer_size) {
+        drop(gr);
+        return Some(std::io::Error::other(reason));
+    }
+
     if ctx.is_multipart {
         drop(gr);
         let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -5716,4 +5842,207 @@ mod tests {
         assert!(!retry_scheduled.load(Ordering::SeqCst));
         assert_eq!(result.unwrap_err().to_string(), "transfer failed");
     }
+
+    /// A replication target's terminal outcome, as the operator sees it.
+    fn failed_target(arn: &str, error: &str) -> ReplicatedTargetInfo {
+        ReplicatedTargetInfo {
+            arn: arn.to_string(),
+            size: 6 * 1024 * 1024 * 1024,
+            op_type: ReplicationType::Object,
+            replication_status: ReplicationStatusType::Failed,
+            endpoint: "s3.wasabisys.com".to_string(),
+            error: Some(error.to_string()),
+            ..Default::default()
+        }
+    }
+
+    /// Capture the log this module writes, filtered exactly the way a stock
+    /// deployment filters it.
+    fn logs_at_default_level(emit: impl FnOnce()) -> String {
+        use std::sync::{Arc, Mutex};
+        use tracing_subscriber::EnvFilter;
+        use tracing_subscriber::fmt::MakeWriter;
+        use tracing_subscriber::layer::SubscriberExt;
+
+        #[derive(Clone, Default)]
+        struct CapturedLogs {
+            buffer: Arc<Mutex<Vec<u8>>>,
+        }
+        struct CapturedLogWriter {
+            buffer: Arc<Mutex<Vec<u8>>>,
+        }
+        impl std::io::Write for CapturedLogWriter {
+            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+                self.buffer
+                    .lock()
+                    .expect("captured logs mutex should not be poisoned")
+                    .extend_from_slice(buf);
+                Ok(buf.len())
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        impl<'a> MakeWriter<'a> for CapturedLogs {
+            type Writer = CapturedLogWriter;
+            fn make_writer(&'a self) -> Self::Writer {
+                CapturedLogWriter {
+                    buffer: Arc::clone(&self.buffer),
+                }
+            }
+        }
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::registry()
+            // Not a hand-picked level: this is the filter an operator who has
+            // changed nothing is actually running.
+            .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_writer(logs.clone())
+                    .with_ansi(false)
+                    .without_time(),
+            );
+        let _guard = tracing::subscriber::set_default(subscriber);
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
+
+        emit();
+
+        let buffer = logs
+            .buffer
+            .lock()
+            .expect("captured logs mutex should not be poisoned")
+            .clone();
+        String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
+    }
+
+    /// rustfs#6825: a 6 GiB object never reached the target and the server said
+    /// nothing an operator could act on, because every failure line in this
+    /// module sat below `DEFAULT_LOG_LEVEL`. The object key, the target, and
+    /// the target's own error have to survive the default filter.
+    #[test]
+    fn failed_replication_names_the_object_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![failed_target("arn:replication::wasabi", "put_object failed: EntityTooLarge")],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/vm-image.qcow2", Some("v-9"), &rinfos);
+        });
+
+        assert!(logs.contains("backups/vm-image.qcow2"), "the failed object must be named: {logs}");
+        assert!(logs.contains("arn:replication::wasabi"), "the target must be named: {logs}");
+        assert!(logs.contains("EntityTooLarge"), "the target's own error must survive: {logs}");
+        assert!(logs.contains("v-9"), "the version must be named: {logs}");
+        assert!(logs.contains(EVENT_REPLICATION_OBJECT_FAILED), "the event must be structured: {logs}");
+    }
+
+    #[test]
+    fn successful_replication_stays_quiet_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo {
+                arn: "arn:replication::wasabi".to_string(),
+                replication_status: ReplicationStatusType::Completed,
+                ..Default::default()
+            }],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/ok.bin", None, &rinfos);
+        });
+
+        assert!(logs.is_empty(), "a completed replication must not log an error: {logs}");
+    }
+
+    /// A failed version purge is the 6822 symptom (the version stays on the
+    /// target); it must be as visible as a failed transfer even though the
+    /// replication status itself is not FAILED.
+    #[test]
+    fn failed_version_purge_is_reported_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo {
+                arn: "arn:replication::wasabi".to_string(),
+                op_type: ReplicationType::Delete,
+                replication_status: ReplicationStatusType::Empty,
+                version_purge_status: VersionPurgeStatusType::Failed,
+                error: Some("remove_object failed: NoSuchVersion".to_string()),
+                ..Default::default()
+            }],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/purged.bin", Some("v-1"), &rinfos);
+        });
+
+        assert!(logs.contains("backups/purged.bin"), "the purged object must be named: {logs}");
+        assert!(logs.contains("NoSuchVersion"), "the target's own error must survive: {logs}");
+    }
+
+    /// The target's error is echoed remote text and can carry a signed URL or
+    /// an auth header, so it goes through the persisted-detail redaction rather
+    /// than straight into the log.
+    #[test]
+    fn failed_replication_redacts_a_sensitive_target_error() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![failed_target(
+                "arn:replication::wasabi",
+                "put_object failed: rejected Authorization: Bearer super-secret",
+            )],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/vm-image.qcow2", None, &rinfos);
+        });
+
+        assert!(logs.contains("backups/vm-image.qcow2"), "the object must still be named: {logs}");
+        assert!(!logs.contains("super-secret"), "the credential must not reach the log: {logs}");
+    }
+
+    /// An empty target slot carries no outcome; reporting it would invent a
+    /// failure for a target that was never attempted.
+    #[test]
+    fn empty_target_slots_are_not_reported_as_failures() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo::default()],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/unattempted.bin", None, &rinfos);
+        });
+
+        assert!(logs.is_empty(), "an empty target slot must not be reported: {logs}");
+    }
+
+    #[test]
+    fn version_identity_drift_re_arms_after_the_throttle_interval() {
+        let arn = "arn:replication::drift-throttle-test";
+        let start = Instant::now();
+
+        assert!(version_identity_drift_log_due(arn, start), "first drift must be reported");
+        assert!(
+            !version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
+            "a second drift inside the interval must stay throttled"
+        );
+        assert!(
+            version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
+            "drift must become visible again once the interval elapses, instead of \
+             going silent for the rest of the process lifetime"
+        );
+    }
+
+    #[test]
+    fn version_identity_drift_throttles_each_target_independently() {
+        let now = Instant::now();
+
+        assert!(version_identity_drift_log_due("arn:replication::drift-a", now));
+        assert!(
+            version_identity_drift_log_due("arn:replication::drift-b", now),
+            "one target's report must not silence another's"
+        );
+    }
 }

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
     ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
     ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
     StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
-    tier_mutation_control_service_client::TierMutationControlServiceClient,
+    TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
+    node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
     pub applied: bool,
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("{message}")]
+struct TierMutationDefinitelyRejected {
+    message: String,
+}
+
+fn tier_mutation_definitely_rejected_error(message: String) -> Error {
+    Error::other(TierMutationDefinitelyRejected { message })
+}
+
+#[cfg(test)]
+pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
+    tier_mutation_definitely_rejected_error(message.to_string())
+}
+
+pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
+    matches!(
+        error,
+        Error::Io(io_error)
+            if io_error
+                .get_ref()
+                .is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
+    )
+}
+
 fn validate_tier_mutation_response_proof(
     version: u32,
     phase: TierMutationRpcPhase,
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
     canonical_payload: &[u8],
     response: &TierMutationControlResponse,
 ) -> Result<()> {
+    if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
+        return Err(Error::other("peer tier mutation response proof exceeds size limit"));
+    }
+    if response
+        .error_info
+        .as_ref()
+        .is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
+    {
+        return Err(Error::other("peer tier mutation error response exceeds size limit"));
+    }
     let canonical_response =
         rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
             version,
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
             state: response.state,
             applied: response.applied,
             error_info: response.error_info.as_deref(),
+            failure_class: response.failure_class,
         })
         .map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
     verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
         TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
         TierMutationRpcPhase::Abort => {
             if payload_len == 0 {
-                return Ok(());
+                return Err(Error::other("tier mutation abort payload is empty"));
             }
-            return Err(Error::other("tier mutation abort payload must be empty"));
+            rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
         }
         _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
     };
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
     }
 }
 
-fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
-    Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
+fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
+    let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
+    let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
+    // RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
+    if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && status.code() == tonic::Code::FailedPrecondition
+        && status.message().as_bytes() == legacy_rejection.as_bytes()
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    Error::other(message)
+}
+
+fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
+    let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
+    if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    // Missing/zero, unknown, and explicit Ambiguous are deliberately the same
+    // fail-closed result: the coordinator must include this peer in Abort.
+    Error::other(message)
 }
 
 impl PeerRestClient {
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
             .await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
-        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
+    pub async fn abort_tier_mutation(
+        &self,
+        mutation_id: Uuid,
+        canonical_prepare_payload: Bytes,
+    ) -> Result<PeerTierMutationOutcome> {
+        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
             .await
     }
 
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
                         client
                             .prepare_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Commit => {
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
                         client
                             .commit_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Abort => {
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
                         client
                             .abort_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
                 };
                 validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
                 if !response.success {
-                    return Err(Error::other(
-                        response
-                            .error_info
-                            .unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
-                    ));
+                    return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
+                }
+                if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+                    && response.failure_class != TierMutationFailureClass::Unspecified as i32
+                {
+                    return Err(Error::other("successful peer tier mutation response carried a failure class"));
                 }
                 let state = decode_tier_mutation_peer_state(response.state)?;
                 Ok(PeerTierMutationOutcome {
@@ -3332,6 +3394,7 @@ mod tests {
         state: i32,
         applied: bool,
         error_info: Option<&'a str>,
+        failure_class: i32,
     }
 
     fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3345,6 +3408,7 @@ mod tests {
                 state: input.state,
                 applied: input.applied,
                 error_info: input.error_info,
+                failure_class: input.failure_class,
             })
             .expect("small tier mutation response should encode");
         let response_proof =
@@ -3355,6 +3419,7 @@ mod tests {
             applied: input.applied,
             error_info: input.error_info.map(str::to_string),
             response_proof: response_proof.into(),
+            failure_class: input.failure_class,
         }
     }
 
@@ -3372,6 +3437,7 @@ mod tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
+            failure_class: TierMutationFailureClass::Unspecified as i32,
         });
         validate_tier_mutation_response_proof(
             rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3396,6 +3462,10 @@ mod tests {
                 applied: false,
                 ..response.clone()
             },
+            TierMutationControlResponse {
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+                ..response.clone()
+            },
         ] {
             let err = validate_tier_mutation_response_proof(
                 rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3419,6 +3489,44 @@ mod tests {
         assert!(err.to_string().contains("invalid tier mutation response proof"));
     }
 
+    #[test]
+    fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
+        let mutation_id = Uuid::new_v4();
+        let payload = b"tier-mutation-prepare";
+        let oversized_proof = TierMutationControlResponse {
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: None,
+            response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
+            failure_class: TierMutationFailureClass::Ambiguous as i32,
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_proof,
+        )
+        .expect_err("oversized proof must fail before cryptographic verification");
+        assert!(err.to_string().contains("response proof exceeds size limit"));
+
+        let oversized_error = TierMutationControlResponse {
+            response_proof: Bytes::new(),
+            error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
+            ..oversized_proof
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_error,
+        )
+        .expect_err("oversized error detail must fail before proof construction");
+        assert!(err.to_string().contains("error response exceeds size limit"));
+    }
+
     #[test]
     fn tier_mutation_peer_state_decode_fails_closed() {
         assert_eq!(
@@ -3463,8 +3571,17 @@ mod tests {
             )
             .is_err()
         );
-        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
-        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
+        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
+            .expect("max abort payload should fit");
+        assert!(
+            validate_tier_mutation_payload_len(
+                TierMutationRpcPhase::Abort,
+                rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -3479,7 +3596,7 @@ mod tests {
                 tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
                 tonic::Status::unavailable("peer tier mutation control unavailable"),
             ] {
-                let err = tier_mutation_control_status_error(phase, status);
+                let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
                 let rendered = err.to_string();
                 assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
                 assert!(
@@ -3493,6 +3610,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let exact = format!("unsupported tier mutation peer protocol version: {version}");
+        let rejected = tier_mutation_control_status_error(
+            TierMutationRpcPhase::Prepare,
+            version,
+            tonic::Status::failed_precondition(exact.clone()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for status in [
+            tonic::Status::failed_precondition(format!("{exact}.")),
+            tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
+            tonic::Status::invalid_argument(exact.clone()),
+            tonic::Status::unimplemented(exact),
+        ] {
+            let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "near-text, wrong-code, and Unimplemented failures must remain ambiguous"
+            );
+        }
+    }
+
+    #[test]
+    fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let rejected = tier_mutation_failed_response_error(
+            version,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("rejected".to_string()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for failure_class in [
+            TierMutationFailureClass::Unspecified as i32,
+            TierMutationFailureClass::Ambiguous as i32,
+            99,
+        ] {
+            let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
+            );
+        }
+
+        let v3_ignores_v4_class = tier_mutation_failed_response_error(
+            rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("legacy failure".to_string()),
+        );
+        assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
+    }
+
     #[tokio::test]
     async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
         let client = test_peer_client();

crates/ecstore/src/core/pools_test.rs

@@ -196,7 +196,7 @@ mod decommission_lock_order_tests {
     use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING;
     use crate::core::pools::{
         DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
-        POOL_META_NAME, set_decommission_capacity_info_overrides_for_test,
+        POOL_META_NAME, decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test,
     };
     use crate::data_movement;
     use crate::disk::RUSTFS_META_BUCKET;
@@ -3047,6 +3047,198 @@ mod decommission_lock_order_tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn exact_delete_reconciles_pending_capacity_before_removing_replicas() {
+        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+        let bucket = test_bucket("exact-delete-capacity");
+        let object = "published-before-exact-delete.bin";
+        let body = vec![0x55; 64 * 1024];
+        let version_id = uuid::Uuid::new_v4().to_string();
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create the exact-delete reconciliation bucket");
+        let incarnation = store
+            .bucket_incarnation_id(&bucket)
+            .await
+            .expect("load the exact-delete bucket incarnation");
+
+        let mut source_data = PutObjReader::from_vec(body.clone());
+        let source = store.pools[0]
+            .put_object(
+                &bucket,
+                object,
+                &mut source_data,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.clone()),
+                    expected_bucket_incarnation_id: Some(incarnation),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed the exact source version");
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        let target_total = body.len().saturating_mul(4);
+        set_decommission_capacity_info_overrides_for_test(
+            store.id,
+            vec![vec![
+                DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
+                DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
+            ]],
+        );
+        store
+            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
+            .await
+            .expect("activate the exact-delete capacity reservation");
+        let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
+        let target_pool_index = store.pool_meta.read().await.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the exact-delete capacity reservation should exist")
+            .targets[0]
+            .pool_index;
+        assert_eq!(target_pool_index, 2);
+
+        let target_options = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.clone()),
+            mod_time: source.mod_time,
+            preserve_etag: source.etag.clone(),
+            user_defined: (*source.user_defined).clone(),
+            data_movement: true,
+            src_pool_idx: 0,
+            expected_bucket_incarnation_id: Some(incarnation),
+            ..Default::default()
+        };
+        let mut target_data = PutObjReader::from_vec(body.clone());
+        let target = store.pools[target_pool_index]
+            .put_object(&bucket, object, &mut target_data, &target_options)
+            .await
+            .expect("publish the target version before capacity progress");
+        assert_eq!(target.version_id, source.version_id);
+        assert_eq!(target.mod_time, source.mod_time);
+        assert_eq!(target.size, source.size);
+
+        let source_version_id = source.version_id.map(|version_id| version_id.to_string());
+        let mutation_id = decommission_capacity_mutation_id(
+            owner,
+            &source.bucket,
+            &source.name,
+            source_version_id.as_deref(),
+            source.delete_marker,
+            source.mod_time,
+        );
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            let source_pool = &mut pool_meta.pools[0];
+            let reservation = source_pool
+                .decommission
+                .as_mut()
+                .and_then(|info| info.capacity_reservation.as_mut())
+                .expect("the exact-delete capacity reservation should remain active");
+            let target = reservation
+                .targets
+                .iter_mut()
+                .find(|target| target.pool_index == target_pool_index)
+                .expect("the exact-delete target allocation should exist");
+            target.pending_physical_bytes = body.len();
+            target.pending_mutation_id = Some(mutation_id);
+            reservation.pending_target_physical_bytes = body.len();
+            source_pool.last_update = time::OffsetDateTime::now_utc();
+        }
+        store
+            .save_current_pool_meta_for_test(&[0])
+            .await
+            .expect("persist the simulated post-commit capacity intent");
+
+        let exact_delete_options = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.clone()),
+            expected_bucket_incarnation_id: Some(incarnation),
+            ..Default::default()
+        };
+        store.pools[target_pool_index]
+            .delete_object(&bucket, object, exact_delete_options.clone())
+            .await
+            .expect("remove the target evidence before the fail-closed exact delete");
+        let delete_err = store
+            .delete_object(&bucket, object, exact_delete_options.clone())
+            .await
+            .expect_err("exact delete must fail while its pending target evidence is absent");
+        assert!(matches!(delete_err, crate::error::Error::DecommissionCapacityBlocked { .. }));
+        store.pools[0]
+            .get_object_info(
+                &bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.clone()),
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("a failed reconciliation must preserve the source evidence");
+        let mut failed = crate::core::pools::PoolMeta::default();
+        failed
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("the failed exact delete must preserve readable capacity metadata");
+        let failed_reservation = failed.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the failed exact delete reservation should remain present");
+        assert_eq!(failed_reservation.pending_target_physical_bytes, body.len());
+        assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
+
+        let mut replacement_target_data = PutObjReader::from_vec(body.clone());
+        store.pools[target_pool_index]
+            .put_object(&bucket, object, &mut replacement_target_data, &target_options)
+            .await
+            .expect("restore the equivalent target evidence for the exact-delete retry");
+
+        store
+            .delete_object(&bucket, object, exact_delete_options)
+            .await
+            .expect("the exact delete should reconcile capacity before removing replicas");
+
+        let mut persisted = crate::core::pools::PoolMeta::default();
+        persisted
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("the exact-delete reconciliation should remain durable");
+        let reservation = persisted.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the reconciled reservation should remain present");
+        assert_eq!(reservation.pending_target_physical_bytes, 0);
+        assert_eq!(reservation.consumed_target_physical_bytes, body.len());
+        assert_eq!(reservation.committed_data_bytes, body.len());
+
+        for pool_index in [0, target_pool_index] {
+            let err = store.pools[pool_index]
+                .get_object_info(
+                    &bucket,
+                    object,
+                    &ObjectOptions {
+                        versioned: true,
+                        version_id: Some(version_id.clone()),
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect_err("the exact version should be absent after reconciliation and delete");
+            assert!(crate::error::is_err_object_not_found(&err) || crate::error::is_err_version_not_found(&err));
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() {

crates/ecstore/src/core/sets.rs

@@ -37,8 +37,9 @@ use crate::{
     runtime::instance::{InstanceContext, bootstrap_ctx},
     runtime::sources as runtime_sources,
     set_disk::{PreparedGetObjectMetadata, SetDisks},
-    store::init_format::{
-        check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
+    store::{
+        RemoteTuplePublicationFence,
+        init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
     },
 };
 use futures::{
@@ -625,6 +626,19 @@ impl Sets {
             .put_object_with_old_current_size(bucket, object, data, opts)
             .await
     }
+
+    pub(crate) async fn put_object_with_old_current_size_for_data_movement(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut PutObjReader,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
+        self.get_disks_by_key(object)
+            .put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
+            .await
+    }
 }
 
 #[async_trait::async_trait]

crates/ecstore/src/data_movement/mod.rs

@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
+use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
     flag.store(false, Ordering::Relaxed);
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 struct DataMovementMultipartAbortBarrierState {
     bucket: String,
     object: String,
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
     release: tokio::sync::Notify,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) struct DataMovementMultipartAbortBarrier {
     state: Arc<DataMovementMultipartAbortBarrierState>,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
     std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
 > = std::sync::OnceLock::new();
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl DataMovementMultipartAbortBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(DataMovementMultipartAbortBarrierState {
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl Drop for DataMovementMultipartAbortBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
     let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
     capacity_owner: Option<DecommissionCapacityOwner>,
 ) -> Result<()> {
     let source = rd.object_info.clone();
-    let _mutation_fence = store
+    let mutation_fence = store
         .acquire_decommission_object_mutation_fence(&bucket, &source.name)
         .await?;
     let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
         op_label,
         None,
         capacity_owner,
-        Some(&_mutation_fence),
+        Some(mutation_fence),
     )
     .await
 }
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
     op_label: &str,
     lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     capacity_owner: Option<DecommissionCapacityOwner>,
-    mutation_fence: Option<&ObjectLockDiagGuard>,
+    mutation_fence: Option<DecommissionFixedReadAnchor>,
 ) -> Result<()> {
+    let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
     let capacity_owner = capacity_owner.map(|owner| {
         let version_id = object_info.version_id.map(|version_id| version_id.to_string());
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
         });
         owner.with_mutation_id(mutation_id)
     });
+    // Capture the exact source/tier identity before any client-paced read, but
+    // defer both the tier lease and source/target write locks to the final
+    // publication. Decommission already owns main's fixed-domain mutation
+    // fence, so reacquiring that domain as a write lock would self-deadlock.
+    let remote_tuple_publication_fence = store
+        .acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
+        .await?;
     let has_part_checksums = object_info
         .parts
         .iter()
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
             }
             let mut cleanup_opts =
                 data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut cleanup_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut cleanup_opts);
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
             }
         }
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
+            .handle_new_multipart_upload_with_pool_idx(
+                &bucket,
+                &object_info.name,
+                &new_multipart_opts,
+                mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
+            )
             .await
         {
             Ok(res) => res,
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut complete_multipart_opts);
             }
+            let remote_tuple_publication_fence = match mutation_fence.take() {
+                Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+                None => remote_tuple_publication_fence,
+            };
             if let Err(err) = store
                 .clone()
-                .complete_multipart_upload_for_data_movement(
-                    (target_pool_idx, mutation_fence),
+                .complete_multipart_upload_for_data_movement_with_publication_fence(
+                    target_pool_idx,
                     &bucket,
                     &object_info.name,
                     &res.upload_id,
                     parts,
                     &complete_multipart_opts,
+                    remote_tuple_publication_fence,
                 )
                 .await
             {
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
             let mut abort_opts =
                 data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut abort_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut abort_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
 
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
-                #[cfg(test)]
+                #[cfg(all(test, feature = "test-util"))]
                 pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
                 let mut abort_opts =
                     data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-                if let Some(fence) = mutation_fence {
-                    fence.add_namespace_lock_fence(&mut abort_opts);
+                if let Some(anchor) = mutation_fence.as_ref() {
+                    anchor.guard().add_namespace_lock_fence(&mut abort_opts);
                 }
                 if let Some(fence) = multipart_mutation_fence.as_ref() {
                     fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
     if let Some(signal) = lock_lost_signal {
         put_opts.add_namespace_lock_lost_signal(signal);
     }
+    let remote_tuple_publication_fence = match mutation_fence.take() {
+        Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+        None => remote_tuple_publication_fence,
+    };
     let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
+        .put_object_for_data_movement_with_publication_fence(
+            &bucket,
+            &object_info.name,
+            &mut data,
+            &put_opts,
+            remote_tuple_publication_fence,
+        )
         .await
         .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
     if let Err(err) = put_result {

crates/ecstore/src/disk/disk_store.rs

@@ -2435,6 +2435,25 @@ mod tests {
         assert_eq!(window.acc_time, 18_000);
     }
 
+    #[test]
+    fn timed_action_slot_snapshot_skips_writer_owned_slot() {
+        let slot = TimedActionSlot::default();
+        slot.unix_sec.store(70, Ordering::Relaxed);
+        slot.count.store(2, Ordering::Relaxed);
+        slot.acc_time.store(18_000, Ordering::Relaxed);
+        slot.version.store(2, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
+
+        assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
+        slot.unix_sec.store(71, Ordering::Relaxed);
+        slot.count.store(1, Ordering::Relaxed);
+        slot.acc_time.store(11_000, Ordering::Relaxed);
+        assert_eq!(slot.snapshot(), None);
+
+        slot.version.store(4, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
+    }
+
     #[test]
     fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
         let metrics = DiskHealthMetricEpoch::default();

crates/ecstore/src/disk/local.rs

@@ -7092,6 +7092,9 @@ impl LocalDisk {
                                 .await?
                         {
                             meta.name.push_str(SLASH_SEPARATOR);
+                            // Conservative listings verify physical prefixes. Never-versioned
+                            // buckets use the bounded fast path and reclaim residue after an
+                            // exact recursive listing proves that prefix empty.
                             if opts.recursive
                                 || opts.incl_deleted
                                 || opts.skip_hidden_prefix_check
@@ -17619,7 +17622,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_scan_dir_nonrecursive_visible_prefix_probe_cost() {
+    async fn test_scan_dir_nonrecursive_fast_path_preserves_probe_bound() {
         use rustfs_filemeta::MetacacheReader;
         use tempfile::tempdir;
 
@@ -17651,6 +17654,10 @@ mod test {
             expected_names.push(format!("{prefix}/"));
         }
 
+        fs::create_dir_all(bucket_dir.join("stale/nested/residue"))
+            .await
+            .expect("stale backing directory should be created");
+
         async fn scan_prefixes(disk: &LocalDisk, bucket: &str, skip_hidden_prefix_check: bool) -> (Vec<String>, usize) {
             let probe_count = Arc::new(AtomicUsize::new(0));
             let (reader, mut writer) = tokio::io::duplex(64 * 1024);
@@ -17693,8 +17700,11 @@ mod test {
         let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
 
         assert_eq!(conservative_names, expected_names);
-        assert_eq!(fast_path_names, expected_names);
-        assert_eq!(conservative_probes, PREFIX_COUNT * 3);
+        let mut expected_fast_path_names = expected_names.clone();
+        expected_fast_path_names.push("stale/".to_owned());
+        assert_eq!(fast_path_names, expected_fast_path_names);
+        let expected_probes = PREFIX_COUNT * 3 + 3;
+        assert_eq!(conservative_probes, expected_probes);
         assert_eq!(fast_path_probes, 0);
     }
 

crates/ecstore/src/disk/os.rs

@@ -298,6 +298,39 @@ pub(crate) mod windows_rename_test_hooks {
     }
 }
 
+/// Test-only hooks into the destination-parent walk of rename preparation.
+///
+/// The prune race lives between two syscalls inside
+/// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
+/// point reproduces it deterministically. Hooks are keyed by the absolute path
+/// of the component just opened and queued per path: a retrying preparation
+/// visits the same component again, so a test models a pruner that keeps
+/// walking upward by queueing one hook per visit.
+#[cfg(all(test, unix))]
+pub(crate) mod prepare_rename_test_hooks {
+    use super::*;
+
+    type Hook = Box<dyn FnOnce() + Send>;
+
+    static AFTER_COMPONENT_OPENED: LazyLock<Mutex<HashMap<PathBuf, VecDeque<Hook>>>> =
+        LazyLock::new(|| Mutex::new(HashMap::new()));
+
+    pub(crate) fn queue_after_component_opened(path: &Path, hook: impl FnOnce() + Send + 'static) {
+        AFTER_COMPONENT_OPENED
+            .lock()
+            .entry(path.to_path_buf())
+            .or_default()
+            .push_back(Box::new(hook));
+    }
+
+    pub(crate) fn run_after_component_opened(path: &Path) {
+        let hook = AFTER_COMPONENT_OPENED.lock().get_mut(path).and_then(VecDeque::pop_front);
+        if let Some(hook) = hook {
+            hook();
+        }
+    }
+}
+
 /// Fsync a directory so recently created or renamed entries survive power loss.
 /// No-op on non-Unix platforms where directories cannot be opened for syncing.
 pub fn fsync_dir_std(dir: impl AsRef<Path>) -> io::Result<()> {
@@ -1905,8 +1938,8 @@ pub(crate) async fn rename_all_with_prepared_source(
         let base_dir = base_dir.clone();
         move || {
             validate_prepared_rename_source(&prepared_source, &src_file_path)?;
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2008,8 +2041,8 @@ async fn reliable_rename_inner_with_lease(
         let dst_file_path = dst_file_path.clone();
         let base_dir = base_dir.clone();
         move || {
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2233,12 +2266,13 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
     let mut attempt = 0;
     loop {
         match prepare_rename(src_file_path, dst_file_path, base_dir, publication_root) {
-            Ok(preparation) => return Ok((preparation, attempt)),
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Ok(preparation) => return Ok(preparation),
+            Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                 attempt += 1;
             }
             Err(err) => return Err(err),
@@ -2252,23 +2286,26 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
     let source_parent = src_file_path
         .parent()
         .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
     let destination_parent = dst_file_path.parent();
-    let mut attempt = 0;
-    let prepare_destination_parent = |attempt: &mut usize| -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
+    // The destination walk and the source open below keep separate counters:
+    // exhausting one must not deny the other its own retry.
+    let prepare_destination_parent = || -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+        let mut attempt = 0;
         loop {
             let result = destination_parent
                 .map(|parent| mkdir_all_below_existing_base_std(parent, base_dir, publication_root))
                 .transpose();
             match result {
                 Ok(parent_guard) => break Ok(parent_guard),
-                Err(err) if should_retry_rename(&err, *attempt) => {
+                Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                     #[cfg(test)]
                     windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                    *attempt += 1;
+                    attempt += 1;
                 }
                 Err(err) => break Err(err),
             }
@@ -2281,7 +2318,7 @@ fn prepare_rename_with_retry(
         None => false,
     };
     let (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity) = if same_parent {
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         let source_parent_guard = parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?
@@ -2293,16 +2330,17 @@ fn prepare_rename_with_retry(
         let source_parent_guard = lock_windows_directory_tree(source_parent, destination_parent, publication_root)?;
         let (source_identity_anchor, expected_source_identity) =
             open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity)
     };
+    let mut source_attempt = 0;
     let source = loop {
         match open_windows_rename_source(src_file_path, &source_parent_guard) {
             Ok(source) => break source,
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Err(err) if should_retry_rename(&err, source_attempt) => {
                 #[cfg(test)]
                 windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                attempt += 1;
+                source_attempt += 1;
             }
             Err(err) => return Err(err),
         }
@@ -2315,14 +2353,11 @@ fn prepare_rename_with_retry(
     }
     drop(source_identity_anchor);
 
-    Ok((
-        RenamePreparation {
-            parent_guard,
-            _source_parent_guard: source_parent_guard,
-            source,
-        },
-        attempt,
-    ))
+    Ok(RenamePreparation {
+        parent_guard,
+        _source_parent_guard: source_parent_guard,
+        source,
+    })
 }
 
 #[cfg(not(windows))]
@@ -2339,24 +2374,22 @@ fn prepare_rename(
     Ok(RenamePreparation { parent_guard })
 }
 
-fn rename_prepared(
-    _src_file_path: &Path,
-    dst_file_path: &Path,
-    preparation: &RenamePreparation,
-    attempt: usize,
-) -> io::Result<()> {
+/// Publish a prepared rename. The retry budget starts fresh here: preparation
+/// keeps its own counter, so a chain rebuilt after a concurrent prune must not
+/// cost the rename its one retry.
+fn rename_prepared(_src_file_path: &Path, dst_file_path: &Path, preparation: &RenamePreparation) -> io::Result<()> {
     #[cfg(windows)]
     {
         let parent_guard = preparation
             .parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?;
-        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, attempt)
+        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, 0)
     }
 
     #[cfg(not(windows))]
     {
-        let mut attempt = attempt;
+        let mut attempt = 0;
         loop {
             let rename_result = rename_into_existing_parent(_src_file_path, dst_file_path, preparation.parent_guard.as_ref());
             match rename_result {
@@ -3756,6 +3789,8 @@ pub(crate) fn mkdir_all_below_existing_base_std(
         let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
         let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
 
+        #[cfg(test)]
+        let mut walked_path = base_dir.to_path_buf();
         for component in relative.components() {
             let Component::Normal(component) = component else {
                 continue;
@@ -3769,6 +3804,11 @@ pub(crate) fn mkdir_all_below_existing_base_std(
                 Err(err) => return Err(err.into()),
             }
             parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
+            #[cfg(test)]
+            {
+                walked_path.push(component);
+                prepare_rename_test_hooks::run_after_component_opened(&walked_path);
+            }
         }
 
         Ok(parents)
@@ -3866,11 +3906,51 @@ fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base
 /// cleanup renames (e.g. `move_to_trash` on an already-removed tmp path) a
 /// pointless second syscall. This predicate is shared by the `rename_data`
 /// commit path via `rename_all`, so any relaxation here must keep genuine
-/// transient errors retryable.
+/// transient errors retryable. The *preparation* phase deliberately uses
+/// [`should_retry_prepare`] instead — see there for why `NotFound` is
+/// recoverable while the destination parent chain is still being built.
 fn should_retry_rename(err: &io::Error, attempt: usize) -> bool {
     attempt == 0 && err.kind() != io::ErrorKind::NotFound
 }
 
+/// How many times rename preparation may retry a `NotFound`.
+///
+/// A pruning walk (`LocalDisk::delete_file`) removes empty ancestors
+/// monotonically upward and stops at the volume root, so it can invalidate
+/// each component *below* the base at most once. One attempt per such
+/// component therefore outlasts a pruning walk, and concurrent walks only
+/// steal an attempt by making that same upward progress. A destination whose
+/// parent *is* the base gets a budget of zero, keeping `NotFound` immediately
+/// terminal for speculative cleanup renames.
+fn prepare_prune_budget(dst_file_path: &Path, base_dir: &Path) -> usize {
+    dst_file_path
+        .parent()
+        .and_then(|parent| parent.strip_prefix(base_dir).ok())
+        .map(|relative| relative.components().count())
+        .unwrap_or(0)
+}
+
+/// Whether a failed rename *preparation* attempt (building the destination
+/// parent chain) should be retried.
+///
+/// Unlike [`should_retry_rename`], `NotFound` is recoverable here: a concurrent
+/// delete prunes now-empty parent directories, so it can unlink an intermediate
+/// destination component between this walk opening a directory and creating the
+/// next child inside it, which a handle-relative `mkdirat`/`openat` reports as
+/// `NotFound`. Each retry rebuilds the whole chain from the base directory,
+/// which no walk below it can remove; `prune_budget` bounds how far a pruner
+/// can push the walk back. A genuinely missing base directory fails identically
+/// on every attempt — the base is only ever opened, never created — so the
+/// missing-base contract holds at the cost of a few extra syscalls on an
+/// already-failing path.
+fn should_retry_prepare(err: &io::Error, attempt: usize, prune_budget: usize) -> bool {
+    if err.kind() == io::ErrorKind::NotFound {
+        attempt < prune_budget
+    } else {
+        attempt == 0
+    }
+}
+
 pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
     let mut i = 0;
 
@@ -4401,6 +4481,141 @@ mod tests {
         assert!(!should_retry_rename(&denied, 1));
     }
 
+    #[test]
+    fn prepare_retry_budget_covers_every_prunable_component() {
+        // A pruner can invalidate each component below the base once, so the
+        // budget must match the chain depth, not a fixed count.
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "pruned");
+        assert!(should_retry_prepare(&not_found, 0, 2));
+        assert!(should_retry_prepare(&not_found, 1, 2));
+        assert!(!should_retry_prepare(&not_found, 2, 2));
+    }
+
+    #[test]
+    fn prepare_retry_keeps_other_errors_at_a_single_retry() {
+        // Only a prune produces a recoverable NotFound; everything else keeps
+        // the historical single retry so persistent failures stay cheap.
+        let denied = io::Error::new(io::ErrorKind::PermissionDenied, "denied");
+        assert!(should_retry_prepare(&denied, 0, 3));
+        assert!(!should_retry_prepare(&denied, 1, 3));
+    }
+
+    #[test]
+    fn prepare_retry_budget_is_zero_when_the_parent_is_the_base() {
+        // Speculative cleanup renames (move_to_trash) land directly in their
+        // base, so a NotFound there is a missing base: terminal, not a prune.
+        let base = Path::new("/vol");
+        assert_eq!(prepare_prune_budget(Path::new("/vol/entry"), base), 0);
+        assert_eq!(prepare_prune_budget(Path::new("/vol/data-movement/sha/id/xl.meta"), base), 3);
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "missing base");
+        assert!(!should_retry_prepare(&not_found, 0, 0));
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_concurrent_empty_parent_prune() {
+        // A multipart staging cleanup prunes the momentarily empty shared
+        // `data-movement/` prefix while a concurrent upload publishes its
+        // xl.meta below that same prefix. The writer's walk holds an fd to the
+        // pruned component, so its next handle-relative mkdirat fails
+        // NotFound; preparation must rebuild the chain and still publish.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let pruned = shared.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            // The cleanup chain's empty-parent prune lands after the writer
+            // opened the shared component but before it creates its child.
+            std::fs::remove_dir(&pruned).expect("prune the empty shared prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a concurrently pruned intermediate directory must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_a_prune_walking_up_every_shared_component() {
+        // The multipart data-movement chain has TWO shared components below the
+        // volume (`data-movement/` and the per-object `<sha>/`), so one cleanup
+        // walk pruning upward can invalidate the writer twice: once at <sha>,
+        // then again at data-movement while the writer rebuilds. A budget that
+        // covers only a single component would still break write quorum here.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let movement = base.join("data-movement");
+        let sha = movement.join("sha");
+        std::fs::create_dir_all(&sha).expect("create shared chain");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = sha.join("upload-id").join("xl.meta");
+
+        // First visit of `data-movement` is the writer's initial walk, which the
+        // pruner has not reached yet; it prunes on the writer's rebuild.
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, || {});
+        let pruned_sha = sha.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&sha, move || {
+            std::fs::remove_dir(&pruned_sha).expect("prune the empty per-object prefix");
+        });
+        let pruned_movement = movement.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, move || {
+            std::fs::remove_dir(&pruned_movement).expect("prune the empty data-movement prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a prune walking up the whole shared chain must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_rejects_a_symlink_swapped_in_between_prepare_attempts() {
+        // The retry must not become a traversal window: replacing the pruned
+        // component with a symlink out of the volume before the rebuilt walk
+        // reopens it must fail closed, exactly as a symlink staged before the
+        // first attempt does.
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        std::fs::create_dir_all(&outside).expect("create outside target");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let swapped = shared.clone();
+        let outside_target = outside.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            std::fs::remove_dir(&swapped).expect("prune the shared prefix");
+            symlink(&outside_target, &swapped).expect("replace the pruned component with a symlink");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect_err("a symlink swapped in between attempts must not be followed");
+
+        assert!(src.exists(), "rejected publish must preserve the staged source");
+        assert!(
+            !outside.join("sha").exists(),
+            "the rebuilt walk must not create or publish through the replacement symlink"
+        );
+    }
+
     #[test]
     fn is_dir_not_empty_error_recognizes_directory_not_empty_kind() {
         let err = io::Error::from(io::ErrorKind::DirectoryNotEmpty);

crates/ecstore/src/error/conversion_roundtrip_tests.rs

@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
     assert_eq!(count, 3);
     assert_eq!(err, Some(DiskError::other("can not get client")));
 }
+
+#[test]
+fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
+    let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
+    let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
+
+    assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
+    let StorageError::Io(io_error) = first else {
+        panic!("stable context must remain an io error");
+    };
+    assert_eq!(io_error.to_string(), "tier mutation intent changed");
+    let context = io_error.get_ref().expect("stable context must remain downcastable");
+    assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
+}

crates/ecstore/src/error/mod.rs

@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;
 
+/// Keeps high-cardinality diagnostic detail in the error source while making
+/// the rendered `io::Error` stable for quorum aggregation.
+#[derive(Debug)]
+struct StableIoContextError {
+    message: &'static str,
+    source: Box<dyn std::error::Error + Send + Sync>,
+}
+
+impl std::fmt::Display for StableIoContextError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(self.message)
+    }
+}
+
+impl std::error::Error for StableIoContextError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(self.source.as_ref())
+    }
+}
+
+pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
+where
+    E: Into<Box<dyn std::error::Error + Send + Sync>>,
+{
+    std::io::Error::other(StableIoContextError {
+        message,
+        source: source.into(),
+    })
+}
+
 /// Storage layer error type covering disk, volume, bucket, object, multipart,
 /// erasure-coding, and operational error conditions.
 ///
@@ -264,6 +294,13 @@ impl StorageError {
         StorageError::Io(std::io::Error::other(error))
     }
 
+    pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        StorageError::Io(stable_io_error(message, source))
+    }
+
     pub fn is_not_found(&self) -> bool {
         matches!(
             self,

crates/ecstore/src/object_api/mod.rs

@@ -37,7 +37,9 @@ use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, RestoreS
 use rustfs_rio::Checksum;
 use rustfs_utils::CompressionAlgorithm;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
-use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
+use rustfs_utils::http::{
+    AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS, SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str,
+};
 use rustfs_utils::path::decode_dir_object;
 use std::collections::HashMap;
 use std::fmt::Debug;

crates/ecstore/src/object_api/types.rs

@@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase {
 #[doc(hidden)]
 #[derive(Default)]
 pub struct LifecycleDeleteAllJournalState {
-    prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
     mutation_started: bool,
 }
 
 impl Debug for LifecycleDeleteAllJournalState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LifecycleDeleteAllJournalState")
-            .field("prepared_count", &self.prepared.len())
             .field("mutation_started", &self.mutation_started)
             .finish()
     }
 }
 
 impl LifecycleDeleteAllJournalState {
-    pub(crate) fn contains(&self, name: &str) -> bool {
-        self.prepared.contains_key(name)
-    }
-
-    pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
-        self.prepared.insert(name, entry);
-    }
-
-    pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
-        self.prepared.values().cloned().collect()
-    }
-
     pub(crate) fn mark_mutation_started(&mut self) {
         self.mutation_started = true;
     }
 
+    #[cfg(test)]
     pub(crate) fn mutation_started(&self) -> bool {
         self.mutation_started
     }
@@ -706,6 +693,12 @@ pub struct ObjectOptions {
     pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
     #[doc(hidden)]
     pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
+    /// Whole-operation authorization created only by consuming a validated
+    /// v6 dispatch-manifest permit. Clones share the authorization, not the
+    /// one-shot permit itself.
+    #[doc(hidden)]
+    pub tier_delete_dispatch_authorization:
+        Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
     /// RustFS-only compare-and-set condition checked under the object write lock.
     pub expected_current_version_id: Option<String>,
     /// Persisted bucket incarnation observed before authorization.
@@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions {
             .field("version_id", &self.version_id.is_some())
             .field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
             .field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
+            .field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
             .field("expected_current_version_id", &self.expected_current_version_id.is_some())
             .field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
             .field("no_lock", &self.no_lock)
@@ -972,7 +966,7 @@ impl ObjectOptions {
     }
 
     #[cfg(test)]
-    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+    pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
         self.namespace_lock_fence
             .get_or_insert_with(NamespaceLockFence::new)
             .extend(fence);
@@ -1777,9 +1771,10 @@ impl ObjectInfo {
         }
 
         if let Some(data) = &self.checksum {
-            if self.is_encrypted() {
+            if self.is_encrypted() && get_consistent_str(&self.user_defined, SUFFIX_PLAINTEXT_CHECKSUM) != Some("true") {
                 // Object-level encrypted checksum bytes require SSE decrypt material,
-                // so do not expose them as plaintext checksum headers here. The
+                // unless RustFS marked the stored bytes as plaintext. Do not expose
+                // unmarked bytes as checksum headers here. The
                 // `false` multipart flag feeds the response-path COMPOSITE
                 // fallback; callers that need accurate multipart routing must
                 // consult `is_multipart()` instead of this value.
@@ -2485,6 +2480,31 @@ mod tests {
         assert!(checksums.is_empty());
     }
 
+    #[test]
+    fn decrypt_checksums_reads_marked_rustfs_encrypted_object_checksum() {
+        let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")
+            .expect("test checksum should be valid");
+        let checksum_key = checksum.checksum_type.to_string();
+        let expected_checksum = checksum.encoded.clone();
+        let mut user_defined =
+            HashMap::from([(rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())]);
+        rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
+        assert_eq!(user_defined.get("x-rustfs-internal-plaintext-checksum").map(String::as_str), Some("true"));
+        assert_eq!(user_defined.get("x-minio-internal-plaintext-checksum").map(String::as_str), Some("true"));
+        let info = ObjectInfo {
+            checksum: Some(checksum.to_bytes(&[])),
+            user_defined: Arc::new(user_defined),
+            ..Default::default()
+        };
+
+        let (checksums, is_multipart) = info
+            .decrypt_checksums(0, &HeaderMap::new())
+            .expect("marked RustFS checksum should decode");
+
+        assert!(!is_multipart);
+        assert_eq!(checksums.get(&checksum_key), Some(&expected_checksum));
+    }
+
     #[test]
     fn decrypt_checksums_keeps_encrypted_multipart_flag_false_for_response_paths() {
         let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")

crates/ecstore/src/services/notification_sys.rs

@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
+use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
-use std::sync::{Arc, Mutex, OnceLock};
+use std::sync::{
+    Arc, Mutex, OnceLock,
+    atomic::{AtomicBool, AtomicUsize, Ordering},
+};
 use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
 const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
+const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
+type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
+
+fn cross_pool_fence_policy_results(
+    peer_epochs: BTreeMap<String, Uuid>,
+    minimum_version: u32,
+) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
+    let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
+        Ok(peer_epochs.clone())
+    } else {
+        Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
+    };
+    (Ok(peer_epochs), journal_result)
+}
 
 #[derive(Clone, Debug)]
 pub struct ScannerPublicationLeaseGrant {
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
     topology_fingerprint: String,
     peer_epochs: Arc<BTreeMap<String, Uuid>>,
     expires_at: Instant,
+    generation: Arc<FleetCapabilityProofGeneration>,
 }
 
 impl FleetCapabilityProof {
+    fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
+        Self {
+            topology_fingerprint,
+            peer_epochs,
+            expires_at,
+            generation: FleetCapabilityProofGeneration::fresh(),
+        }
+    }
+
     fn token(&self) -> FleetCapabilityProofToken {
         FleetCapabilityProofToken {
             topology_fingerprint: self.topology_fingerprint.clone(),
             peer_epochs: self.peer_epochs.clone(),
         }
     }
+
+    #[cfg(any(test, feature = "test-util"))]
+    fn with_fresh_generation(&self) -> Self {
+        Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
+    }
+}
+
+/// Admission generation for effects that must not straddle a fleet-proof
+/// replacement. Revocation is deliberately non-blocking: it closes admission
+/// immediately, while the proof slot withholds the successor generation until
+/// every admitted operation has drained.
+#[derive(Default)]
+struct FleetCapabilityProofGeneration {
+    accepting: AtomicBool,
+    active: AtomicUsize,
+}
+
+impl FleetCapabilityProofGeneration {
+    fn fresh() -> Arc<Self> {
+        Arc::new(Self {
+            accepting: AtomicBool::new(true),
+            active: AtomicUsize::new(0),
+        })
+    }
+
+    fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
+        if !self.accepting.load(Ordering::Acquire) {
+            return None;
+        }
+        self.active.fetch_add(1, Ordering::AcqRel);
+        if self.accepting.load(Ordering::Acquire) {
+            Some(FleetCapabilityProofPermit {
+                generation: Arc::clone(self),
+            })
+        } else {
+            self.release();
+            None
+        }
+    }
+
+    fn revoke(&self) {
+        self.accepting.store(false, Ordering::Release);
+    }
+
+    fn is_accepting(&self) -> bool {
+        self.accepting.load(Ordering::Acquire)
+    }
+
+    fn is_drained(&self) -> bool {
+        self.active.load(Ordering::Acquire) == 0
+    }
+
+    fn release(&self) {
+        let previous = self.active.fetch_sub(1, Ordering::AcqRel);
+        debug_assert!(previous > 0, "fleet capability permit count underflow");
+    }
+}
+
+struct FleetCapabilityProofPermit {
+    generation: Arc<FleetCapabilityProofGeneration>,
+}
+
+impl Drop for FleetCapabilityProofPermit {
+    fn drop(&mut self) {
+        self.generation.release();
+    }
 }
 
 #[derive(Clone, PartialEq, Eq)]
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
 #[derive(Default)]
 struct FleetCapabilityProofState {
     proof: Option<FleetCapabilityProof>,
+    draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
     topology_conflict: bool,
 }
 
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
 #[derive(Clone, PartialEq, Eq)]
 pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
 
+/// A point-in-time proof that every current storage member implements the v6
+/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
+/// one acquisition authorizes one manifest construction attempt.
+pub(crate) struct TierDeleteJournalFleetProofToken {
+    token: FleetCapabilityProofToken,
+    _permit: FleetCapabilityProofPermit,
+}
+
 static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
+static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
     REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
-fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
-    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
+    TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
+}
+
+fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
+    if state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        state.draining_generation = None;
+    }
+}
+
+fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    revoke_fleet_capability_proof_state(&mut state);
+}
+
+fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.topology_conflict = true;
+    revoke_fleet_capability_proof_state(&mut state);
 }
 
 fn publish_fleet_capability_probe_result(
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
     match result {
         Ok(peer_epochs) => {
             let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-            let peer_epochs = state
+            if let Some(current) = state
                 .proof
-                .as_ref()
+                .as_mut()
                 .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
-                .map(|proof| Arc::clone(&proof.peer_epochs))
-                .unwrap_or_else(|| Arc::new(peer_epochs));
-            state.proof = Some(FleetCapabilityProof {
-                topology_fingerprint: topology_fingerprint.to_string(),
-                peer_epochs,
-                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
-            });
+            {
+                current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
+                return None;
+            }
+
+            if let Some(previous) = state.proof.take() {
+                previous.generation.revoke();
+                if !previous.generation.is_drained() {
+                    state.draining_generation = Some(previous.generation);
+                }
+            }
+            if state
+                .draining_generation
+                .as_ref()
+                .is_some_and(|generation| generation.is_drained())
+            {
+                state.draining_generation = None;
+            }
+            if state.draining_generation.is_some() {
+                return Some(Error::other(
+                    "fleet capability proof successor waits for the previous generation to drain",
+                ));
+            }
+            state.proof = Some(FleetCapabilityProof::new(
+                topology_fingerprint.to_string(),
+                Arc::new(peer_epochs),
+                observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            ));
             None
         }
         Err(err) => {
-            replace_fleet_capability_proof(slot, None);
+            revoke_fleet_capability_proof(slot);
             Some(err)
         }
     }
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
     fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
 }
 
-#[cfg(test)]
+pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_tier_delete_journal_fleet_proof_from(
+    state: &FleetCapabilityProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<TierDeleteJournalFleetProofToken> {
+    let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
+    let permit = state.proof.as_ref()?.generation.try_acquire()?;
+    Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
+}
+
+pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn tier_delete_journal_fleet_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &TierDeleteJournalFleetProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof._permit.generation.is_accepting()
+        && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
+        && state
+            .proof
+            .as_ref()
+            .is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
+}
+
+pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
+    stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
+        || state
+            .draining_generation
+            .as_ref()
+            .is_some_and(|generation| !generation.is_drained())
+}
+
+fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
+    hasher.update(topology_fingerprint.as_bytes());
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
         .get()
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let now = Instant::now();
+    let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
+        state.proof.clone()
+    } else {
+        Some(FleetCapabilityProof::new(
+            topology,
+            Arc::new(BTreeMap::new()),
+            now + Duration::from_secs(60 * 60),
+        ))
+    };
     state.topology_conflict = false;
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: topology,
-        peer_epochs: Arc::new(BTreeMap::new()),
-        expires_at: Instant::now() + Duration::from_secs(60 * 60),
-    });
+    state.proof = proof.clone();
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    debug_assert!(
+        journal_state
+            .proof
+            .as_ref()
+            .is_none_or(|current| current.generation.is_drained())
+    );
+    journal_state.topology_conflict = false;
+    journal_state.draining_generation = None;
+    journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
 }
 
 #[cfg(test)]
 pub(crate) struct CrossPoolFenceFleetProofGuard {
     previous_proof: Option<FleetCapabilityProof>,
     previous_topology_conflict: bool,
+    previous_journal_proof: Option<FleetCapabilityProof>,
+    previous_journal_topology_conflict: bool,
 }
 
 #[cfg(test)]
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
         let mut state = cross_pool_fence_fleet_proof_slot()
             .write()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
-        state.proof = self.previous_proof.take();
+        state.proof = self
+            .previous_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        state.draining_generation = None;
         state.topology_conflict = self.previous_topology_conflict;
+        drop(state);
+        let mut journal_state = tier_delete_journal_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        journal_state.proof = self
+            .previous_journal_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        journal_state.draining_generation = None;
+        journal_state.topology_conflict = self.previous_journal_topology_conflict;
     }
 }
 
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
     let guard = CrossPoolFenceFleetProofGuard {
         previous_proof: state.proof.clone(),
         previous_topology_conflict: state.topology_conflict,
+        previous_journal_proof: journal_state.proof.clone(),
+        previous_journal_topology_conflict: journal_state.topology_conflict,
     };
-    state.proof = None;
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
     state.topology_conflict = true;
+    if let Some(proof) = journal_state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            journal_state.draining_generation = Some(proof.generation);
+        }
+    }
+    journal_state.topology_conflict = true;
     guard
 }
 
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
     let Some(current) = state.proof.as_ref() else {
         return false;
     };
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: current.topology_fingerprint.clone(),
-        peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
-        expires_at: current.expires_at,
-    });
+    let proof = FleetCapabilityProof::new(
+        current.topology_fingerprint.clone(),
+        Arc::new(current.peer_epochs.as_ref().clone()),
+        current.expires_at,
+    );
+    state.proof = Some(proof.clone());
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    journal_state.topology_conflict = false;
+    if let Some(previous) = journal_state.proof.take() {
+        previous.generation.revoke();
+        if !previous.generation.is_drained() {
+            journal_state.draining_generation = Some(previous.generation);
+        }
+    }
+    if journal_state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        journal_state.draining_generation = None;
+    }
+    if journal_state.draining_generation.is_none() {
+        journal_state.proof = Some(proof.with_fresh_generation());
+    }
     true
 }
 
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
         return false;
     };
     let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
-    if state.topology_conflict {
-        return false;
-    }
-    state.proof.as_ref().is_some_and(|current| {
-        current.topology_fingerprint == *expected_topology
-            && current.topology_fingerprint == proof.topology_fingerprint
-            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
-            && Instant::now() < current.expires_at
-    })
+    fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn fleet_capability_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &FleetCapabilityProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    !state.topology_conflict
+        && state.proof.as_ref().is_some_and(|current| {
+            current.topology_fingerprint == expected_topology
+                && current.topology_fingerprint == proof.topology_fingerprint
+                && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+                && now < current.expires_at
+        })
 }
 
 fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
 #[cfg(test)]
 impl Drop for RemoteVersionStateFleetProofGuard {
     fn drop(&mut self) {
-        replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
+        revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
     }
 }
 
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
 pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
     if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
         if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
-            for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
-                let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-                state.topology_conflict = true;
-                state.proof = None;
+            for slot in [
+                remote_version_state_fleet_proof_slot(),
+                cross_pool_fence_fleet_proof_slot(),
+                tier_delete_journal_fleet_proof_slot(),
+            ] {
+                mark_fleet_capability_topology_conflict(slot);
             }
         }
         return;
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 }
                 None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
             };
-            let fence_result = match get_global_notification_sys() {
+            let fence_probe = match get_global_notification_sys() {
                 Some(notification_sys) => timeout(
                     REMOTE_VERSION_STATE_PROBE_TIMEOUT,
                     notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
                 None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
             };
+            let (fence_result, journal_result) = match fence_probe {
+                Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
+                Err(err) => {
+                    let message = err.to_string();
+                    (Err(Error::other(message.clone())), Err(Error::other(message)))
+                }
+            };
             let topology_conflict = remote_version_state_fleet_proof_slot()
                 .read()
                 .unwrap_or_else(std::sync::PoisonError::into_inner)
                 .topology_conflict;
             if topology_conflict {
-                replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
-                replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
+                revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
+                revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
+                revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                     event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_NOTIFICATION,
-                    capability = "cross_pool_fence_v2",
+                    capability = "cross_pool_fence",
+                    state = "failed_closed",
+                    error = %err,
+                    "notification capability probe"
+                );
+            }
+            if !topology_conflict
+                && let Some(err) = publish_fleet_capability_probe_result(
+                    tier_delete_journal_fleet_proof_slot(),
+                    &topology_fingerprint,
+                    journal_result,
+                    Instant::now(),
+                )
+            {
+                debug!(
+                    event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    capability = "tier_delete_journal_v6_policy",
                     state = "failed_closed",
                     error = %err,
                     "notification capability probe"
@@ -483,7 +811,7 @@ impl NotificationSys {
         Ok(peer_epochs)
     }
 
-    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
         if self.peer_clients.len() != self.peer_topology_hosts.len() {
             return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
         }
@@ -494,14 +822,21 @@ impl NotificationSys {
             client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
         });
         let mut peer_epochs = BTreeMap::new();
+        let mut minimum_version = u32::MAX;
         for result in join_all(probes).await {
             let (peer, version, epoch) = result?;
             if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
                 return Err(Error::other("cross-pool fence capability version is unsupported"));
             }
+            minimum_version = minimum_version.min(version);
             insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
         }
-        Ok(peer_epochs)
+        // A single-node deployment has no remote member to lower the local
+        // policy version advertised by this binary.
+        if minimum_version == u32::MAX {
+            minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
+        }
+        Ok((peer_epochs, minimum_version))
     }
 }
 
@@ -1827,12 +2162,13 @@ impl NotificationSys {
         join_all(futures).await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
+    pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter().cloned() {
+            let payload = canonical_prepare_payload.clone();
             futures.push(async move {
                 if let Some(client) = client {
-                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
+                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
                 } else {
                     unreachable_notification_peer_err()
                 }
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
+        let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
+        let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
+        assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
+        assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
+
+        let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
+        assert!(generic_v3.is_ok());
+        assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
         let now = Instant::now();
         let mut peer_epochs = BTreeMap::new();
         peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(peer_epochs),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
         assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
@@ -2495,11 +2839,7 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_accepts_single_node_membership() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::new()),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
     }
@@ -2507,21 +2847,97 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new(
+            "topology-a".to_string(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
         let captured = proof.token();
-        let restarted = FleetCapabilityProof {
-            topology_fingerprint: proof.topology_fingerprint.clone(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: proof.expires_at,
-        };
+        let restarted = FleetCapabilityProof::new(
+            proof.topology_fingerprint.clone(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            proof.expires_at,
+        );
 
         assert!(captured != restarted.token());
     }
 
+    #[test]
+    fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
+        let topology = "topology-a";
+        let now = Instant::now();
+        let node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let node_b_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let restarted_node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+
+        let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
+            .map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
+        assert_eq!(generations[0], generations[1]);
+        assert_eq!(generations[0], generations[2]);
+        assert_ne!(
+            generations[0],
+            stable_tier_delete_journal_topology_generation("topology-b"),
+            "a real topology change must produce a different durable generation"
+        );
+    }
+
+    #[test]
+    fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+        let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
+
+        let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
+                .is_none()
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        let fresh = state
+            .proof
+            .as_ref()
+            .expect("restart probe should publish a fresh proof")
+            .token();
+
+        assert!(!fleet_capability_proof_matches_at(
+            &state,
+            &original,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert!(fleet_capability_proof_matches_at(
+            &state,
+            &fresh,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert_eq!(
+            original_generation,
+            stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
         let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
@@ -2561,15 +2977,106 @@ mod tests {
         assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
     }
 
+    #[test]
+    fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(
+                tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+                "a freshly admitted journal proof must remain current"
+            );
+            assert!(
+                acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
+                    .is_none(),
+                "TTL expiry must stop new admission"
+            );
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(
+                    &state,
+                    &admitted,
+                    "topology-a",
+                    now + REMOTE_VERSION_STATE_PROOF_TTL,
+                ),
+                "TTL expiry must also stop an admitted proof at its next durable fence"
+            );
+            assert!(!admitted._permit.generation.is_drained());
+        }
+
+        let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let blocked = publish_fleet_capability_probe_result(
+            &slot,
+            "topology-a",
+            Ok(restarted_peers.clone()),
+            now + Duration::from_millis(1),
+        )
+        .expect("a successor proof must wait for the admitted generation");
+        assert!(blocked.to_string().contains("previous generation to drain"));
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
+            assert!(state.draining_generation.is_some());
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
+                "a restarted peer must revoke an admitted proof before its next durable fence"
+            );
+        }
+
+        drop(admitted);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
+                .is_none(),
+            "the successor may publish after the in-flight operation releases its permit"
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.proof.is_some());
+        assert!(state.draining_generation.is_none());
+    }
+
+    #[test]
+    fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+
+        mark_fleet_capability_topology_conflict(&slot);
+
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.topology_conflict);
+        assert!(state.proof.is_none());
+        assert!(state.draining_generation.is_some());
+        assert!(!admitted._permit.generation.is_accepting());
+        assert!(
+            !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+            "topology conflict must revoke an already admitted journal proof"
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
         let now = Instant::now();
         let mut state = FleetCapabilityProofState {
-            proof: Some(FleetCapabilityProof {
-                topology_fingerprint: "topology-a".to_string(),
-                peer_epochs: Arc::new(BTreeMap::new()),
-                expires_at: now + Duration::from_secs(1),
-            }),
+            proof: Some(FleetCapabilityProof::new(
+                "topology-a".to_string(),
+                Arc::new(BTreeMap::new()),
+                now + Duration::from_secs(1),
+            )),
+            draining_generation: None,
             topology_conflict: false,
         };
         assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
@@ -3279,7 +3786,7 @@ mod tests {
         assert_eq!(commit.len(), 1);
         assert!(commit[0].err.is_some());
 
-        let abort = sys.abort_tier_mutation(mutation_id).await;
+        let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
         assert_eq!(abort.len(), 1);
         assert!(abort[0].err.is_some());
     }

crates/ecstore/src/services/rebalance/control.rs

@@ -1329,8 +1329,9 @@ mod tests {
     use super::*;
     use crate::config::com::delete_config;
     use crate::core::pools::{
-        POOL_META_NAME, PoolActivationDurableSaveBarrier, PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState,
-        persist_pool_meta_identity_for_startup,
+        DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
+        PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
+        set_decommission_capacity_info_overrides_for_test,
     };
     use crate::object_api::NamespaceLockFence;
     use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
@@ -1751,26 +1752,15 @@ mod tests {
         ];
         set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
         set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
-        crate::core::pools::set_decommission_space_info_override_for_test(
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        let capacity_snapshot = vec![
+            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
+            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
+        ];
+        // Decommission start samples capacity before and inside its durable activation fence.
+        set_decommission_capacity_info_overrides_for_test(
             decommission_store.id,
-            vec![
-                (
-                    0,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 0,
-                        total: 100,
-                        used: 100,
-                    },
-                ),
-                (
-                    1,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 200,
-                        total: 200,
-                        used: 0,
-                    },
-                ),
-            ],
+            vec![capacity_snapshot.clone(), capacity_snapshot],
         );
         let (first_object, competing_object, competing_kind) = match paused_kind {
             PoolActivationStartKind::Rebalance => {

crates/ecstore/src/services/rebalance/entry.rs

@@ -356,7 +356,7 @@ impl ECStore {
             };
             run_guard.ensure_held("rebalance version migration")?;
             let result = migrate_entry_version(
-                &RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
+                &RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
                 bucket.clone(),
                 pool_index,
                 version,
@@ -1478,6 +1478,7 @@ mod tests {
         let (_temp_dirs, store, _unused_store) =
             crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
         prepare_rebalance_test_volumes(store.as_ref()).await;
+        crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
         let source_set = store.pools[0].get_disks_by_key(object);
         let target_set = store.pools[1].get_disks_by_key(object);
         let version_id = uuid::Uuid::new_v4();
@@ -1520,14 +1521,17 @@ mod tests {
         let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
         let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
         let barrier = TieredMetadataCommitBarrier::install(bucket, object);
-        let task = spawn_real_rebalance_entry(
+        let mut task = spawn_real_rebalance_entry(
             Arc::clone(&store),
             Arc::clone(&source_set),
             entry,
             REBALANCE_ID,
             Arc::new(RebalanceBucketConfigs::default()),
         );
-        barrier.wait_until_paused().await;
+        tokio::select! {
+            _ = barrier.wait_until_paused() => {}
+            result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
+        }
         run_signal_fence.mark_lost();
         barrier.release();
         drop(barrier);

crates/ecstore/src/services/rebalance/migration.rs

@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
 
 pub(crate) struct RebalanceMigrationBackend<'a> {
     source: &'a SetDisks,
-    store: &'a ECStore,
+    store: std::sync::Arc<ECStore>,
     lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 }
 
 impl<'a> RebalanceMigrationBackend<'a> {
     pub(crate) fn new(
         source: &'a SetDisks,
-        store: &'a ECStore,
+        store: std::sync::Arc<ECStore>,
         lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     ) -> Self {
         Self {

crates/ecstore/src/services/rebalance/mod.rs

@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
     test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
     rebalance_meta: Option<RebalanceMeta>,
 ) -> (

crates/ecstore/src/services/tier/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub mod test_util;
 pub mod tier;
 pub mod tier_admin;

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
 use serde::{Deserialize, Serialize};
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
 pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
 pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
+pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
 const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
 pub(crate) type TierMutationDigest = [u8; 32];
 
+static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
+    LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
+
+/// Serializes every local phase and recovery action for one mutation id while
+/// retaining bounded parallelism for unrelated mutations.
+pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
+    TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
+        .lock()
+        .await
+}
+
+fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
+    let raw = mutation_id.as_u128();
+    let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
+    // MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
+    // or relying on RandomState, whose seed differs between processes.
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
+    mixed ^= mixed >> 33;
+    (mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
+}
+
 pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
 
 #[derive(Debug, thiserror::Error)]
@@ -265,6 +290,30 @@ impl TierMutationIntent {
             && self.expires_at_unix_nanos == other.expires_at_unix_nanos
     }
 
+    /// Reconstruct the exact Prepared record that originally produced this
+    /// intent. Abort RPCs are identity-bound to that payload; serializing an
+    /// Aborted terminal record would both violate the wire contract and use a
+    /// different revision if a missing peer has to persist a tombstone.
+    pub(crate) fn original_prepared(&self) -> Result<Self> {
+        if self.state == TierMutationIntentState::Prepared {
+            self.validate()?;
+            return Ok(self.clone());
+        }
+        let mut prepared = self.clone();
+        prepared.revision =
+            prepared
+                .revision
+                .checked_sub(1)
+                .filter(|revision| *revision != 0)
+                .ok_or(TierMutationIntentError::Corrupt(
+                    "terminal intent cannot reconstruct its prepared revision",
+                ))?;
+        prepared.state = TierMutationIntentState::Prepared;
+        prepared.committed_config_etag = None;
+        prepared.validate()?;
+        Ok(prepared)
+    }
+
     pub(crate) fn encode(&self) -> Result<Vec<u8>> {
         self.validate()?;
         let intent_bytes = serde_json::to_vec(self)?;
@@ -439,6 +488,16 @@ where
     load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
+pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+) -> EcstoreResult<(TierMutationIntent, String)>
+where
+    S: EcstoreObjectIO,
+{
+    load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
 async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
     api: Arc<S>,
     prefix: &str,
@@ -507,6 +566,7 @@ where
     .await
 }
 
+#[cfg(test)]
 pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -514,13 +574,36 @@ where
     delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
-pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
+pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
 {
-    delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+    delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
+        .await
 }
 
+pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    delete_tier_mutation_intent_record_if_current_with_prefix(
+        api,
+        TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
+        mutation_id,
+        current_etag,
+    )
+    .await
+}
+
+#[cfg(test)]
 async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -533,6 +616,40 @@ where
     }
 }
 
+async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    if current_etag.trim().is_empty() {
+        return Err(Error::other("tier mutation intent current ETag is empty"));
+    }
+    let object =
+        tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
+    match api
+        .delete_object(
+            RUSTFS_META_BUCKET,
+            &object,
+            ObjectOptions {
+                http_preconditions: Some(HTTPPreconditions {
+                    if_match: Some(current_etag.to_string()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
+        Err(err) => Err(err),
+    }
+}
+
 pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
     api: Arc<S>,
     mutation_id: Uuid,
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::time::Duration;
 
     const OLD_IDENTITY: TierDestinationId = [1; 32];
     const NEW_IDENTITY: TierDestinationId = [2; 32];
@@ -736,6 +854,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn mutation_mutex_uses_exactly_64_stable_shards() {
+        assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
+        assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
+
+        let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
+        let shard = tier_mutation_mutex_shard_index(mutation_id);
+        assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
+        assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_serializes_the_same_id() {
+        let mutation_id = Uuid::new_v4();
+        let first = acquire_tier_mutation_mutex(mutation_id).await;
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            started_tx.send(()).expect("test receiver should remain alive");
+            let _second = acquire_tier_mutation_mutex(mutation_id).await;
+            acquired_tx.send(()).expect("test receiver should remain alive");
+        });
+        started_rx.await.expect("waiter should start");
+        assert!(
+            tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
+                .await
+                .is_err(),
+            "the same mutation id must not enter concurrently"
+        );
+
+        drop(first);
+        tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
+            .await
+            .expect("waiter should acquire after release")
+            .expect("waiter should report acquisition");
+        waiter.await.expect("waiter task should finish");
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_allows_different_shards_to_progress() {
+        let first_id = Uuid::new_v4();
+        let first_shard = tier_mutation_mutex_shard_index(first_id);
+        let second_id = (0..1024)
+            .map(|_| Uuid::new_v4())
+            .find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
+            .expect("a distinct shard should be easy to find");
+        let first = acquire_tier_mutation_mutex(first_id).await;
+
+        let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
+            .await
+            .expect("a different shard must not wait for the first mutation");
+        drop(first);
+    }
+
     #[test]
     fn intent_round_trip_preserves_committed_state() {
         let mut intent = prepared_intent();
@@ -873,6 +1046,37 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn terminal_intent_reconstructs_original_prepared_abort_payload() {
+        for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
+            let original = prepared_intent();
+            let mut intent = original.clone();
+            let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
+            intent
+                .advance(terminal, committed_etag)
+                .expect("terminal transition should succeed");
+
+            let reconstructed = intent
+                .original_prepared()
+                .expect("terminal record should recover prepared payload");
+            assert_eq!(reconstructed, original);
+            assert!(intent.same_identity_as(&reconstructed));
+        }
+    }
+
+    #[test]
+    fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
+        let mut corrupt = prepared_intent();
+        corrupt.state = TierMutationIntentState::Aborted;
+
+        assert!(matches!(
+            corrupt.original_prepared(),
+            Err(TierMutationIntentError::Corrupt(
+                "terminal intent cannot reconstruct its prepared revision"
+            ))
+        ));
+    }
+
     #[test]
     fn intent_validation_rejects_placeholder_identity() {
         let mut intent = prepared_intent();

crates/ecstore/src/services/tier/tier_mutation_peer.rs

@@ -15,12 +15,13 @@
 use std::sync::Arc;
 
 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
+use time::OffsetDateTime;
 use uuid::Uuid;
 
 use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
 use super::tier_mutation_intent::{
-    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
-    load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
+    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
+    advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
 };
 use crate::error::{Error, StorageError};
 use crate::store::ECStore;
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
     CommitProofMismatch,
     #[error("tier mutation peer abort proof does not match the persisted tier configuration")]
     AbortProofMismatch,
+    #[error("tier mutation peer prepared intent has expired")]
+    ExpiredIntent,
     #[error("tier mutation peer runtime error: {0}")]
     Runtime(#[source] AdminError),
     #[error("tier mutation peer store error: {0}")]
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
     validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
+    let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
     match phase {
         TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
         TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
@@ -103,43 +107,57 @@ async fn handle_prepare(
     }
     let tier_config_mgr = api.tier_config_mgr();
 
-    match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
-        Ok(()) => {
-            TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
-                .await
-                .map_err(TierMutationPeerError::Runtime)?;
-            Ok(TierMutationPeerOutcome {
-                state: TierMutationPeerState::Prepared,
-                applied: true,
-            })
-        }
-        Err(Error::PreconditionFailed) => {
-            let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
-            if !existing.same_identity_as(&intent) {
-                return Err(TierMutationPeerError::ConflictingIntent);
+    for _ in 0..3 {
+        let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&intent) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                (existing, false)
             }
-            match existing.state {
-                TierMutationIntentState::Prepared => {
-                    TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
+            Err(Error::ConfigNotFound) => {
+                let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
+                if intent.expires_at_unix_nanos <= now {
+                    return Err(TierMutationPeerError::ExpiredIntent);
                 }
-                TierMutationIntentState::Committed => {
-                    TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
-                }
-                TierMutationIntentState::Aborted => {
-                    TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
+                    Ok(()) => (intent.clone(), true),
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
                 }
             }
-            Ok(TierMutationPeerOutcome {
-                state: peer_state_from_intent(existing.state),
-                applied: false,
-            })
+            Err(err) => return Err(err.into()),
+        };
+
+        match stored.state {
+            TierMutationIntentState::Prepared => {
+                TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Committed => {
+                TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Aborted => {
+                TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+            }
         }
-        Err(err) => Err(err.into()),
+        return Ok(TierMutationPeerOutcome {
+            state: peer_state_from_intent(stored.state),
+            applied,
+        });
     }
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation prepare raced repeatedly with another decision",
+    )))
 }
 
 async fn handle_commit(
@@ -201,26 +219,106 @@ async fn handle_abort(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
-    if !canonical_payload.is_empty() {
-        return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
+    let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+    if prepared.state != TierMutationIntentState::Prepared {
+        return Err(TierMutationPeerError::InvalidPayload(
+            "abort payload must carry the original prepared intent".to_string(),
+        ));
     }
-    let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
-    if existing.state == TierMutationIntentState::Prepared
-        && !tier_config_abort_matches(api.clone(), &existing)
-            .await
-            .map_err(Error::other)?
-    {
-        return Err(TierMutationPeerError::AbortProofMismatch);
+    let mut tombstone = prepared.clone();
+    tombstone
+        .advance(TierMutationIntentState::Aborted, None)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+
+    for _ in 0..3 {
+        match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&prepared) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                match existing.state {
+                    TierMutationIntentState::Committed => {
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Committed,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Aborted => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Prepared => {}
+                }
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                let advanced = advance_tier_mutation_intent_record_idempotent(
+                    api.clone(),
+                    mutation_id,
+                    TierMutationIntentState::Aborted,
+                    None,
+                )
+                .await;
+                let (intent, applied) = match advanced {
+                    Ok(result) => result,
+                    Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+                        Ok(current)
+                            if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
+                        {
+                            (current, false)
+                        }
+                        _ => return Err(err.into()),
+                    },
+                };
+                if intent.state == TierMutationIntentState::Aborted {
+                    TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                    TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                        .await
+                        .map_err(TierMutationPeerError::Runtime)?;
+                }
+                return Ok(TierMutationPeerOutcome {
+                    state: peer_state_from_intent(intent.state),
+                    applied,
+                });
+            }
+            Err(Error::ConfigNotFound) => {
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
+                    Ok(()) => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: true,
+                        });
+                    }
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
+                }
+            }
+            Err(err) => return Err(err.into()),
+        }
     }
-    let (intent, applied) =
-        advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
-    if intent.state == TierMutationIntentState::Aborted {
-        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
-    }
-    Ok(TierMutationPeerOutcome {
-        state: peer_state_from_intent(intent.state),
-        applied,
-    })
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation abort raced repeatedly with prepare",
+    )))
 }
 
 fn validate_peer_request_envelope(
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<()> {
-    if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
+    if !matches!(
+        protocol_version,
+        rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
+    ) {
         return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
     }
     if mutation_id.is_nil() {
@@ -276,6 +377,8 @@ mod tests {
     #[test]
     fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
         let mutation_id = Uuid::new_v4();
+        validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
+            .expect("v3 must remain accepted during the v4 rollout");
         assert!(matches!(
             validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
             Err(TierMutationPeerError::UnsupportedProtocolVersion(_))

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -41,22 +41,22 @@ use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
 #[cfg(test)]
 use super::super::get_metadata_slowtail_fault_delay;
 use super::super::{
-    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
-    EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion, GetCodecStreamingFallbackReason,
-    GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource,
-    LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions,
-    ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq,
-    ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError,
-    UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
-    capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason,
-    current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info,
-    inline_erasure_shard_file_offset, inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found,
-    is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
-    is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
-    is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
-    merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
-    reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
-    try_read_inline_data_shards_direct, warn,
+    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+    EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion,
+    GetCodecStreamingFallbackReason, GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult,
+    HealChannelPriority, HealRequestSource, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext,
+    OBJECT_OP_IGNORED_ERRS, ObjectOptions, ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET,
+    RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks,
+    SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs,
+    can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
+    collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
+    file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
+    inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
+    is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
+    is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
+    issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
+    object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
+    send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
 };
 #[cfg(test)]
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
@@ -3733,16 +3733,34 @@ fn dangling_delete_grace() -> time::Duration {
 /// Result of scanning one disk's copy of a directory prefix while deciding
 /// whether an orphan (metadata-less) directory tree can be safely purged.
 enum OrphanDirScan {
-    /// The subtree holds at least one regular file (object metadata or data), so
-    /// it is a real object and must not be purged.
+    /// The subtree holds object metadata or uncommitted data, so it must not be
+    /// purged.
     HasData,
-    /// The prefix exists on this disk and contains only nested empty directories.
-    /// Carries every directory path in pre-order (parents before children).
-    Empty(Vec<String>),
+    /// The prefix contains only empty directories and/or UUID data directories
+    /// carrying a committed delete marker.
+    Purgeable {
+        empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    },
     /// The prefix does not exist on this disk.
     Missing,
 }
 
+fn is_safe_orphan_dir_entry(entry: &str) -> bool {
+    let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
+    !component.is_empty()
+        && component != "."
+        && component != ".."
+        && !component.contains(SLASH_SEPARATOR)
+        && !component.contains('\\')
+}
+
+fn is_committed_delete_marker(entry: &str) -> bool {
+    entry
+        .strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
+        .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil()))
+}
+
 /// Outcome of a *post-quorum* `rename_data` commit, classifying whether the
 /// committed replicas converged so the caller can decide heal admission
 /// WITHOUT conflating "a version signature exists" with "this write needs
@@ -6125,52 +6143,151 @@ impl SetDisks {
     }
 
     /// Scan a single disk's copy of `prefix` and decide whether it is an orphan
-    /// (metadata-less) directory subtree. Walks the tree iteratively and returns
-    /// [`OrphanDirScan::HasData`] as soon as any regular file is found.
+    /// directory subtree. Only empty directories and UUID data directories with
+    /// valid committed delete markers are purgeable; every child is still scanned.
     async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
         let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
         let mut stack = vec![root.clone()];
         // Pre-order list of directories (a parent always precedes its descendants),
         // so reversing it yields a safe children-first removal order.
         let mut dirs: Vec<String> = Vec::new();
+        let mut committed_files: Vec<String> = Vec::new();
         let mut existed = false;
 
         while let Some(dir) = stack.pop() {
             let entries = match disk.list_dir("", bucket, &dir, 0).await {
                 Ok(entries) => entries,
-                Err(_) => {
-                    // The root missing (or never existing) means there is nothing to
-                    // purge on this disk. A nested directory vanishing mid-scan is a
-                    // benign race, so skip it and keep walking.
+                Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {
                     if dir == root {
                         return OrphanDirScan::Missing;
                     }
+                    // A nested directory vanishing mid-scan is a benign race.
                     continue;
                 }
+                // Classification must fail closed: committed residue is safe to
+                // remove only after every reachable child was inspected.
+                Err(_) => return OrphanDirScan::HasData,
             };
 
             existed = true;
-            dirs.push(dir.clone());
+            let mut child_dirs = Vec::new();
+            let mut files = Vec::new();
 
             for entry in entries {
+                if !is_safe_orphan_dir_entry(&entry) {
+                    return OrphanDirScan::HasData;
+                }
                 match entry.strip_suffix(SLASH_SEPARATOR) {
-                    // `read_dir` marks directories with a trailing slash; anything else
-                    // is a regular file, which means real object data lives here.
-                    Some(child) => stack.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
-                    None => return OrphanDirScan::HasData,
+                    Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
+                    None => files.push(entry),
                 }
             }
+
+            if !files.is_empty() {
+                let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
+                let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
+                let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
+
+                if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
+                    return OrphanDirScan::HasData;
+                }
+
+                committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
+                dirs.push(dir);
+                stack.extend(child_dirs);
+                continue;
+            }
+
+            dirs.push(dir);
+            stack.extend(child_dirs);
         }
 
         if existed {
-            OrphanDirScan::Empty(dirs)
+            OrphanDirScan::Purgeable {
+                empty_dirs: dirs,
+                committed_files,
+            }
         } else {
             OrphanDirScan::Missing
         }
     }
 
+    async fn delete_purgeable_orphan_entries(
+        disk: &DiskStore,
+        bucket: &str,
+        object: &str,
+        mut empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    ) {
+        // Keep every committed marker until all ordinary residue files are gone.
+        // If any delete fails, a later request can still recognize and retry the
+        // committed cleanup instead of stranding an unmarked partial residue.
+        for delete_markers in [false, true] {
+            for file in &committed_files {
+                let is_marker = file.rsplit(SLASH_SEPARATOR).next().is_some_and(is_committed_delete_marker);
+                if is_marker != delete_markers {
+                    continue;
+                }
+                if let Err(err) = disk
+                    .delete(
+                        bucket,
+                        file,
+                        DeleteOptions {
+                            recursive: false,
+                            immediate: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                {
+                    debug!(
+                        event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_SET_DISK,
+                        bucket,
+                        object,
+                        path = file,
+                        error = ?err,
+                        "Orphan prefix purge skipped"
+                    );
+                    return;
+                }
+            }
+        }
+        empty_dirs.reverse();
+        for dir in empty_dirs {
+            if let Err(err) = disk
+                .delete(
+                    bucket,
+                    &dir,
+                    DeleteOptions {
+                        recursive: false,
+                        immediate: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                // Best effort: a sibling removal may have already cleared a shared
+                // parent, or a concurrent writer repopulated the directory. Neither
+                // is fatal to purging the orphan tree.
+                debug!(
+                    event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object,
+                    path = dir,
+                    error = ?err,
+                    "Orphan prefix purge skipped"
+                );
+            }
+        }
+    }
+
     /// Purge an orphan directory prefix — a trailing-slash key that exists on disk
-    /// as an empty directory tree with no object metadata on any disk of this set.
+    /// as empty directories or committed delete residue, with no object metadata
+    /// or uncommitted data on any disk of this set.
     /// Such prefixes are listable (see `scan_dir`) yet are not real objects, so the
     /// normal delete path returns NotFound and leaves them stranded (issue #4189).
     ///
@@ -6187,15 +6304,18 @@ impl SetDisks {
         // Phase 1: classify every online disk. Refuse to purge if ANY disk holds
         // object data under the prefix, so a degraded/healable object is never
         // destroyed.
-        let mut per_disk_dirs: Vec<(usize, Vec<String>)> = Vec::new();
+        let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
         let mut existed = false;
         for (i, disk) in disks.iter().enumerate() {
             let Some(disk) = disk else { continue };
             match Self::scan_orphan_dir(disk, bucket, object).await {
                 OrphanDirScan::HasData => return Ok(false),
-                OrphanDirScan::Empty(dirs) => {
+                OrphanDirScan::Purgeable {
+                    empty_dirs,
+                    committed_files,
+                } => {
                     existed = true;
-                    per_disk_dirs.push((i, dirs));
+                    per_disk_dirs.push((i, empty_dirs, committed_files));
                 }
                 OrphanDirScan::Missing => {}
             }
@@ -6205,32 +6325,14 @@ impl SetDisks {
             return Ok(false);
         }
 
-        // Phase 2: remove the empty directories children-first on each disk. A
-        // non-recursive delete performs an empty-only `rmdir`, so a directory that
-        // concurrently gained an object fails with DirectoryNotEmpty and is skipped —
-        // a racing PutObject is never clobbered.
-        for (i, mut dirs) in per_disk_dirs {
+        // Phase 2: remove only the files classified as committed residue, then
+        // remove directories children-first. Every directory delete is
+        // non-recursive, so a directory that concurrently gained an object fails
+        // with DirectoryNotEmpty and is skipped — a racing PutObject is never
+        // clobbered.
+        for (i, empty_dirs, committed_files) in per_disk_dirs {
             let Some(disk) = disks[i].as_ref() else { continue };
-            dirs.reverse();
-            for dir in dirs {
-                if let Err(err) = disk
-                    .delete(
-                        bucket,
-                        &dir,
-                        DeleteOptions {
-                            recursive: false,
-                            immediate: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                {
-                    // Best effort: a sibling removal may have already cleared a shared
-                    // parent, or a concurrent writer repopulated the directory. Neither
-                    // is fatal to purging the orphan tree.
-                    debug!(bucket, object, dir, error = ?err, "purge_orphan_dir_object: skipped non-empty/absent directory");
-                }
-            }
+            Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
         }
 
         Ok(true)
@@ -7049,6 +7151,16 @@ mod tests {
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
 
+    #[test]
+    fn orphan_dir_entries_must_be_single_relative_components() {
+        for entry in ["part.1", "child/", "delete-data.00000000-0000-0000-0000-000000000001"] {
+            assert!(is_safe_orphan_dir_entry(entry), "{entry:?} should be accepted");
+        }
+        for entry in ["", "/", ".", "..", "../", "child//", "a/b", r"a\b", "./"] {
+            assert!(!is_safe_orphan_dir_entry(entry), "{entry:?} should be rejected");
+        }
+    }
+
     #[test]
     #[serial_test::serial(codec_streaming_env)]
     fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() {
@@ -7227,6 +7339,96 @@ mod tests {
         (dir, disk)
     }
 
+    #[tokio::test]
+    async fn orphan_cleanup_preserves_object_published_after_scan() {
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let transaction = Uuid::new_v4();
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+
+        let nested_object = residue.join("nested");
+        tokio::fs::create_dir_all(&nested_object)
+            .await
+            .expect("concurrent object directory should be created");
+        tokio::fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"new metadata")
+            .await
+            .expect("concurrent object metadata should be written");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        assert!(
+            nested_object.join(STORAGE_FORMAT_FILE).exists(),
+            "an object published after classification must survive cleanup"
+        );
+        assert!(!residue.join("part.1").exists(), "classified stale data should be reclaimed");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn orphan_cleanup_keeps_commit_marker_when_residue_delete_fails() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4());
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(&marker_name), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
+            .await
+            .expect("residue directory should become read-only");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        let part_remains = residue.join("part.1").exists();
+        let marker_remains = residue.join(marker_name).exists();
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o755))
+            .await
+            .expect("residue directory permissions should be restored");
+        assert!(part_remains, "the injected residue delete failure should retain the part");
+        assert!(marker_remains, "the commit marker must remain so a later cleanup can retry");
+    }
+
     async fn io_primitives_test_set(disks: Vec<Option<DiskStore>>, default_parity_count: usize) -> Arc<SetDisks> {
         let set_drive_count = disks.len();
         SetDisks::new(

crates/ecstore/src/set_disk/mod.rs

@@ -127,7 +127,7 @@ use rustfs_filemeta::{
 };
 use rustfs_heal_contracts::heal_channel::{
     DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
-    send_heal_disk, send_heal_request_with_admission,
+    send_heal_replacement_disk, send_heal_request_with_admission,
 };
 use rustfs_io_metrics::{
     record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
@@ -329,6 +329,7 @@ const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
 const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
 const EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED: &str = "set_disk_rename_tail_drain_failed";
 const EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY: &str = "set_disk_put_object_stage_summary";
+const EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED: &str = "set_disk_orphan_purge_skipped";
 const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
 const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
 const DEFAULT_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 64 * 1024 * 1024;
@@ -858,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
 
 mod core;
 #[cfg(test)]
-pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
+pub(crate) use core::io_primitives::disk_call_counters;
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
 mod ctx;
 mod metadata;
 mod ops;
@@ -871,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
 #[cfg(test)]
 pub(crate) use ops::object::DeleteObjectCommitBarrier;
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
@@ -1638,10 +1641,13 @@ mod prepared_get_object_metadata_tests {
         .await;
     }
 
-    #[tokio::test]
+    #[test]
     #[serial_test::serial(body_cache_hook)]
-    async fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
-        let (_dirs, set_disks) = make_local_set_disks(6, 2).await;
+    fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("current-thread runtime should build");
         let bucket = "inline-read-plan-unequal";
         let object = object_with_initial_data_shards_for_geometry(bucket, "inline-object", 6, 2);
         let payload = b"inline quorum payload".repeat(256);
@@ -1650,55 +1656,65 @@ mod prepared_get_object_metadata_tests {
             ..Default::default()
         };
 
-        set_disks
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("bucket should be created");
-        let mut put_reader = PutObjReader::from_vec(payload.clone());
-        set_disks
-            .put_object(bucket, &object, &mut put_reader, &opts)
-            .await
-            .expect("inline object should be written");
-
-        let read_once = |enabled: bool| {
-            let set_disks = Arc::clone(&set_disks);
-            let bucket = bucket.to_string();
-            let object = object.clone();
-            let payload = payload.clone();
-            let opts = opts.clone();
-            async move {
-                temp_env::async_with_vars(
-                    [
-                        (
-                            "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
-                            Some(if enabled { "true" } else { "false" }),
-                        ),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
-                    ],
-                    async {
-                        let calls = disk_call_counters::observe(&object);
-                        let mut reader = set_disks
-                            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &opts)
-                            .await
-                            .expect("inline GET reader should open");
-                        let mut restored = Vec::new();
-                        reader
-                            .stream
-                            .read_to_end(&mut restored)
-                            .await
-                            .expect("inline GET body should stream");
-                        assert_eq!(restored, payload);
-                        calls.total(disk_call_counters::KIND_READ_VERSION)
-                    },
-                )
+        let (_dirs, set_disks) = runtime.block_on(async {
+            let (dirs, set_disks) = make_local_set_disks(6, 2).await;
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
                 .await
-            }
-        };
+                .expect("bucket should be created");
+            let mut put_reader = PutObjReader::from_vec(payload.clone());
+            set_disks
+                .put_object(bucket, &object, &mut put_reader, &opts)
+                .await
+                .expect("inline object should be written");
+            (dirs, set_disks)
+        });
 
-        let gate_off_calls = read_once(false).await;
-        let gate_on_calls = read_once(true).await;
-        assert_eq!(gate_on_calls, gate_off_calls, "inline gate must not add reserve fanout");
+        // disk_call_counters counts tasks that started running, so an
+        // early-stop abort races the single-pending inline hedge into a ±1
+        // count per read. The fanout lifecycle histogram records the
+        // scheduling decision itself and stays deterministic under load.
+        let recorder = CapturingRecorder::default();
+        let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
+        rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            runtime.block_on(async {
+                for enabled in [false, true] {
+                    temp_env::async_with_vars(
+                        [
+                            (
+                                "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
+                                Some(if enabled { "true" } else { "false" }),
+                            ),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
+                        ],
+                        async {
+                            let mut reader = set_disks
+                                .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
+                                .await
+                                .expect("inline GET reader should open");
+                            let mut restored = Vec::new();
+                            reader
+                                .stream
+                                .read_to_end(&mut restored)
+                                .await
+                                .expect("inline GET body should stream");
+                            assert_eq!(restored, payload);
+                        },
+                    )
+                    .await;
+                }
+            })
+        });
+        rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
+
+        let scheduled = recorder.histogram_values(
+            "rustfs_io_get_object_metadata_fanout_scheduled",
+            &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
+        );
+        assert_eq!(scheduled.len(), 2, "each GET should run exactly one metadata fanout");
+        assert_eq!(scheduled[1], scheduled[0], "inline gate must not add reserve fanout");
     }
 
     #[test]
@@ -8775,6 +8791,111 @@ mod tests {
         assert!(root.join("bucket").exists(), "bucket volume should remain");
     }
 
+    #[tokio::test]
+    async fn purge_orphan_dir_object_removes_committed_delete_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let data_dir = Uuid::new_v4();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("nested")
+            .join("object")
+            .join(data_dir.to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed delete residue should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("purge should succeed");
+
+        assert!(purged, "committed delete residue should be purgeable");
+        assert!(!root.join("bucket").join("pfx").exists(), "prefix directory should be gone");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_uncommitted_data_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("uncommitted data residue should be created");
+        fs::write(residue.join("part.1"), b"possibly live")
+            .await
+            .expect("data part should be written");
+        fs::write(residue.join("delete-data.not-a-uuid"), [])
+            .await
+            .expect("malformed marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "data without a valid committed marker must be preserved");
+        assert!(residue.join("part.1").exists(), "possibly live data must remain");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_nested_object_below_committed_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+        let nested_object = residue.join("nested");
+        fs::create_dir_all(&nested_object)
+            .await
+            .expect("nested object directory should be created");
+        fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"meta")
+            .await
+            .expect("nested object metadata should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "nested object metadata must veto committed-residue cleanup");
+        assert!(nested_object.join(STORAGE_FORMAT_FILE).exists(), "nested object metadata must remain");
+        assert!(residue.join("part.1").exists(), "committed residue must remain when cleanup is vetoed");
+    }
+
     // issue #4189: a prefix that still anchors a real object must be left intact.
     #[tokio::test]
     async fn purge_orphan_dir_object_preserves_prefix_with_object() {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -2812,9 +2812,20 @@ mod heal_result_report_tests {
             }
 
             let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-            set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("source object should be written");
+            // This fixture reads and removes physical shards immediately after
+            // PUT. A lock-owning PUT may quorum-ack before its rename tail
+            // drains, so keep the setup on the full-fanout commit path.
+            set.put_object(
+                &bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("source object should be written");
             let source = disks[2]
                 .read_version("", &bucket, object, "", &ReadOptions::default())
                 .await
@@ -3230,9 +3241,20 @@ mod heal_result_report_tests {
         }
 
         let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-        set.put_object(bucket, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("source object should be written");
+        // The target-evidence readback below asserts per-disk state right
+        // after PUT. A lock-owning PUT may quorum-ack before its rename tail
+        // drains, so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source object should be written");
         let source = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
             .await
@@ -3279,6 +3301,9 @@ mod heal_result_report_tests {
         .await
         .expect("versioned bucket should be created");
 
+        // The per-version target-evidence readback below asserts per-disk
+        // state right after PUT. A lock-owning PUT may quorum-ack before its
+        // rename tail drains, so keep the setup on the full-fanout commit path.
         let mut old_reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
         let old_info = set
             .put_object(
@@ -3287,6 +3312,7 @@ mod heal_result_report_tests {
                 &mut old_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -3304,6 +3330,7 @@ mod heal_result_report_tests {
                 &mut latest_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -4325,9 +4352,20 @@ mod heal_result_report_tests {
 
         const PAYLOAD_SIZE: usize = 1024 * 1024;
         let mut initial_reader = PutObjReader::from_vec(vec![0x11; PAYLOAD_SIZE]);
-        set.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
-            .await
-            .expect("initial object should be written");
+        // This fixture reads and removes physical shards immediately after
+        // PUT. A lock-owning PUT may quorum-ack before its rename tail drains,
+        // so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut initial_reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("initial object should be written");
 
         let current = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
@@ -4415,11 +4453,12 @@ mod heal_result_report_tests {
                 // Give the heal something to rebuild on alternating rounds: remove a
                 // shard of the current data dir right before the race.
                 if round % 2 == 1 {
-                    let current = disks[2]
-                        .read_version("", bucket, object, "", &ReadOptions::default())
-                        .await
-                        .expect("current metadata should be readable");
-                    if let Some(data_dir) = current.data_dir {
+                    // The previous round's lock-owning PUT may still be
+                    // draining its rename tail on this disk; shard damage is
+                    // best-effort here, so skip injection when it lags.
+                    if let Ok(current) = disks[2].read_version("", bucket, object, "", &ReadOptions::default()).await
+                        && let Some(data_dir) = current.data_dir
+                    {
                         let shard = temp_dirs[3]
                             .path()
                             .join(bucket)

crates/ecstore/src/set_disk/ops/locking.rs

@@ -22,12 +22,10 @@
 use super::super::{
     Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
     NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
-    send_heal_disk, warn,
+    send_heal_replacement_disk, warn,
 };
-use crate::disk::DiskAPI;
 use crate::disk::health_state::DriveMembershipSnapshot;
-#[cfg(test)]
-use crate::disk::new_disk;
+use crate::disk::{DiskAPI, new_disk};
 use crate::runtime::sources as runtime_sources;
 use rand::prelude::SliceRandom;
 #[cfg(test)]
@@ -356,11 +354,28 @@ impl SetDisks {
             Ok(res) => res,
             Err(e) => {
                 warn!("renew_disk: connect_endpoint err {:?}", &e);
-                if ep.is_local && e == DiskError::UnformattedDisk {
-                    info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep);
-                    let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx);
-                    let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await;
+                if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
+                    return;
                 }
+
+                let attached = match self.attach_unformatted_replacement_disk(ep).await {
+                    Ok(attached) => attached,
+                    Err(err) => {
+                        warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
+                        return;
+                    }
+                };
+                if !attached {
+                    return;
+                }
+
+                info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
+                let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
+                    warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
+                    return;
+                };
+                let _ =
+                    send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
                 return;
             }
         };
@@ -412,6 +427,60 @@ impl SetDisks {
         disk_lock[disk_idx] = Some(new_disk);
     }
 
+    /// Attach a replacement target only after proving that the exact local slot
+    /// is present and unformatted. A health-checked reconnect may reject a
+    /// blank target before it reaches the format-heal path; that target still
+    /// has to be visible in this set for the formatter to claim it safely.
+    async fn attach_unformatted_replacement_disk(&self, ep: &Endpoint) -> disk::error::Result<bool> {
+        if !ep.is_local
+            || usize::try_from(ep.pool_idx).ok() != Some(self.pool_index)
+            || usize::try_from(ep.set_idx).ok() != Some(self.set_index)
+        {
+            return Ok(false);
+        }
+
+        let Some(disk_idx) = self.set_endpoints.iter().position(|candidate| candidate == ep) else {
+            return Ok(false);
+        };
+
+        let replacement = new_disk(
+            ep,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await?;
+        match load_format_erasure(&replacement, false).await {
+            Err(DiskError::UnformattedDisk) => {}
+            Ok(_) => return Ok(false),
+            Err(err) => return Err(err),
+        }
+
+        {
+            let mut disks = self.disks.write().await;
+            if disks[disk_idx].as_ref().is_some_and(|existing| existing.endpoint() != *ep) {
+                warn!(endpoint = %ep, disk_idx, "renew_disk rejected unformatted replacement for an occupied foreign slot");
+                return Ok(false);
+            }
+            disks[disk_idx] = Some(replacement.clone());
+        }
+
+        let local_disk_map = runtime_sources::local_disk_map_handle();
+        local_disk_map
+            .write()
+            .await
+            .insert(replacement.endpoint().to_string(), Some(replacement.clone()));
+
+        if runtime_sources::setup_is_dist_erasure().await {
+            let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
+            let mut local_set_drives = local_disk_set_drives.write().await;
+            local_set_drives[self.pool_index][self.set_index][disk_idx] = Some(replacement);
+        }
+
+        Ok(true)
+    }
+
     pub(in crate::set_disk) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> {
         self.format.check_other(fm)?;
 
@@ -779,6 +848,75 @@ mod tests {
         drop(temp_dirs);
     }
 
+    #[tokio::test]
+    async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
+        let disk_count = 4;
+        let format = FormatV3::new(1, disk_count);
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count - 1 {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let replacement_dir = tempfile::tempdir().expect("replacement tempdir should be created");
+        let mut replacement_endpoint =
+            Endpoint::try_from(replacement_dir.path().to_str().expect("replacement path should be utf8"))
+                .expect("replacement endpoint should parse");
+        replacement_endpoint.set_pool_index(0);
+        replacement_endpoint.set_set_index(0);
+        replacement_endpoint.set_disk_index(disk_count - 1);
+        temp_dirs.push(replacement_dir);
+        endpoints.push(replacement_endpoint.clone());
+        disks.push(None);
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        assert!(
+            set_disks
+                .attach_unformatted_replacement_disk(&replacement_endpoint)
+                .await
+                .expect("a blank local replacement should be admitted")
+        );
+
+        let attached = set_disks.get_disks_internal().await;
+        let replacement = attached[disk_count - 1]
+            .as_ref()
+            .expect("the verified replacement must occupy its exact set slot");
+        assert_eq!(replacement.endpoint(), replacement_endpoint);
+        assert!(
+            !replacement.health_check_enabled_for_test(),
+            "the blank replacement must not start health checks before it receives a format"
+        );
+        assert_eq!(
+            load_format_erasure(replacement, false).await.unwrap_err(),
+            DiskError::UnformattedDisk,
+            "only a still-unformatted replacement may be attached by the fallback"
+        );
+
+        runtime_sources::local_disk_map_handle()
+            .write()
+            .await
+            .remove(&replacement_endpoint.to_string());
+        drop(set_disks);
+        drop(temp_dirs);
+    }
+
     #[tokio::test]
     async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
         let disk_count = 3;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
 }
 
 #[cfg(any(test, feature = "test-util"))]
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum MultipartCommitPause {
     NewUploadBeforeLockLost,
     PutPartBeforeLockAcquire,
@@ -5459,19 +5459,42 @@ mod tests {
             disk.make_volume(bucket).await.expect("bucket volume should be created");
         }
         let mut initial_reader = PutObjReader::from_vec(b"old multipart body".to_vec());
+        // A lock-owning PUT may quorum-ack before its rename tail drains, and
+        // cache priming refuses to publish while a straggler disk still reads
+        // as an error; keep the setup on the full-fanout commit path.
         set_disks
-            .put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
+            .put_object(
+                bucket,
+                object,
+                &mut initial_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
             .await
             .expect("initial object should be written");
-        set_disks
-            .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
-            .await
-            .expect("initial metadata should resolve");
-        let generation = set_disks
-            .get_object_metadata_cache_generation(bucket, object)
-            .expect("metadata cache generation should be active");
-        let retired_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
-        assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_some());
+        // The publish is also bounded by the cache TTL, so re-prime until the
+        // current generation is observably cached instead of asserting on a
+        // single read that a loaded host can stall past expiry.
+        let retired_key = tokio::time::timeout(std::time::Duration::from_secs(30), async {
+            loop {
+                set_disks
+                    .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+                    .await
+                    .expect("initial metadata should resolve");
+                let generation = set_disks
+                    .get_object_metadata_cache_generation(bucket, object)
+                    .expect("metadata cache generation should be active");
+                let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+                if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
+                    return key;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("metadata priming should publish the current generation");
 
         let upload = set_disks
             .new_multipart_upload(bucket, object, &ObjectOptions::default())

crates/ecstore/src/store/bucket.rs

@@ -29,6 +29,25 @@ use std::future::Future;
 
 const DELETED_BUCKETS_PREFIX: &str = ".deleted";
 const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
+const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
+
+fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
+    metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
+    debug!(
+        event = EVENT_BUCKET_DELETE_BLOCKED,
+        component = "ecstore",
+        subsystem = "bucket",
+        bucket,
+        blocker = kind.as_str(),
+        files = residue.files,
+        uuid_data_dirs = residue.uuid_data_dirs,
+        entries_scanned = residue.entries_scanned,
+        diagnostic_bytes_read = residue.diagnostic_bytes_read,
+        diagnostic_truncated = residue.diagnostic_truncated,
+        sample = residue.sample.as_deref().unwrap_or("<none>"),
+        "Bucket deletion was blocked by durable local state"
+    );
+}
 
 fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
     set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
@@ -156,6 +175,7 @@ where
 async fn bucket_delete_local_blocker(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+    budget: &mut BucketDeleteDiagnosticBudget,
 ) -> Result<Option<StorageError>> {
     let local_disks = runtime_sources::local_disks_in(ctx).await;
     let mut residue = BucketMetadataLessResidue::default();
@@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker(
         let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
             continue;
         };
-        let scan = scan_metadata_less_residue(&bucket_path?).await?;
+        let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
         if scan.xlmeta_found {
+            record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
             return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
         }
         residue.files = residue.files.saturating_add(scan.files);
         residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
+        residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
+        residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
         if residue.sample.is_none() {
-            residue.sample = scan.sample;
+            residue.sample = scan.sample.clone();
+        }
+        if scan.diagnostic_truncated {
+            residue.diagnostic_truncated = true;
+            record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
+            return Ok(Some(StorageError::BucketNotEmptyWithDetails {
+                bucket: bucket.to_string(),
+                details: residue.describe(),
+            }));
         }
     }
 
     if residue.has_residue_without_xlmeta() {
+        record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
         return Ok(Some(StorageError::BucketNotEmptyWithDetails {
             bucket: bucket.to_string(),
             details: residue.describe(),
@@ -783,12 +815,14 @@ impl ECStore {
                 }
             }
         };
+        let mut diagnostic_budget = None;
 
         if bucket_exists {
             validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
 
             if !opts.force {
-                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? {
+                let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new);
+                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? {
                     return Err(blocker);
                 }
                 delete_opts.force_if_empty = true;
@@ -827,7 +861,12 @@ impl ECStore {
         {
             if delete_opts.force_if_empty
                 && matches!(&err, StorageError::BucketNotEmpty(_))
-                && let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await?
+                && let Some(blocker) = bucket_delete_local_blocker(
+                    &self.ctx,
+                    bucket,
+                    diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new),
+                )
+                .await?
             {
                 return Err(blocker);
             }
@@ -856,15 +895,17 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::{
-        SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
-        bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
-        scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
+        BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind,
+        BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation,
+        bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
+        run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
+        scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
         validate_table_bucket_delete_allowed,
     };
     use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::bucket::metadata_sys;
     use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
-    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
     use crate::error::StorageError;
     use crate::object_api::{ObjectOptions, PutObjReader};
     use crate::runtime::instance::InstanceContext;
@@ -879,6 +920,7 @@ mod tests {
         layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
     };
     use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
+    use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
     use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
     use serial_test::serial;
     use std::path::{Path, PathBuf};
@@ -892,6 +934,88 @@ mod tests {
 
     static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
 
+    #[tokio::test(start_paused = true)]
+    async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
+        tokio::time::advance(Duration::from_secs(10)).await;
+
+        let first_polled = Arc::new(AtomicBool::new(false));
+        let first_polled_for_io = first_polled.clone();
+        let first = budget
+            .run_io(async move {
+                first_polled_for_io.store(true, Ordering::SeqCst);
+                Ok::<_, std::io::Error>(7_u8)
+            })
+            .await
+            .expect("the first diagnostic IO should succeed");
+        assert_eq!(first, Some(7));
+        assert!(first_polled.load(Ordering::SeqCst));
+
+        tokio::time::advance(Duration::from_millis(101)).await;
+        let expired_polled = Arc::new(AtomicBool::new(false));
+        let expired_polled_for_io = expired_polled.clone();
+        let expired = budget
+            .run_io(async move {
+                expired_polled_for_io.store(true, Ordering::SeqCst);
+                Ok::<_, std::io::Error>(9_u8)
+            })
+            .await
+            .expect("an expired diagnostic budget should not become an IO error");
+        assert_eq!(expired, None);
+        assert!(
+            !expired_polled.load(Ordering::SeqCst),
+            "the deadline must remain latched after the first scan IO"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
+        let io_polled = Arc::new(AtomicBool::new(false));
+        let io_polled_for_future = io_polled.clone();
+
+        let result = budget
+            .run_io(std::future::poll_fn(move |_cx| {
+                io_polled_for_future.store(true, Ordering::SeqCst);
+                std::task::Poll::<std::io::Result<()>>::Pending
+            }))
+            .await
+            .expect("a diagnostic timeout should fail closed without an IO error");
+
+        assert_eq!(result, None);
+        assert!(
+            io_polled.load(Ordering::SeqCst),
+            "the first diagnostic IO must be polled before its timeout"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
+        let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
+        let orphan_root = root.path().join("orphan-root");
+        let xlmeta_root = root.path().join("xlmeta-root");
+        std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
+        std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
+        std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
+        std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
+
+        let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
+        let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
+        tokio::time::advance(Duration::from_secs(60)).await;
+
+        let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
+            .await
+            .expect("delayed orphan scan should complete");
+        assert!(orphan.has_residue_without_xlmeta());
+        assert!(!orphan.diagnostic_truncated);
+
+        let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
+            .await
+            .expect("delayed xl.meta scan should complete");
+        assert!(xlmeta.xlmeta_found);
+        assert!(!xlmeta.diagnostic_truncated);
+    }
+
     #[tokio::test(start_paused = true)]
     async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
         let ttl = Duration::from_millis(20);
@@ -1313,6 +1437,175 @@ mod tests {
         assert!(sample.ends_with("/part.1"));
     }
 
+    #[tokio::test]
+    async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
+        let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
+        let first_root = root.path().join("disk-a");
+        let second_root = root.path().join("disk-b");
+        tokio::fs::create_dir_all(&first_root)
+            .await
+            .expect("first diagnostic root should be created");
+        tokio::fs::create_dir_all(&second_root)
+            .await
+            .expect("second diagnostic root should be created");
+        for index in 0..3 {
+            std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
+            std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
+        }
+
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
+        let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
+            .await
+            .expect("first root should fit the shared budget");
+        assert!(!first.diagnostic_truncated);
+        let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
+            .await
+            .expect("second root should stop at the remaining shared budget");
+        assert!(second.diagnostic_truncated);
+        assert!(first.entries_scanned + second.entries_scanned <= 4);
+    }
+
+    #[tokio::test]
+    async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
+        let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
+        std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
+
+        let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
+            .await
+            .expect("an expired diagnostic budget should fail closed without an IO error");
+
+        assert!(scan.diagnostic_truncated);
+        assert_eq!(scan.entries_scanned, 0);
+        assert_eq!(scan.diagnostic_bytes_read, 0);
+    }
+
+    #[tokio::test]
+    async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
+        let root = tempfile::tempdir().expect("temporary bucket root should be created");
+        let bucket_path = root.path().join("bucket");
+        tokio::fs::create_dir_all(&bucket_path)
+            .await
+            .expect("budget fixture directory should be created");
+        for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
+            std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
+        }
+
+        let residue = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("budgeted residue scan should fail closed without an IO error");
+        assert!(residue.diagnostic_truncated);
+        assert!(residue.has_residue_without_xlmeta());
+        assert!(!residue.xlmeta_found);
+        assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
+        assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
+        assert_eq!(residue.diagnostic_bytes_read, 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
+        let root = tempfile::tempdir().expect("temporary bucket root should be created");
+        let bucket_path = root.path().join("bucket");
+        let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
+            .await
+            .expect("visible object directory should be created");
+        let mut visible = FileMeta::new();
+        visible
+            .add_version(FileInfo {
+                version_id: Some(Uuid::new_v4()),
+                mod_time: Some(OffsetDateTime::now_utc()),
+                ..Default::default()
+            })
+            .expect("visible version should encode");
+        tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
+            .await
+            .expect("visible xl.meta should be written");
+
+        let visible_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("visible xl.meta scan should succeed");
+        assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
+
+        tokio::fs::remove_dir_all(bucket_path.join("visible"))
+            .await
+            .expect("visible fixture should be removed");
+        let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
+            .await
+            .expect("free-version object directory should be created");
+        let source_version_id = Uuid::new_v4();
+        let mut free = FileMeta::new();
+        free.add_version(FileInfo {
+            version_id: Some(source_version_id),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_version_id: Some(Uuid::new_v4()),
+            transition_tier: "WARM".to_string(),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        })
+        .expect("transitioned source should encode");
+        let mut delete = FileInfo {
+            version_id: Some(source_version_id),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
+        free.delete_version(&delete)
+            .expect("transitioned source delete should create a free-version");
+        tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
+            .await
+            .expect("free-version xl.meta should be written");
+
+        let free_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("free-version xl.meta scan should succeed");
+        assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
+
+        tokio::fs::remove_dir_all(bucket_path.join("free"))
+            .await
+            .expect("free-version fixture should be removed");
+
+        let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
+            .await
+            .expect("exact-limit object directory should be created");
+        let exact_limit = tokio::fs::File::create(&exact_limit_path)
+            .await
+            .expect("exact-limit xl.meta should be created");
+        exact_limit
+            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
+            .await
+            .expect("exact-limit xl.meta should be extended without allocating its contents");
+        let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("exact-limit xl.meta scan should remain fail closed");
+        assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
+        assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
+        tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
+            .await
+            .expect("exact-limit fixture should be removed");
+
+        let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
+            .await
+            .expect("oversized object directory should be created");
+        let oversized = tokio::fs::File::create(&oversized_path)
+            .await
+            .expect("oversized xl.meta should be created");
+        oversized
+            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
+            .await
+            .expect("oversized xl.meta should be extended without allocating its contents");
+        let oversized_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("oversized xl.meta scan should remain fail closed");
+        assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
+        assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
+        assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
+    }
+
     #[tokio::test]
     #[serial]
     async fn scanner_bucket_listing_unions_every_erasure_set() {

crates/ecstore/src/store/heal.rs

@@ -36,6 +36,10 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
+    bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
+}
+
 #[derive(Debug, Clone, Copy)]
 enum HealFormatPoolSkip {
     Completed,
@@ -492,7 +496,7 @@ impl ECStore {
         #[cfg(test)]
         let store_id = self.id;
 
-        let mut futures = Vec::with_capacity(pools.len());
+        let mut heal_pools = Vec::with_capacity(pools.len());
         for pool in pools.iter() {
             let suspended_complete = {
                 let pool_meta = self.pool_meta.read().await;
@@ -520,19 +524,58 @@ impl ECStore {
                 }
                 continue;
             }
-            let pool_idx = pool.pool_idx;
-            let pool = Arc::clone(pool);
-            let pool_object = object.clone();
-            let opts = *opts;
-            futures.push(
-                self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move {
-                    #[cfg(test)]
-                    crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
-                    pool.heal_object(bucket, &pool_object, version_id, &opts).await
-                }),
-            );
+            heal_pools.push(Arc::clone(pool));
         }
-        let results = join_all(futures).await;
+        let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
+            let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
+            match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
+                Ok((pool_meta_guard, admissions)) => {
+                    let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
+                    let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
+                        let pool = Arc::clone(pool);
+                        let pool_object = object.clone();
+                        let fixed_set = fixed_set.clone();
+                        let mut opts = *opts;
+                        async move {
+                            admission?;
+                            let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
+                            let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
+                            opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
+                            #[cfg(test)]
+                            if !opts.no_lock {
+                                crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
+                            }
+                            #[cfg(test)]
+                            crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                            pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                        }
+                    });
+                    let results = join_all(futures).await;
+                    drop(pool_meta_guard);
+                    results
+                }
+                Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
+            }
+        } else {
+            let mut futures = Vec::with_capacity(heal_pools.len());
+            for pool in heal_pools {
+                let pool_idx = pool.pool_idx;
+                let pool_object = object.clone();
+                let opts = *opts;
+                futures.push(self.run_external_decommission_capacity_heal(
+                    pool_idx,
+                    bucket,
+                    &object,
+                    opts,
+                    move |opts| async move {
+                        #[cfg(test)]
+                        crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                        pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                    },
+                ));
+            }
+            join_all(futures).await
+        };
 
         let mut errs = Vec::with_capacity(self.pools.len());
         let mut ress = Vec::with_capacity(self.pools.len());
@@ -631,7 +674,7 @@ mod tests {
     };
     use crate::core::sets::HealFormatAfterSaveBarrier;
     use crate::disk::error::Result as DiskResult;
-    use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
+    use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::services::rebalance::{
@@ -784,6 +827,30 @@ mod tests {
         }
     }
 
+    async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
+        let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
+        let missing_disk = target_set.disks.read().await[0]
+            .clone()
+            .expect("pool metadata fixture disk should be online");
+        missing_disk
+            .delete(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("one pool metadata shard should be removable");
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
+            "pool metadata fixture must start with one missing shard"
+        );
+        missing_disk
+    }
+
     #[tokio::test]
     async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
         let store = minimal_heal_store().await;
@@ -1243,6 +1310,121 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_read_repair_reuses_its_write_fence() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let missing_disk = remove_pool_meta_shard(&store, 0).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    read_repair: true,
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata read repair must not wait on its own capacity fence")
+        .expect("pool metadata read repair should complete");
+
+        assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "pool metadata read repair should restore the missing shard"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_preserves_typed_lock_timeout() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let lock = store.pools[0]
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("pool metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("pool metadata lock should be acquired");
+
+        let (_, err) = temp_env::async_with_vars(
+            [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata lock timeout should be mapped into the heal result");
+
+        assert!(
+            matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
+            "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
+        );
+        drop(guard);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let object = "pool.bin.backup";
+        save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
+            .await
+            .expect("neighbor metadata fixture should be written");
+
+        let target_set = store.pools[0].get_disks_by_key(object);
+        let lock = target_set
+            .new_ns_lock(RUSTFS_META_BUCKET, object)
+            .await
+            .expect("neighbor metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("neighbor metadata lock should be acquired");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    object,
+                    "",
+                    &HealOpts {
+                        pool: Some(0),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        barrier.wait_until_external_heal_target_lock_attempted().await;
+        tokio::task::yield_now().await;
+        assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
+        drop(guard);
+
+        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
+            .await
+            .expect("neighbor metadata heal should finish after the object lock is released")
+            .expect("neighbor metadata heal task should not panic")
+            .expect("neighbor metadata heal should complete");
+        assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
+        assert_eq!(result.object, object);
+        drop(barrier);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
@@ -1322,6 +1504,72 @@ mod tests {
             missing_disk.read_xl(&bucket, object, false).await.is_err(),
             "capacity-blocked targeted heal must not rewrite the missing shard"
         );
+
+        let (_, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata admission should not recurse on its namespace lock")
+        .expect("pool metadata capacity rejection should be mapped");
+        assert!(
+            matches!(err, Some(Error::SlowDown)),
+            "pool metadata heal must preserve target reservation admission: {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_per_target_capacity_admission() {
+        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        set_decommission_capacity_info_overrides_for_test(
+            store.id,
+            vec![vec![
+                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
+                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
+            ]],
+        );
+        store
+            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
+            .await
+            .expect("pool metadata reservation should activate");
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let reservation = pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .expect("pool metadata reservation should be durable");
+            assert_eq!(reservation.targets.len(), 1);
+            assert_eq!(reservation.targets[0].pool_index, 1);
+        }
+
+        let missing_disk = remove_pool_meta_shard(&store, 2).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
+        )
+        .await
+        .expect("unscoped pool metadata heal should complete")
+        .expect("unscoped pool metadata heal should return a mapped result");
+
+        assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "unscoped metadata heal should repair the admitted target while another target is reserved"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/store/init_format.rs

@@ -14,6 +14,7 @@
 
 use crate::cluster::rpc::client::is_network_like_disk_error;
 use crate::config::storageclass;
+use crate::core::pools::PoolMetaBootstrapAuthority;
 use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs};
 use crate::disk::{self, DiskAPI};
 use crate::error::{Error, Result};
@@ -84,7 +85,7 @@ pub async fn connect_load_init_formats(
 
 pub(crate) struct LoadedFormat {
     pub(crate) format: FormatV3,
-    pub(crate) fresh_bootstrap_proven: bool,
+    pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
 }
 
 pub(crate) async fn connect_load_init_formats_with_instance_ctx(
@@ -133,7 +134,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
                 retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
                 return Ok(LoadedFormat {
                     format: *format,
-                    fresh_bootstrap_proven: false,
+                    pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
                 });
             }
             Ok(LegacyFormatOutcome::Incompatible) => {
@@ -153,7 +154,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
             let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
             return Ok(LoadedFormat {
                 format: fm,
-                fresh_bootstrap_proven: true,
+                pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
             });
         }
     }
@@ -182,10 +183,29 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
 
     Ok(LoadedFormat {
         format: fm,
-        fresh_bootstrap_proven: false,
+        pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
     })
 }
 
+async fn verified_legacy_adoption_source(
+    disks: &[Option<DiskStore>],
+    rustfs_formats: &[Option<FormatV3>],
+    set_count: usize,
+    set_drive_count: usize,
+) -> Result<PoolMetaBootstrapAuthority> {
+    match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
+        Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
+        Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
+        Err(err) => {
+            debug!(
+                error = %err,
+                "legacy adoption proof skipped because legacy format verification failed"
+            );
+            Ok(PoolMetaBootstrapAuthority::None)
+        }
+    }
+}
+
 async fn retain_format_quorum_members(
     instance_ctx: &Arc<InstanceContext>,
     disks: &mut [Option<DiskStore>],
@@ -1311,10 +1331,7 @@ mod tests {
         let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 3, None)
             .await
             .expect("fresh disks should receive a storage format");
-        assert!(
-            loaded.fresh_bootstrap_proven,
-            "every configured disk explicitly reporting unformatted should establish fresh topology proof"
-        );
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
         let format = loaded.format;
 
         let (formats, errors) = load_format_erasure_all(&disks, false).await;
@@ -1358,12 +1375,11 @@ mod tests {
 
         let mut expected = legacy;
         expected.erasure.this = Uuid::nil();
-        assert_eq!(
-            connect_load_init_formats(true, &mut disks, 1, 3, None)
-                .await
-                .expect("compatible legacy format should migrate"),
-            expected
-        );
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 3, None)
+            .await
+            .expect("compatible legacy format should migrate");
+        assert_eq!(loaded.format, expected);
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
         let (formats, errors) = load_format_erasure_all(&disks, false).await;
         assert!(
             errors.iter().all(Option::is_none),
@@ -1378,6 +1394,51 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn compatible_single_drive_legacy_format_marks_adoption_proof() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let legacy = FormatV3::new(1, 1);
+        write_legacy_majority(&disks, &legacy).await;
+
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("single-drive MinIO format should migrate");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
+    }
+
+    #[tokio::test]
+    async fn existing_migrated_format_keeps_legacy_adoption_proof() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let legacy = FormatV3::new(1, 1);
+        write_legacy_majority(&disks, &legacy).await;
+
+        connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("first run should migrate the MinIO format");
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("retry after a partial adoption should reload the migrated RustFS format");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
+    }
+
+    #[tokio::test]
+    async fn existing_rustfs_format_without_legacy_source_is_not_legacy_adoption() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let mut format = FormatV3::new(1, 1);
+        format.erasure.this = format.erasure.sets[0][0];
+        save_format_file(&disks[0], &Some(format))
+            .await
+            .expect("existing RustFS format should be written");
+
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("existing RustFS format should load");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+
     #[tokio::test]
     async fn compatible_legacy_format_migrates_when_the_file_is_missing() {
         let (_temp_dir, mut disks) = local_disks(3).await;

crates/ecstore/src/store/list_objects.rs

@@ -20,7 +20,9 @@ fn to_filemeta_err(err: Error) -> rustfs_filemeta::Error {
     err.narrow_to_filemeta().unwrap_or_else(rustfs_filemeta::Error::other)
 }
 
-use crate::bucket::metadata_sys::{get_versioning_config, has_authoritative_never_versioned_state};
+use crate::bucket::metadata_sys::{
+    get_versioning_config, has_authoritative_never_versioned_state, has_authoritative_never_versioned_state_in,
+};
 use crate::bucket::utils::check_list_objs_args;
 use crate::bucket::versioning::VersioningApi;
 use crate::cache_value::metacache_set::{FallbackClaimTracker, ListPathRawOptions, list_path_raw_with_claim_tracker};
@@ -314,6 +316,25 @@ async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
         .unwrap_or(false)
 }
 
+fn should_purge_empty_directory_listing(
+    prefix: &str,
+    marker: Option<&str>,
+    delimiter: Option<&str>,
+    max_keys: i32,
+    incl_deleted: bool,
+    result: &ListObjectsInfo,
+) -> bool {
+    !prefix.is_empty()
+        && prefix.ends_with(SLASH_SEPARATOR)
+        && marker.is_none()
+        && delimiter.is_none_or(str::is_empty)
+        && max_keys == 1
+        && !incl_deleted
+        && !result.is_truncated
+        && result.objects.is_empty()
+        && result.prefixes.is_empty()
+}
+
 const MARKER_TAG_VERSION: &str = "v2";
 const LEGACY_MARKER_TAG_VERSIONS: &[&str] = &["v1", MARKER_TAG_VERSION];
 const LIST_CACHE_MARKER_PREFIX: &str = "[rustfs_cache:";
@@ -2501,6 +2522,7 @@ fn list_metadata_resolution_params(
     listing_quorum: usize,
     latest_object_quorum: usize,
     versioned: bool,
+    write_quorum_slack: usize,
 ) -> MetadataResolutionParams {
     let quorum = if versioned {
         listing_quorum
@@ -2511,6 +2533,7 @@ fn list_metadata_resolution_params(
         dir_quorum: quorum,
         obj_quorum: quorum,
         bucket,
+        write_quorum_slack,
         ..Default::default()
     };
 
@@ -2811,7 +2834,10 @@ fn cached_entry_needs_supplement(
 
     let mut selected_object_versions = 0;
     for version in cached.versions.iter() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions) {
             return true;
         }
@@ -2836,8 +2862,15 @@ fn listing_entries_supplement_target(
         return None;
     }
 
-    for entry in entries.0.iter().flatten() {
-        if entry.is_dir() {
+    for (idx, entry) in entries.0.iter().enumerate() {
+        let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
+            continue;
+        };
+        if entries.0[..idx]
+            .iter()
+            .flatten()
+            .any(|previous| previous.name == entry.name && previous.is_object())
+        {
             continue;
         }
 
@@ -2850,6 +2883,15 @@ fn listing_entries_supplement_target(
                     .is_some_and(|candidate| candidate.name == entry.name && candidate.is_object())
             })
             .count();
+        let entries_disagree = entries.0.iter().flatten().any(|candidate| {
+            candidate.name == entry.name && candidate.is_object() && !entry.matches(Some(candidate), resolver.strict).1
+        });
+        // A split primary sample can fall back to an older version even when a
+        // newer version reaches write quorum only after the fallback disks join.
+        if entries_disagree {
+            return Some(entry.name.clone());
+        }
+
         let mut entry = entry.clone();
         if let Ok(cached) = entry.xl_meta()
             && cached_entry_needs_supplement(&cached, reader_disks, resolver, enforce_write_quorum)
@@ -2946,7 +2988,10 @@ fn resolve_agreed_listing_entry(
     let mut needs_supplement = false;
 
     for (idx, version) in cached.versions.iter().enumerate() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if reader_disks < required_quorum {
             needs_supplement |=
                 version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions);
@@ -3000,21 +3045,51 @@ fn latest_listing_object_quorum(
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
-    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum)
+    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum, unreachable_disks)
 }
 
+/// Object quorum a listed "latest" version must reach among the drives the
+/// listing can actually consult.
+///
+/// An object legally committed at write quorum can have up to
+/// `unreachable_disks` of its metadata copies on drives that are offline for
+/// this listing, so the write-quorum requirement is relaxed by that amount.
+/// The result is floored at the erasure read quorum (data drives): below that
+/// the object could not be read back either, and a quorum-deleted object
+/// leaves at most `drive_count - write_quorum < read_quorum` stale copies, so
+/// the floor also keeps deleted objects from resurfacing.
+///
+/// Trade-off: while a drive is offline, a torn overwrite that reached only
+/// `write_quorum - 1` drives becomes indistinguishable from a committed write
+/// whose missing copy sits on the offline drive, so it can be listed as
+/// latest. GET at read quorum serves that same version in that state, so the
+/// listing stays consistent with reads instead of hiding readable objects.
 fn latest_listing_required_object_quorum(
     listing_quorum: usize,
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
     if !enforce_write_quorum {
         return listing_quorum;
     }
 
-    write_quorum_for_drive_count(drive_count, parity_count).max(listing_quorum)
+    let read_quorum = drive_count.saturating_sub(parity_count);
+    write_quorum_for_drive_count(drive_count, parity_count)
+        .saturating_sub(unreachable_disks)
+        .max(read_quorum)
+        .max(listing_quorum)
+}
+
+fn latest_listing_write_quorum_slack(enforce_write_quorum: bool, drive_count: usize, online_disks: usize) -> usize {
+    if !enforce_write_quorum {
+        return 0;
+    }
+
+    drive_count.saturating_sub(online_disks)
 }
 
 fn enforce_latest_listing_write_quorum(strict_latest: bool, ask_disks: &str) -> bool {
@@ -3772,6 +3847,19 @@ impl ECStore {
                 .list_objects_from_opt_in_key_only_provider(&opts, mode, max_keys, incl_deleted)
                 .await?
         {
+            if should_purge_empty_directory_listing(
+                prefix,
+                opts.marker.as_deref(),
+                delimiter.as_deref(),
+                max_keys,
+                incl_deleted,
+                &result,
+            ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+                .await
+                .unwrap_or(false)
+            {
+                self.purge_orphan_dir_object(bucket, prefix).await;
+            }
             return Ok(result);
         }
 
@@ -3802,6 +3890,7 @@ impl ECStore {
         };
 
         let mut list_result = self
+            .clone()
             .list_path(&opts)
             .await
             .unwrap_or_else(|err| MetaCacheEntriesSortedResult {
@@ -3820,7 +3909,7 @@ impl ECStore {
         }
 
         if let Some(result) = list_result.entries.as_mut() {
-            result.forward_past(opts.marker);
+            result.forward_past(opts.marker.clone());
         }
 
         // contextCanceled
@@ -3848,12 +3937,26 @@ impl ECStore {
         );
         let _ = next_version_idmarker;
 
-        Ok(ListObjectsInfo {
+        let result = ListObjectsInfo {
             is_truncated,
             next_marker,
             objects,
             prefixes,
-        })
+        };
+        if should_purge_empty_directory_listing(
+            prefix,
+            opts.marker.as_deref(),
+            delimiter.as_deref(),
+            max_keys,
+            incl_deleted,
+            &result,
+        ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+            .await
+            .unwrap_or(false)
+        {
+            self.purge_orphan_dir_object(bucket, prefix).await;
+        }
+        Ok(result)
     }
 
     pub async fn inner_list_object_versions(
@@ -4306,6 +4409,9 @@ impl ECStore {
         for eset in self.pools.iter() {
             for set in eset.disk_set.iter() {
                 let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+                // Captured before any quorum-based filtering: only genuinely
+                // unreachable drives may relax the write-quorum requirement.
+                let online_disks = disks.len();
                 let opts = opts.clone();
 
                 let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
@@ -4331,11 +4437,14 @@ impl ECStore {
                     let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                     let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                     let write_quorum_parity = set.default_parity_count;
+                    let write_quorum_slack =
+                        latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                     let required_obj_quorum = latest_listing_required_object_quorum(
                         listing_quorum,
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                     let fallback_disks = {
@@ -4357,11 +4466,17 @@ impl ECStore {
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
 
-                    let resolver =
-                        list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                    let resolver = list_metadata_resolution_params(
+                        bucket.to_owned(),
+                        listing_quorum,
+                        obj_quorum,
+                        !opts.latest_only,
+                        write_quorum_slack,
+                    );
                     let agreed_resolver = resolver.clone();
                     let partial_resolver = resolver.clone();
                     let reader_disks = disks.len();
@@ -5548,6 +5663,9 @@ impl Sets {
 
         for set in &self.disk_set {
             let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+            // Captured before any quorum-based filtering: only genuinely
+            // unreachable drives may relax the write-quorum requirement.
+            let online_disks = disks.len();
             let opts = opts.clone();
             let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
             inputs.push(list_out_rx);
@@ -5573,11 +5691,14 @@ impl Sets {
                 let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                 let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                 let write_quorum_parity = set.default_parity_count;
+                let write_quorum_slack =
+                    latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                 let required_obj_quorum = latest_listing_required_object_quorum(
                     listing_quorum,
                     set.set_drive_count,
                     write_quorum_parity,
                     enforce_write_quorum,
+                    write_quorum_slack,
                 );
                 ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                 let fallback_disks = if let Some(asked_disks) = positive_ask_disks(ask_disks)
@@ -5592,10 +5713,21 @@ impl Sets {
                 let fallback_disks = Arc::new(fallback_disks);
                 let claim_tracker = FallbackClaimTracker::default();
 
-                let obj_quorum =
-                    latest_listing_object_quorum(listing_quorum, set.set_drive_count, write_quorum_parity, enforce_write_quorum);
+                let obj_quorum = latest_listing_object_quorum(
+                    listing_quorum,
+                    set.set_drive_count,
+                    write_quorum_parity,
+                    enforce_write_quorum,
+                    write_quorum_slack,
+                );
                 let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
-                let resolver = list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                let resolver = list_metadata_resolution_params(
+                    bucket.to_owned(),
+                    listing_quorum,
+                    obj_quorum,
+                    !opts.latest_only,
+                    write_quorum_slack,
+                );
                 let agreed_resolver = resolver.clone();
                 let partial_resolver = resolver.clone();
                 let reader_disks = disks.len();
@@ -6517,6 +6649,9 @@ impl SetDisks {
         let list_path_started = std::time::Instant::now();
 
         let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await;
+        // Captured before any quorum-based filtering: only genuinely
+        // unreachable drives may relax the write-quorum requirement.
+        let online_disks = disks.len();
 
         let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
         if ask_disks == -1 {
@@ -6540,11 +6675,13 @@ impl SetDisks {
 
         let enforce_write_quorum = enforce_latest_listing_write_quorum(!opts.versioned, &opts.ask_disks);
         let write_quorum_parity = self.default_parity_count;
+        let write_quorum_slack = latest_listing_write_quorum_slack(enforce_write_quorum, self.set_drive_count, online_disks);
         let required_obj_quorum = latest_listing_required_object_quorum(
             listing_quorum,
             self.set_drive_count,
             write_quorum_parity,
             enforce_write_quorum,
+            write_quorum_slack,
         );
         ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
         let mut fallback_disks = Vec::new();
@@ -6562,10 +6699,21 @@ impl SetDisks {
 
         let bucket = opts.bucket.clone();
         let base_dir = opts.base_dir.clone();
-        let latest_object_quorum =
-            latest_listing_object_quorum(listing_quorum, self.set_drive_count, write_quorum_parity, enforce_write_quorum);
+        let latest_object_quorum = latest_listing_object_quorum(
+            listing_quorum,
+            self.set_drive_count,
+            write_quorum_parity,
+            enforce_write_quorum,
+            write_quorum_slack,
+        );
         let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, latest_object_quorum, enforce_write_quorum);
-        let resolver = list_metadata_resolution_params(bucket.clone(), listing_quorum, latest_object_quorum, opts.versioned);
+        let resolver = list_metadata_resolution_params(
+            bucket.clone(),
+            listing_quorum,
+            latest_object_quorum,
+            opts.versioned,
+            write_quorum_slack,
+        );
         let agreed_resolver = resolver.clone();
         let partial_resolver = resolver.clone();
         let reader_disks = disks.len();
@@ -6577,6 +6725,7 @@ impl SetDisks {
             asked_disks = ask_disks,
             listing_quorum = listing_quorum,
             latest_object_quorum = latest_object_quorum,
+            write_quorum_slack = write_quorum_slack,
             raw_min_disks = raw_min_disks,
             fallback_disks = fallback_disks.len(),
             limit = opts.limit,
@@ -6818,34 +6967,36 @@ mod test {
         LIST_CURSOR_GENERATION_LIVE, LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY,
         LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, ListIndexFallbackReason, ListIndexLifecycle, ListIndexLifecycleState,
         ListIndexSourceDecision, ListMetadataAuthority, ListMetadataIndexHealth, ListObjectsIndexProviderKind,
-        ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution,
-        ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
+        ListObjectsIndexProviderState, ListObjectsInfo, ListPathOptions, ListPathRawOptions, ListSourceMode,
+        ListingEntryResolution, ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
         NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
         PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
         PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex,
         PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker,
-        current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum,
-        expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects,
-        latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key,
-        list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
+        cached_entry_needs_supplement, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
+        enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
+        latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
+        latest_listing_required_object_quorum, latest_listing_write_quorum_slack, list_marker_key, list_merged_entry_channel,
+        list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
         list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
         list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
         list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
         list_objects_metadata_fast_guardrails_from_env, list_objects_paginate, list_objects_quorum_from_env,
-        load_namespace_mutation_journal_state, load_persistent_key_only_index, max_keys_plus_one, merge_entry_channels,
-        namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env,
-        namespace_mutation_journal_chaos_enabled_from_env, namespace_mutation_journal_chaos_sequence_from_env,
-        namespace_mutation_journal_chaos_status_from_env, normalize_list_quorum, observe_list_objects_mutations_with_store,
-        parse_namespace_mutation_journal_state, parse_persistent_key_only_index, parse_persistent_list_metadata_object,
-        parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
+        listing_entries_supplement_target, load_namespace_mutation_journal_state, load_persistent_key_only_index,
+        max_keys_plus_one, merge_entry_channels, namespace_mutation_journal_chaos_bucket_from_env,
+        namespace_mutation_journal_chaos_config_from_env, namespace_mutation_journal_chaos_enabled_from_env,
+        namespace_mutation_journal_chaos_sequence_from_env, namespace_mutation_journal_chaos_status_from_env,
+        normalize_list_quorum, observe_list_objects_mutations_with_store, parse_namespace_mutation_journal_state,
+        parse_persistent_key_only_index, parse_persistent_list_metadata_object, parse_version_marker,
+        persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
         persistent_key_only_index_health, persistent_key_only_index_matches_provider,
         reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
-        scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
-        send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
-        write_persistent_key_only_index_with_metadata,
+        resolve_listing_entries_with_supplement, scanner_namespace_mutation_generation, select_list_index_provider_source_mode,
+        select_list_index_source_mode, send_or_cancel, should_purge_empty_directory_listing, version_marker_for_entries,
+        walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index_with_metadata,
     };
     use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
-    use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
+    use crate::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, error::DiskError, new_disk};
     use crate::error::StorageError;
     use crate::object_api::ObjectInfo;
     use rustfs_filemeta::{
@@ -7168,6 +7319,32 @@ mod test {
         }
     }
 
+    fn test_object_with_delete_marker_meta_entry(
+        name: &str,
+        object_mod_time: time::OffsetDateTime,
+        delete_mod_time: time::OffsetDateTime,
+    ) -> MetaCacheEntry {
+        let mut object = test_object_meta_entry_with_erasure_versions(name, &[(object_mod_time, "object-etag", 4, 2)]);
+        let delete = test_delete_marker_meta_entry(name, delete_mod_time);
+        let mut metadata = object.cached.take().expect("test object metadata should be cached");
+        let delete_version = delete
+            .cached
+            .expect("test delete marker metadata should be cached")
+            .versions
+            .into_iter()
+            .next()
+            .expect("test delete marker should contain one version");
+        metadata.versions.insert(0, delete_version);
+        let encoded = metadata.marshal_msg().expect("test metadata should marshal");
+
+        MetaCacheEntry {
+            name: name.to_owned(),
+            metadata: encoded,
+            cached: Some(metadata),
+            reusable: false,
+        }
+    }
+
     fn test_dir_meta_entry(name: &str) -> MetaCacheEntry {
         MetaCacheEntry {
             name: name.to_owned(),
@@ -8665,6 +8842,80 @@ mod test {
         assert_eq!(scanner_namespace_mutation_generation(), 2);
     }
 
+    #[test]
+    fn empty_directory_listing_purge_requires_complete_exact_recursive_request() {
+        let empty = ListObjectsInfo::default();
+        assert!(should_purge_empty_directory_listing("ghost/", None, None, 1, false, &empty));
+        assert!(should_purge_empty_directory_listing("ghost/", None, Some(""), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost", None, None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", Some("marker"), None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, Some("/"), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 0, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 2, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, true, &empty));
+
+        let mut live = ListObjectsInfo::default();
+        live.objects.push(ObjectInfo::default());
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &live));
+
+        let truncated = ListObjectsInfo {
+            is_truncated: true,
+            ..Default::default()
+        };
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &truncated));
+    }
+
+    #[tokio::test]
+    async fn empty_recursive_listing_purges_committed_delete_residue() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "listing-purge-bucket";
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created with authoritative metadata");
+        let data_dir = uuid::Uuid::new_v4();
+        let transaction = uuid::Uuid::new_v4();
+        for dir in &dirs {
+            let residue = dir
+                .path()
+                .join(bucket)
+                .join("ghost")
+                .join("nested")
+                .join("object")
+                .join(data_dir.to_string());
+            tokio::fs::create_dir_all(&residue)
+                .await
+                .expect("committed delete residue should be created");
+            tokio::fs::write(residue.join("part.1"), b"stale")
+                .await
+                .expect("stale part should be written");
+            tokio::fs::write(
+                residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+                [],
+            )
+            .await
+            .expect("committed delete marker should be written");
+        }
+
+        let result = store
+            .list_objects_generic(bucket, "ghost/", None, None, 1, false)
+            .await
+            .expect("empty recursive listing should succeed");
+
+        assert!(result.objects.is_empty());
+        assert!(result.prefixes.is_empty());
+        for dir in &dirs {
+            assert!(
+                !dir.path().join(bucket).join("ghost").exists(),
+                "the empty listing should reclaim its committed delete residue"
+            );
+        }
+    }
+
     #[test]
     fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
         let provider = ListObjectsIndexProviderState::walker_key_only();
@@ -8947,7 +9198,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_limits_plain_listing_to_latest_version() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8957,7 +9208,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_keeps_all_versions_for_version_listing() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8967,22 +9218,22 @@ mod test {
 
     #[test]
     fn latest_listing_object_quorum_uses_write_quorum_for_strict_latest_listing() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 5);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
     }
 
     #[test]
     fn latest_listing_object_quorum_calculates_low_parity_write_quorum() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 7);
         assert_eq!(ask_disks, 7);
-        assert_eq!(latest_listing_object_quorum(2, 8, 1, true), 7);
+        assert_eq!(latest_listing_object_quorum(2, 8, 1, true, 0), 7);
     }
 
     #[test]
@@ -8991,21 +9242,196 @@ mod test {
         assert!(!enforce_latest_listing_write_quorum(true, "disk"));
         assert!(enforce_latest_listing_write_quorum(true, "optimal"));
         assert!(!enforce_latest_listing_write_quorum(false, "optimal"));
-        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false), 2);
-        assert_eq!(latest_listing_object_quorum(2, 4, 2, false), 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false, 0), 2);
+        assert_eq!(latest_listing_object_quorum(2, 4, 2, false, 0), 2);
         assert_eq!(expand_ask_disks_for_object_quorum(2, 4, 2), 2);
     }
 
+    #[test]
+    fn latest_listing_object_quorum_relaxes_write_quorum_by_unreachable_drives() {
+        // 4-drive set, EC 2+2: with one drive offline an object committed at
+        // write quorum 3 can only ever show 2 metadata copies to the listing.
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        assert_eq!(slack, 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, slack), 2);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, 0), 3);
+
+        // The relaxed quorum never drops below the erasure read quorum, so a
+        // quorum-deleted object (at most one stale copy) stays hidden even
+        // with half the set unreachable.
+        let slack = latest_listing_write_quorum_slack(true, 4, 2);
+        assert_eq!(slack, 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, true, slack), 2);
+
+        assert_eq!(latest_listing_write_quorum_slack(false, 4, 2), 0);
+    }
+
+    #[test]
+    fn latest_listing_resolves_degraded_object_when_a_set_drive_is_unreachable() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]);
+
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        let obj_quorum = latest_listing_object_quorum(2, 4, 2, true, slack);
+        assert_eq!(obj_quorum, 2);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, obj_quorum, false, slack);
+        let resolved = resolve_listing_entries(entries(), resolver, true)
+            .expect("object committed at write quorum should stay listable with one holder drive offline");
+        assert_eq!(resolved.name, "object");
+
+        // Without the unreachable-drive slack the same sample is dropped even
+        // though the object still satisfies read quorum for GET.
+        let strict_quorum = latest_listing_object_quorum(2, 4, 2, true, 0);
+        let strict_resolver = list_metadata_resolution_params("bucket".to_string(), 2, strict_quorum, false, 0);
+        assert!(resolve_listing_entries(entries(), strict_resolver, true).is_none());
+    }
+
+    #[test]
+    fn agreed_listing_entry_relaxes_write_quorum_by_unreachable_drives() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry.clone(), 2, resolver.clone(), true),
+            ListingEntryResolution::Resolved(_)
+        ));
+
+        resolver.write_quorum_slack = 0;
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry, 2, resolver, true),
+            ListingEntryResolution::NeedsSupplement(_, _)
+        ));
+    }
+
+    #[test]
+    fn cached_entry_supplement_check_honors_write_quorum_slack() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let cached = entry.xl_meta().expect("test entry should decode");
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(!cached_entry_needs_supplement(&cached, 2, &resolver, true));
+
+        resolver.write_quorum_slack = 0;
+        assert!(cached_entry_needs_supplement(&cached, 2, &resolver, true));
+    }
+
     #[test]
     fn latest_listing_object_quorum_requires_write_quorum_when_degraded_cannot_satisfy_it() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 4, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 4);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
+    }
+
+    #[tokio::test]
+    async fn latest_listing_supplements_a_split_delete_marker_sample() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+            Some(deleted.clone()),
+        ]);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, false), None);
+
+        let mut primary = resolve_listing_entries(MetaCacheEntries(entries.0.clone()), resolver.clone(), true)
+            .expect("the partial sample should fall back to the stale object version");
+        assert!(!primary.is_latest_delete_marker());
+
+        let mut fallback_disks = Vec::new();
+        let mut fallback_tempdirs = Vec::new();
+        for _ in 0..2 {
+            let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
+            let endpoint =
+                Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")).expect("valid endpoint");
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("fallback disk should be created");
+            disk.make_volume("bucket").await.expect("fallback bucket should be created");
+            disk.write_all(
+                "bucket",
+                &format!("object/{STORAGE_FORMAT_FILE}"),
+                bytes::Bytes::copy_from_slice(&deleted.metadata),
+            )
+            .await
+            .expect("fallback metadata should be written");
+            fallback_disks.push(disk);
+            fallback_tempdirs.push(tempdir);
+        }
+        let supplement = ListingSupplement::new(
+            ListingSupplementOptions {
+                bucket: "bucket".to_string(),
+                path: String::new(),
+                recursive: true,
+                incl_deleted: false,
+                skip_hidden_prefix_check: false,
+                filter_prefix: None,
+                forward_to: None,
+                per_disk_limit: 100,
+                skip_total_timeout: true,
+                walkdir_timeout: None,
+                walkdir_stall_timeout: None,
+            },
+            Arc::new(fallback_disks),
+            FallbackClaimTracker::default(),
+        );
+
+        let mut supplemented = resolve_listing_entries_with_supplement(entries, resolver, true, supplement)
+            .await
+            .expect("the supplemented sample should resolve the committed delete marker");
+        assert!(supplemented.is_latest_delete_marker());
+    }
+
+    #[test]
+    fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+        ]);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+
+        let mut resolved = resolve_listing_entries(
+            MetaCacheEntries(vec![
+                Some(stale.clone()),
+                Some(stale.clone()),
+                Some(stale),
+                Some(deleted.clone()),
+                Some(deleted.clone()),
+                Some(deleted),
+            ]),
+            resolver,
+            true,
+        )
+        .expect("the previous object version should remain visible below delete-marker write quorum");
+
+        assert!(!resolved.is_latest_delete_marker());
     }
 
     #[tokio::test]
@@ -9082,7 +9508,7 @@ mod test {
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
         let fallback_old_entry = test_object_meta_entry_with_erasure_versions("object", &[(old_mod_time, "old-etag", 4, 4)]);
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9137,7 +9563,7 @@ mod test {
             "object",
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9181,7 +9607,7 @@ mod test {
         let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
         let entry = test_object_meta_entry_with_erasure_versions("object", &[(new_mod_time, "new-etag", 7, 1)]);
         let fallback_entry = entry.clone();
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 

crates/ecstore/src/store/mod.rs

@@ -65,7 +65,7 @@ use http::HeaderMap;
 use lazy_static::lazy_static;
 use rand::RngExt as _;
 use rustfs_config::server_config::Config;
-use rustfs_filemeta::FileInfo;
+use rustfs_filemeta::{FileInfo, FileMeta};
 use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
 use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
 use rustfs_madmin::heal_commands::HealResultItem;
@@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
 pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
+pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
+
+#[derive(Debug)]
+pub(crate) struct BucketDeleteDiagnosticBudget {
+    deadline: Option<tokio::time::Instant>,
+    max_elapsed: Duration,
+    entries_remaining: usize,
+}
+
+impl BucketDeleteDiagnosticBudget {
+    pub(crate) fn new() -> Self {
+        Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
+    }
+
+    fn with_limits(entries: usize, elapsed: Duration) -> Self {
+        Self {
+            deadline: None,
+            max_elapsed: elapsed,
+            entries_remaining: entries,
+        }
+    }
+
+    fn deadline(&mut self) -> tokio::time::Instant {
+        let max_elapsed = self.max_elapsed;
+        *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
+    }
+
+    fn claim_entry(&mut self) -> bool {
+        if self.entries_remaining == 0 {
+            return false;
+        }
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+        self.entries_remaining -= 1;
+        true
+    }
+
+    async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
+    where
+        F: std::future::Future<Output = std::io::Result<T>>,
+    {
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(None);
+        }
+        match tokio::time::timeout_at(deadline, future).await {
+            Ok(result) => result.map(Some),
+            Err(_) => Ok(None),
+        }
+    }
+}
 
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub(crate) struct BucketMetadataLessResidue {
     pub(crate) xlmeta_found: bool,
+    pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
     pub(crate) files: usize,
     pub(crate) uuid_data_dirs: usize,
+    pub(crate) entries_scanned: usize,
+    pub(crate) diagnostic_bytes_read: u64,
+    pub(crate) diagnostic_truncated: bool,
     pub(crate) sample: Option<String>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BucketDeleteBlockerKind {
+    VisibleVersion,
+    TierFreeVersion,
+    UnknownXlMeta,
+    OrphanDirectory,
+    DiagnosticBudgetExceeded,
+}
+
+impl BucketDeleteBlockerKind {
+    pub(crate) const fn as_str(self) -> &'static str {
+        match self {
+            Self::VisibleVersion => "visible_version",
+            Self::TierFreeVersion => "tier_free_version",
+            Self::UnknownXlMeta => "unknown_xlmeta",
+            Self::OrphanDirectory => "orphan_directory",
+            Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
+        }
+    }
+}
+
 impl BucketMetadataLessResidue {
     pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
-        !self.xlmeta_found && self.files > 0
+        !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
     }
 
     pub(crate) fn describe(&self) -> String {
         let sample = self.sample.as_deref().unwrap_or("<none>");
         format!(
-            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
-            self.files, self.uuid_data_dirs
+            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
+            self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
         )
     }
 }
@@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
     Ok(false)
 }
 
+#[cfg(test)]
 pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
+    let mut budget = BucketDeleteDiagnosticBudget::new();
+    scan_metadata_less_residue_with_budget(path, &mut budget).await
+}
+
+async fn scan_metadata_less_residue_with_budget(
+    path: &std::path::Path,
+    budget: &mut BucketDeleteDiagnosticBudget,
+) -> std::io::Result<BucketMetadataLessResidue> {
     use crate::disk::STORAGE_FORMAT_FILE;
     use tokio::fs;
+    use tokio::io::AsyncReadExt as _;
 
     let mut scan = BucketMetadataLessResidue::default();
     let mut stack = vec![path.to_path_buf()];
 
+    let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
+        scan.diagnostic_truncated = true;
+        scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
+    };
+
     while let Some(current_path) = stack.pop() {
-        let mut entries = match fs::read_dir(&current_path).await {
-            Ok(entries) => entries,
+        let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
+            Ok(Some(entries)) => entries,
+            Ok(None) => {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
             Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
             Err(err) => return Err(err),
         };
 
-        while let Some(entry) = entries.next_entry().await? {
-            let file_type = entry.file_type().await?;
+        loop {
+            let entry = match budget.run_io(entries.next_entry()).await? {
+                Some(Some(entry)) => entry,
+                Some(None) => break,
+                None => {
+                    mark_budget_exhausted(&mut scan);
+                    return Ok(scan);
+                }
+            };
+            if !budget.claim_entry() {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
+            scan.entries_scanned = scan.entries_scanned.saturating_add(1);
+            let Some(file_type) = budget.run_io(entry.file_type()).await? else {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            };
             let file_name = entry.file_name();
             let file_name_str = file_name.to_string_lossy();
 
             if file_name_str == STORAGE_FORMAT_FILE {
                 scan.xlmeta_found = true;
-                continue;
+                if scan.xlmeta_blocker.is_none() {
+                    let entry_path = entry.path();
+                    let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
+                        mark_budget_exhausted(&mut scan);
+                        scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
+                        return Ok(scan);
+                    };
+                    scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
+                        BucketDeleteBlockerKind::UnknownXlMeta
+                    } else {
+                        match budget.run_io(fs::File::open(&entry_path)).await {
+                            Ok(Some(file)) => {
+                                let mut data = Vec::new();
+                                let read = budget
+                                    .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
+                                    .await;
+                                scan.diagnostic_bytes_read = data.len() as u64;
+                                match read {
+                                    Ok(Some(_)) => match FileMeta::load(&data) {
+                                        Ok(meta)
+                                            if !meta.versions.is_empty()
+                                                && meta.versions.iter().all(|version| version.header.free_version()) =>
+                                        {
+                                            BucketDeleteBlockerKind::TierFreeVersion
+                                        }
+                                        Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
+                                        Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                    },
+                                    Ok(None) => {
+                                        mark_budget_exhausted(&mut scan);
+                                        BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                                    }
+                                    Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                }
+                            }
+                            Ok(None) => {
+                                mark_budget_exhausted(&mut scan);
+                                BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                            }
+                            Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                        }
+                    });
+                    let sample = entry_path
+                        .strip_prefix(path)
+                        .unwrap_or(entry_path.as_path())
+                        .to_string_lossy()
+                        .replace(std::path::MAIN_SEPARATOR, "/");
+                    scan.sample = Some(sample);
+                }
+                return Ok(scan);
             }
 
             if file_type.is_dir() {
@@ -222,7 +386,10 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
+pub(crate) use object::{
+    DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
+    SourceCleanupMutationFence, tiered_data_movement_source_matches,
+};
 pub use object::{
     PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
     SnapshotConsistencyError,

crates/ecstore/src/store/multipart.rs

@@ -846,6 +846,7 @@ impl ECStore {
             .await
     }
 
+    #[cfg(all(test, feature = "test-util"))]
     pub(crate) async fn complete_multipart_upload_for_data_movement(
         self: Arc<Self>,
         target: (usize, Option<&ObjectLockDiagGuard>),
@@ -854,6 +855,44 @@ impl ECStore {
         upload_id: &str,
         uploaded_parts: Vec<CompletePart>,
         opts: &ObjectOptions,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
+            .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
+        self: Arc<Self>,
+        target_pool_idx: usize,
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(
+            (target_pool_idx, None),
+            bucket,
+            object,
+            upload_id,
+            uploaded_parts,
+            opts,
+            Some(publication_fence),
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn complete_multipart_upload_for_data_movement_inner(
+        self: Arc<Self>,
+        target: (usize, Option<&ObjectLockDiagGuard>),
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: Option<RemoteTuplePublicationFence>,
     ) -> Result<ObjectInfo> {
         let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
@@ -885,8 +924,36 @@ impl ECStore {
             snapshot.add_lock_fences(&mut opts);
             opts.object_lock_config_snapshot = Some(snapshot);
         }
-        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+        let fixed_read_anchor = publication_fence
+            .as_ref()
+            .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
             .await;
+        // NewMultipart/UploadPart are staging only. Acquire and consume the
+        // non-cloneable publication capability immediately before Complete,
+        // then retain its guards until Complete has drained the commit path.
+        let publication_object = encode_dir_object(object);
+        let publication_guard = match publication_fence {
+            Some(publication_fence) => {
+                let guard = publication_fence
+                    .into_commit_guard(target_pool_idx, bucket, &publication_object)
+                    .await?;
+                guard.add_namespace_lock_fence(&mut opts);
+                opts.no_lock = true;
+                Some(guard)
+            }
+            None => {
+                if rustfs_utils::http::metadata_compat::contains_key_str(
+                    &opts.user_defined,
+                    rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
+                ) {
+                    return Err(Error::other(
+                        "data movement multipart completion cannot publish transition ownership without a publication capability",
+                    ));
+                }
+                None
+            }
+        };
         #[cfg(test)]
         pause_data_movement_multipart_before_selected_completion(bucket).await;
         let pool = self
@@ -911,6 +978,7 @@ impl ECStore {
                 },
             )
             .await;
+        drop(publication_guard);
         let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
         if result.is_ok() {
             list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;

crates/ecstore/src/store/rebalance.rs

@@ -309,6 +309,17 @@ impl ECStore {
     }
 
     pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
+            let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+            let authorization = opts
+                .tier_delete_dispatch_authorization
+                .as_ref()
+                .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
+            authorization.ensure_current(bucket, incarnation, object)?;
+            Some((authorization, incarnation))
+        } else {
+            None
+        };
         if opts.lifecycle_delete_all.is_some() {
             let mut preflight_opts = opts.clone();
             preflight_opts
@@ -317,6 +328,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
                 pool.delete_object(bucket, object, preflight_opts.clone()).await?;
@@ -326,6 +340,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .lock()
                 .mark_mutation_started();
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.mark_mutation_started(bucket, incarnation, object)?;
+            }
             let mut non_trigger_opts = opts.clone();
             non_trigger_opts
                 .lifecycle_delete_all
@@ -333,6 +350,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::History;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
                 let mut pool_opts = non_trigger_opts.clone();
@@ -348,6 +368,9 @@ impl ECStore {
                 .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
             let mut trigger_pools = Vec::new();
             for (pool_index, pool) in self.pools.iter().enumerate() {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
                 let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
@@ -366,6 +389,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
             for pool_index in trigger_pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
                 let mut pool_opts = trigger_opts.clone();
@@ -378,7 +404,13 @@ impl ECStore {
         let mut first_error = None;
         let mut first_volume_error = None;
         let mut has_success = false;
+        if let Some((authorization, incarnation)) = dispatch_scope {
+            authorization.mark_mutation_started(bucket, incarnation, object)?;
+        }
         for pool in &self.pools {
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.ensure_current(bucket, incarnation, object)?;
+            }
             let mut opts = opts.clone();
             opts.delete_prefix = true;
             match pool.delete_object(bucket, object, opts).await {
@@ -907,8 +939,12 @@ impl ECStore {
         bucket: &str,
         object: &str,
         opts: &ObjectOptions,
+        exact: &ObjectInfo,
         errs: Vec<PoolErr>,
     ) -> Result<ObjectInfo> {
+        self.reconcile_decommission_capacity_before_exact_delete(bucket, object, opts, exact)
+            .await?;
+
         let mut results = Vec::with_capacity(errs.len());
 
         for pe in errs.iter() {

crates/filemeta/src/filemeta/version.rs

@@ -3235,16 +3235,17 @@ pub fn merge_file_meta_versions(
     requested_versions: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
-    merge_file_meta_versions_inner(quorum, strict, requested_versions, false, versions)
+    merge_file_meta_versions_inner(quorum, strict, requested_versions, false, 0, versions)
 }
 
 pub(crate) fn merge_file_meta_versions_with_write_quorum(
     quorum: usize,
     strict: bool,
     requested_versions: usize,
+    write_quorum_slack: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
-    merge_file_meta_versions_inner(quorum, strict, requested_versions, true, versions)
+    merge_file_meta_versions_inner(quorum, strict, requested_versions, true, write_quorum_slack, versions)
 }
 
 fn merge_file_meta_versions_inner(
@@ -3252,6 +3253,7 @@ fn merge_file_meta_versions_inner(
     mut strict: bool,
     requested_versions: usize,
     enforce_write_quorum: bool,
+    write_quorum_slack: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
     if quorum == 0 {
@@ -3269,7 +3271,7 @@ fn merge_file_meta_versions_inner(
 
         let required_quorum = versions[0]
             .first()
-            .map(|version| version.write_quorum(quorum).max(quorum))
+            .map(|version| version.write_quorum(quorum).saturating_sub(write_quorum_slack).max(quorum))
             .unwrap_or(quorum);
         if versions.len() >= required_quorum {
             return versions[0].clone();
@@ -3283,7 +3285,7 @@ fn merge_file_meta_versions_inner(
 
     let required_quorum = |version: &FileMetaShallowVersion| {
         if enforce_write_quorum {
-            version.write_quorum(quorum).max(quorum)
+            version.write_quorum(quorum).saturating_sub(write_quorum_slack).max(quorum)
         } else {
             quorum
         }

crates/filemeta/src/metacache.rs

@@ -53,6 +53,12 @@ pub struct MetadataResolutionParams {
     pub requested_versions: usize,
     pub bucket: String,
     pub strict: bool,
+    /// Number of set drives that were unreachable when the listing snapshot was
+    /// taken. Write-quorum enforcement relaxes each version's required quorum by
+    /// this amount (never below `obj_quorum`): a version legally committed at
+    /// write quorum can have that many of its metadata copies on drives no
+    /// reader could consult, and must not be dropped for it.
+    pub write_quorum_slack: usize,
     pub candidates: Vec<Vec<FileMetaShallowVersion>>,
 }
 
@@ -416,6 +422,7 @@ impl MetaCacheEntries {
             requested_versions: 0,
             bucket: bucket.to_string(),
             strict: false,
+            write_quorum_slack: 0,
             candidates: Vec::new(),
         })
     }
@@ -693,7 +700,12 @@ impl MetaCacheEntries {
                     .cached
                     .as_ref()
                     .and_then(|cached| cached.versions.first())
-                    .map(|version| version.write_quorum(params.obj_quorum).max(params.obj_quorum))
+                    .map(|version| {
+                        version
+                            .write_quorum(params.obj_quorum)
+                            .saturating_sub(params.write_quorum_slack)
+                            .max(params.obj_quorum)
+                    })
                     .unwrap_or(params.obj_quorum)
             } else {
                 params.obj_quorum
@@ -720,6 +732,7 @@ impl MetaCacheEntries {
                 params.obj_quorum,
                 params.strict,
                 params.requested_versions,
+                params.write_quorum_slack,
                 &params.candidates,
             )
         } else {
@@ -2415,6 +2428,97 @@ mod tests {
         assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
     }
 
+    #[test]
+    fn resolve_with_write_quorum_relaxes_requirement_by_unreachable_drives() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // EC 2+2 version, write quorum 3 of 4: committed with one holder drive
+        // now offline, so only 2 of the 3 reachable drives carry the metadata.
+        let entry = metacache_entry_with_erasure(mod_time, "etag", 2, 2);
+        let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]);
+
+        let resolved = entries()
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed version should resolve when the missing copy sits on an unreachable drive");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(mod_time));
+
+        // Without the slack the same sample is rejected outright.
+        let rejected = entries().resolve_with_write_quorum(MetadataResolutionParams {
+            obj_quorum: 3,
+            requested_versions: 1,
+            bucket: "bucket".to_string(),
+            strict: true,
+            ..Default::default()
+        });
+        assert!(rejected.is_none());
+    }
+
+    #[test]
+    fn resolve_with_write_quorum_slack_accepts_committed_latest_during_merge() {
+        let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // The newer EC 2+2 version committed at write quorum 3 of 4; one holder
+        // drive is offline, and a third reachable drive still carries only the
+        // previous version. The candidates disagree, so the merge path (not the
+        // all-agree branch) must honor the slack.
+        let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2);
+        let new_and_old_entry =
+            metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]);
+
+        let resolved = MetaCacheEntries(vec![Some(new_and_old_entry.clone()), Some(new_and_old_entry), Some(old_entry)])
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed latest should survive the merge when its missing copy is on an unreachable drive");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(new_mod_time));
+        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new-etag"));
+    }
+
+    #[test]
+    fn resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge() {
+        let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // The newer EC 2+2 version (write quorum 3) reached only ONE drive: even
+        // with one drive unreachable (slack 1) it cannot prove the relaxed
+        // quorum of 2, so the committed previous version must win the merge.
+        let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2);
+        let new_and_old_entry =
+            metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]);
+
+        let resolved = MetaCacheEntries(vec![Some(new_and_old_entry), Some(old_entry.clone()), Some(old_entry)])
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed previous version should resolve after rejecting the partial latest");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(old_mod_time));
+        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
+    }
+
     #[test]
     fn resolve_rejects_partial_directory_below_dir_quorum() {
         let partial_dir = metacache_dir_entry("prefix/");

crates/heal-contracts/src/heal_channel.rs

@@ -347,6 +347,9 @@ pub struct HealChannelRequest {
     pub id: String,
     /// Disk ID for heal disk/erasure set task
     pub disk: Option<String>,
+    /// Exact endpoints of replacement disks for an automatic erasure-set
+    /// rebuild. An empty list retains the generic erasure-set heal behavior.
+    pub heal_endpoints: Vec<String>,
     /// Bucket name
     pub bucket: String,
     /// Object prefix (optional)
@@ -594,6 +597,7 @@ pub fn create_heal_request(
         timeout_seconds: None,
         source: HealRequestSource::Internal,
         disk: None,
+        heal_endpoints: Vec::new(),
     }
 }
 
@@ -634,12 +638,13 @@ pub fn create_heal_response(
     }
 }
 
-pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
-    let req = HealChannelRequest {
+fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChannelPriority>) -> HealChannelRequest {
+    HealChannelRequest {
         id: Uuid::new_v4().to_string(),
         bucket: "".to_string(),
         object_prefix: None,
         disk: Some(set_disk_id),
+        heal_endpoints: Vec::new(),
         object_version_id: None,
         force_start: false,
         priority: priority.unwrap_or(HealChannelPriority::Low),
@@ -654,8 +659,71 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
         no_lock: None,
         timeout_seconds: None,
         source: HealRequestSource::AutoHeal,
-    };
-    send_heal_request(req).await
+    }
+}
+
+fn create_auto_replacement_disk_request(
+    pool_index: usize,
+    set_index: usize,
+    replacement_endpoint: String,
+    priority: Option<HealChannelPriority>,
+) -> HealChannelRequest {
+    let mut request = create_auto_heal_disk_request(format!("pool_{pool_index}_set_{set_index}"), priority);
+    request.heal_endpoints = vec![replacement_endpoint];
+    request.pool_index = Some(pool_index);
+    request.set_index = Some(set_index);
+    request
+}
+
+/// Submit the legacy generic erasure-set auto-heal request.
+pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
+    send_heal_request(create_auto_heal_disk_request(set_disk_id, priority)).await
+}
+
+/// Submit an automatic replacement heal for one known disk endpoint.
+///
+/// The endpoint makes the request eligible for the durable replacement intent
+/// and completion-proof path in the heal task.
+pub async fn send_heal_replacement_disk(
+    pool_index: usize,
+    set_index: usize,
+    replacement_endpoint: String,
+    priority: Option<HealChannelPriority>,
+) -> Result<(), String> {
+    send_heal_request(create_auto_replacement_disk_request(
+        pool_index,
+        set_index,
+        replacement_endpoint,
+        priority,
+    ))
+    .await
+}
+
+#[cfg(test)]
+mod auto_heal_disk_request_tests {
+    use super::*;
+
+    #[test]
+    fn replacement_disk_request_carries_its_exact_endpoint() {
+        let request =
+            create_auto_replacement_disk_request(2, 3, "http://node2:9000/drive3".to_string(), Some(HealChannelPriority::Normal));
+
+        assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
+        assert_eq!(request.heal_endpoints, ["http://node2:9000/drive3"]);
+        assert_eq!(request.pool_index, Some(2));
+        assert_eq!(request.set_index, Some(3));
+        assert_eq!(request.source, HealRequestSource::AutoHeal);
+    }
+
+    #[test]
+    fn legacy_auto_heal_disk_request_has_no_replacement_endpoint() {
+        let request = create_auto_heal_disk_request("pool_2_set_3".to_string(), None);
+
+        assert!(request.heal_endpoints.is_empty());
+        assert_eq!(request.pool_index, None);
+        assert_eq!(request.set_index, None);
+        assert_eq!(request.source, HealRequestSource::AutoHeal);
+    }
 }
 
 #[cfg(test)]

crates/heal/src/heal/channel.rs

@@ -646,10 +646,12 @@ impl HealChannelProcessor {
     /// Convert channel request to heal request
     fn convert_to_heal_request(&self, request: HealChannelRequest) -> Result<HealRequest> {
         let recursive = request.recursive.unwrap_or(false);
+        let mut inferred_set_scope = None;
         let heal_type = if let Some(disk_id) = &request.disk {
             let set_disk_id = utils::normalize_set_disk_id(disk_id).ok_or_else(|| Error::InvalidHealType {
                 heal_type: format!("erasure-set({disk_id})"),
             })?;
+            inferred_set_scope = utils::parse_set_disk_id(&set_disk_id).ok();
             HealType::ErasureSet {
                 buckets: vec![],
                 set_disk_id,
@@ -697,6 +699,11 @@ impl HealChannelProcessor {
             | HealRequestSource::Mrf => true,
         });
 
+        // `no_lock` is an internal coordination hint. An admin request can
+        // carry the legacy field over the wire, but cannot use it as ambient
+        // authority to bypass storage namespace locking.
+        let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin;
+
         // Build HealOptions with all available fields
         let options = HealOptions {
             scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
@@ -705,15 +712,16 @@ impl HealChannelProcessor {
             update_parity: request.update_parity.unwrap_or(true),
             recursive,
             dry_run: request.dry_run.unwrap_or(false),
-            no_lock: request.no_lock.unwrap_or(false),
+            no_lock,
             timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
-            pool_index: request.pool_index,
-            set_index: request.set_index,
+            pool_index: request.pool_index.or_else(|| inferred_set_scope.map(|(pool, _)| pool)),
+            set_index: request.set_index.or_else(|| inferred_set_scope.map(|(_, set)| set)),
         };
 
         let mut heal_request = HealRequest::new(heal_type, options, priority);
         heal_request.id = request.id;
         heal_request.source = request.source;
+        heal_request.heal_endpoints = request.heal_endpoints;
         // force_start controls admission/queue semantics only. Do not reinterpret it as
         // destructive heal options: admin clients commonly pass forceStart=true together
         // with remove=false, and turning that into remove_corrupted=true can delete the
@@ -901,6 +909,7 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Normal,
             scan_mode: None,
             remove_corrupted: None,
@@ -933,6 +942,7 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::High,
             scan_mode: Some(HealScanMode::Normal),
             remove_corrupted: Some(false),
@@ -965,6 +975,7 @@ mod tests {
             object_prefix: Some("test-object".to_string()),
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::High,
             scan_mode: Some(HealScanMode::Deep),
             remove_corrupted: Some(true),
@@ -990,6 +1001,23 @@ mod tests {
         assert!(heal_request.options.no_lock);
     }
 
+    #[tokio::test]
+    async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() {
+        let heal_manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(heal_manager);
+        let channel_request = HealChannelRequest {
+            id: "admin-no-lock".to_string(),
+            bucket: "test-bucket".to_string(),
+            object_prefix: Some("test-object".to_string()),
+            no_lock: Some(true),
+            source: HealRequestSource::Admin,
+            ..Default::default()
+        };
+
+        let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
+        assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority");
+    }
+
     #[tokio::test]
     async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() {
         let heal_manager = create_test_heal_manager();
@@ -1001,6 +1029,7 @@ mod tests {
             object_prefix: Some("test-object".to_string()),
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Low,
             scan_mode: None,
             remove_corrupted: None,
@@ -1038,6 +1067,7 @@ mod tests {
                 object_prefix: Some("test-object".to_string()),
                 object_version_id: None,
                 disk: None,
+                heal_endpoints: Vec::new(),
                 priority: HealChannelPriority::Normal,
                 scan_mode: None,
                 remove_corrupted: None,
@@ -1077,6 +1107,7 @@ mod tests {
                 object_prefix: Some("test-object".to_string()),
                 object_version_id: None,
                 disk: None,
+                heal_endpoints: Vec::new(),
                 priority: HealChannelPriority::Normal,
                 scan_mode: None,
                 remove_corrupted: None,
@@ -1109,6 +1140,7 @@ mod tests {
             object_prefix: Some("logs/".to_string()),
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::High,
             scan_mode: Some(HealScanMode::Normal),
             remove_corrupted: Some(false),
@@ -1144,7 +1176,10 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: Some("pool_0_set_1".to_string()),
+            heal_endpoints: vec!["http://node0:9000/drive1".to_string()],
             priority: HealChannelPriority::Critical,
+            pool_index: Some(0),
+            set_index: Some(1),
             scan_mode: None,
             remove_corrupted: None,
             recreate_missing: None,
@@ -1153,15 +1188,16 @@ mod tests {
             dry_run: None,
             no_lock: None,
             timeout_seconds: None,
-            pool_index: None,
-            set_index: None,
             force_start: false,
-            source: HealRequestSource::Internal,
+            source: HealRequestSource::AutoHeal,
         };
 
         let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
         assert!(matches!(heal_request.heal_type, HealType::ErasureSet { .. }));
         assert_eq!(heal_request.priority, HealPriority::Urgent);
+        assert_eq!(heal_request.heal_endpoints, ["http://node0:9000/drive1"]);
+        assert_eq!(heal_request.options.pool_index, Some(0));
+        assert_eq!(heal_request.options.set_index, Some(1));
     }
 
     #[tokio::test]
@@ -1175,6 +1211,7 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: Some("invalid-disk-id".to_string()),
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Normal,
             scan_mode: None,
             remove_corrupted: None,
@@ -1213,6 +1250,7 @@ mod tests {
                 object_prefix: None,
                 object_version_id: None,
                 disk: None,
+                heal_endpoints: Vec::new(),
                 priority: channel_priority,
                 scan_mode: None,
                 remove_corrupted: None,
@@ -1244,6 +1282,7 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Normal,
             scan_mode: None,
             remove_corrupted: Some(false),
@@ -1277,6 +1316,7 @@ mod tests {
             object_prefix: Some("".to_string()), // Empty prefix should be treated as bucket heal
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Normal,
             scan_mode: None,
             remove_corrupted: None,
@@ -1314,6 +1354,7 @@ mod tests {
             object_prefix: Some("object".to_string()),
             object_version_id: None,
             disk: None,
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Low,
             scan_mode: Some(HealScanMode::Normal),
             remove_corrupted: None,
@@ -1592,6 +1633,7 @@ mod tests {
             object_prefix: None,
             object_version_id: None,
             disk: Some("invalid".to_string()),
+            heal_endpoints: Vec::new(),
             priority: HealChannelPriority::Normal,
             scan_mode: None,
             remove_corrupted: None,

crates/heal/src/heal/mod.rs

@@ -26,12 +26,10 @@ pub mod utils;
 
 use storage_api::owner::{
     ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
-    EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult,
-    EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
-    ecstore_local_disk_map_read,
+    EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption,
+    EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO,
+    ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
 };
-#[cfg(test)]
-use storage_api::owner::{EcstoreDiskOption, ecstore_new_disk};
 
 pub use erasure_healer::ErasureSetHealer;
 pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
@@ -247,10 +245,8 @@ pub(crate) async fn local_disk_map_read() -> tokio::sync::OwnedRwLockReadGuard<L
     ecstore_local_disk_map_read().await
 }
 
-#[cfg(test)]
 pub(crate) type DiskOption = EcstoreDiskOption;
 
-#[cfg(test)]
 pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> DiskResult<DiskStore> {
     ecstore_new_disk(ep, opt).await
 }

crates/heal/src/heal/replacement_readiness.rs

@@ -14,9 +14,9 @@
 
 use std::{fs, path::Path};
 
-#[cfg(test)]
-use super::Endpoint;
-use super::{DiskStore, HealDiskExt as _, local_disk_map_read, resume::ReplacementTargetIdentity};
+use super::{
+    DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
+};
 
 pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
     auto_replacement_target_identity(disk, local_disks).await.is_some()
@@ -72,8 +72,38 @@ pub(crate) async fn auto_replacement_target_identity(
     .flatten()
 }
 
-pub(crate) async fn auto_replacement_targets_ready(targets: &[String]) -> bool {
-    auto_replacement_target_identities(targets).await.is_some()
+fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Option<Endpoint> {
+    let mut endpoint = Endpoint::try_from(target).ok()?;
+    if endpoint.is_local {
+        return Some(endpoint);
+    }
+
+    let grid_host = endpoint.grid_host();
+    if grid_host.is_empty() || !local_grid_hosts.iter().any(|local_host| local_host == &grid_host) {
+        return None;
+    }
+
+    endpoint.is_local = true;
+    Some(endpoint)
+}
+
+async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
+    if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
+        return Some(disk.clone());
+    }
+
+    let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
+    let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
+
+    new_disk(
+        &endpoint,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await
+    .ok()
 }
 
 pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
@@ -88,8 +118,8 @@ pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Op
 
     let mut identities = Vec::with_capacity(targets.len());
     for target in targets {
-        let disk = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target)?;
-        identities.push(auto_replacement_target_identity(disk, &local_disks).await?);
+        let disk = replacement_target_disk(target, &local_disks).await?;
+        identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
     }
     identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
     identities.dedup_by(|left, right| left.endpoint == right.endpoint);
@@ -131,6 +161,29 @@ mod tests {
     use super::*;
     use tempfile::TempDir;
 
+    #[test]
+    fn local_replacement_endpoint_accepts_a_url_on_a_registered_local_grid_host() {
+        let local_grid_hosts = vec!["http://127.0.0.1:9000".to_owned()];
+        let endpoint = local_replacement_endpoint("http://127.0.0.1:9000/replacement", &local_grid_hosts)
+            .expect("matching local grid host should be accepted");
+
+        assert!(endpoint.is_local);
+    }
+
+    #[test]
+    fn local_replacement_endpoint_rejects_a_url_on_an_unregistered_grid_host() {
+        let local_grid_hosts = vec!["http://127.0.0.1:9000".to_owned()];
+
+        assert!(local_replacement_endpoint("http://127.0.0.1:9001/replacement", &local_grid_hosts).is_none());
+    }
+
+    #[test]
+    fn local_replacement_endpoint_keeps_a_local_path_local() {
+        let endpoint = local_replacement_endpoint("/replacement", &[]).expect("local path should be accepted");
+
+        assert!(endpoint.is_local);
+    }
+
     #[tokio::test]
     async fn runtime_environment_cannot_bypass_mount_admission() {
         temp_env::async_with_vars(

crates/heal/src/heal/storage.rs

@@ -394,11 +394,6 @@ pub trait HealStorageAPI: Send + Sync {
         Err(Error::other("target-scoped replacement format is unsupported"))
     }
 
-    /// Recheck admitted replacement targets immediately before destructive work.
-    async fn replacement_targets_ready(&self, _targets: &[String]) -> Result<bool> {
-        Ok(false)
-    }
-
     /// Read target-specific physical evidence for one replacement version.
     ///
     /// This is only used by automatic replacement healing after the normal
@@ -1171,10 +1166,6 @@ impl HealStorageAPI for ECStoreHealStorage {
             .map_err(Error::Storage)
     }
 
-    async fn replacement_targets_ready(&self, targets: &[String]) -> Result<bool> {
-        Ok(super::replacement_readiness::auto_replacement_targets_ready(targets).await)
-    }
-
     async fn replacement_targets_have_version(
         &self,
         bucket: &str,

crates/heal/src/heal/storage_api.rs

@@ -24,7 +24,6 @@ pub(crate) use rustfs_ecstore::api::disk::{
     DiskStore as EcstoreDiskStore, HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH,
     RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
 };
-#[cfg(test)]
 pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
 pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
 pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
@@ -43,7 +42,6 @@ pub(crate) mod owner {
         EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
     };
 
-    #[cfg(test)]
     pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
 }
 

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -93,16 +93,6 @@ impl HealTask {
             None
         };
 
-        if is_auto_replacement
-            && !self
-                .await_with_control(self.storage.replacement_targets_ready(&self.heal_endpoints))
-                .await?
-        {
-            return Err(Error::TaskExecutionFailed {
-                message: format!("Replacement target is no longer ready for automatic heal {set_disk_id}"),
-            });
-        }
-
         let replacement_resume_disk = if is_auto_replacement {
             Some(match replacement_resume_disk {
                 Some(disk) => disk,
@@ -492,6 +482,7 @@ impl HealTask {
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
+            Err(e) if e.is_recoverable_heal() => Err(e),
             Err(e) => {
                 error!(
                     target: "rustfs::heal::task",

crates/heal/src/heal/task/tests.rs

@@ -84,7 +84,7 @@ async fn automatic_replacement_uses_target_scoped_format() {
     let temp = TempDir::new().expect("temporary resume disk directory should be created");
     let disk = make_resume_disk(&temp).await;
     let storage = Arc::new(MockStorage {
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         resume_disk: Mutex::new(Some(disk)),
         ..Default::default()
     });
@@ -123,7 +123,7 @@ async fn automatic_replacement_uses_target_scoped_format() {
 #[tokio::test]
 async fn automatic_replacement_persists_intent_before_format() {
     let storage = Arc::new(MockStorage {
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         ..Default::default()
     });
     let mut request = HealRequest::new(
@@ -155,7 +155,7 @@ async fn automatic_replacement_persists_intent_before_format() {
 #[tokio::test]
 async fn recovered_replacement_never_uses_a_fresh_resume_disk() {
     let storage = Arc::new(MockStorage {
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         ..Default::default()
     });
     let mut request = HealRequest::new(
@@ -193,7 +193,7 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
     let first_identity = replacement_identity("replacement-a", "device-a", "filesystem-a");
     let second_identity = replacement_identity("replacement-a", "device-b", "filesystem-b");
     let storage = Arc::new(MockStorage {
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         replacement_target_identity_sequences: Mutex::new(VecDeque::from([
             vec![first_identity.clone()],
             vec![first_identity.clone()],
@@ -259,7 +259,7 @@ async fn automatic_replacement_reuses_an_existing_non_target_resume_anchor() {
     .await
     .expect("existing intent should be stored on the non-target anchor");
     let storage = Arc::new(MockStorage {
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         replacement_resume_disk: Mutex::new(Some(anchor.clone())),
         ..Default::default()
     });
@@ -493,7 +493,7 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
 
     let storage = Arc::new(MockStorage {
         replacement_resume_disk: Mutex::new(Some(anchor.clone())),
-        replacement_targets_ready: Mutex::new(true),
+        replacement_target_identities_ready: Mutex::new(true),
         ..Default::default()
     });
     let mut request = HealRequest::new(
@@ -551,7 +551,7 @@ struct MockStorage {
     format_error: Mutex<Option<Error>>,
     global_format_calls: Mutex<u32>,
     replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
-    replacement_targets_ready: Mutex<bool>,
+    replacement_target_identities_ready: Mutex<bool>,
     replacement_target_identity_sequences: Mutex<VecDeque<Vec<crate::heal::resume::ReplacementTargetIdentity>>>,
     listed_prefixes: Mutex<Vec<String>>,
     truncate_without_token: Mutex<bool>,
@@ -943,10 +943,6 @@ impl HealStorageAPI for MockStorage {
         ))
     }
 
-    async fn replacement_targets_ready(&self, _targets: &[String]) -> Result<bool> {
-        Ok(*self.replacement_targets_ready.lock().unwrap())
-    }
-
     async fn list_objects_for_heal_page(
         &self,
         bucket: &str,
@@ -1028,7 +1024,7 @@ impl HealStorageAPI for MockStorage {
         &self,
         targets: &[String],
     ) -> Result<Vec<crate::heal::resume::ReplacementTargetIdentity>> {
-        if !*self.replacement_targets_ready.lock().unwrap() {
+        if !*self.replacement_target_identities_ready.lock().unwrap() {
             return Err(Error::other("replacement target is not ready"));
         }
         if let Some(identities) = self.replacement_target_identity_sequences.lock().unwrap().pop_front() {
@@ -2428,6 +2424,37 @@ async fn erasure_set_format_slowdown_is_propagated() {
     assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
 }
 
+#[tokio::test]
+async fn erasure_set_retry_signal_remains_typed_across_task_boundary() {
+    let temp = TempDir::new().expect("temporary directory should be created");
+    let disk = make_resume_disk(&temp).await;
+    let storage = Arc::new(MockStorage {
+        heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::RetryableSlowDown)),
+        resume_disk: Mutex::new(Some(disk)),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::ErasureSet {
+            buckets: vec!["bucket-a".to_string()],
+            set_disk_id: "pool_0_set_0".to_string(),
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage);
+
+    let error = task
+        .execute()
+        .await
+        .expect_err("an incomplete resumable pass must remain retryable");
+
+    assert!(
+        matches!(&error, Error::TransientSkip { message } if message.contains("retry scheduled")),
+        "the resumable retry signal must keep its typed identity: {error}"
+    );
+    assert!(error.is_recoverable_heal(), "the scheduler must accept the preserved retry signal");
+}
+
 #[tokio::test]
 async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/license/Cargo.toml

@@ -0,0 +1,29 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-license"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "License and entitlement provider contracts for RustFS"
+
+[lints]
+workspace = true
+
+[dependencies]
+thiserror = { workspace = true }

crates/license/src/lib.rs

@@ -0,0 +1,133 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Provider-neutral license and entitlement contracts.
+
+use std::{fmt, sync::Arc};
+
+use thiserror::Error;
+
+pub type LicenseResult<T> = Result<T, LicenseError>;
+pub type SharedLicenseProvider = Arc<dyn LicenseProvider>;
+
+/// Entitlement required by the existing server-wide license gate.
+pub const SERVER_ENTITLEMENT: &str = "rustfs.server";
+
+/// Provider-neutral metadata exposed through existing RustFS status APIs.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct LicenseMetadata {
+    pub subject: String,
+    pub expires_at: Option<u64>,
+}
+
+/// Sanitized provider state suitable for status and diagnostics output.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub enum LicenseStatus {
+    #[default]
+    Uninitialized,
+    Valid,
+    Missing,
+    Invalid(String),
+    Unavailable,
+}
+
+impl fmt::Display for LicenseStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Uninitialized => write!(f, "uninitialized"),
+            Self::Valid => write!(f, "valid"),
+            Self::Missing => write!(f, "missing"),
+            Self::Invalid(message) => write!(f, "{message}"),
+            Self::Unavailable => write!(f, "unavailable"),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum LicenseError {
+    #[error("License state is unavailable")]
+    StatePoisoned,
+    #[error("License is required when building with feature `license`.")]
+    Missing,
+    #[error("Incorrect license, please contact RustFS. {0}")]
+    Invalid(String),
+    #[error("Incorrect license, please contact RustFS. expired_at={expired_at}, now={now}")]
+    Expired { expired_at: u64, now: u64 },
+    #[error("Failed to read system time: {0}")]
+    Clock(String),
+    #[error("Entitlement is not granted: {entitlement}")]
+    Denied { entitlement: String },
+    #[error("License provider is unavailable: {0}")]
+    Unavailable(String),
+    #[error("Entitlement identifier is empty or not normalized")]
+    InvalidEntitlement,
+}
+
+/// Runtime boundary between RustFS and a license implementation.
+///
+/// Providers must sanitize all strings returned in errors, status, and
+/// metadata. In particular, they must never include raw license material.
+/// `initialize` must be safe to call repeatedly with the same input. `check`
+/// must be idempotent and must not consume quota, acquire a lease, or mutate
+/// external state. Providers that require a license must fail closed before
+/// successful initialization.
+pub trait LicenseProvider: Send + Sync {
+    fn initialize(&self, raw_license: Option<&str>) -> LicenseResult<()>;
+
+    fn check(&self, entitlement: &str) -> LicenseResult<()>;
+
+    fn status(&self) -> LicenseStatus;
+
+    fn metadata(&self) -> Option<LicenseMetadata> {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug)]
+    struct TestProvider;
+
+    impl LicenseProvider for TestProvider {
+        fn initialize(&self, _raw_license: Option<&str>) -> LicenseResult<()> {
+            Ok(())
+        }
+
+        fn check(&self, _entitlement: &str) -> LicenseResult<()> {
+            Ok(())
+        }
+
+        fn status(&self) -> LicenseStatus {
+            LicenseStatus::Valid
+        }
+    }
+
+    #[test]
+    fn provider_is_object_safe() {
+        let provider: SharedLicenseProvider = Arc::new(TestProvider);
+
+        assert_eq!(provider.status(), LicenseStatus::Valid);
+    }
+
+    #[test]
+    fn status_display_is_stable() {
+        assert_eq!(LicenseStatus::Uninitialized.to_string(), "uninitialized");
+        assert_eq!(LicenseStatus::Valid.to_string(), "valid");
+        assert_eq!(LicenseStatus::Missing.to_string(), "missing");
+        assert_eq!(LicenseStatus::Invalid("invalid key".to_string()).to_string(), "invalid key");
+        assert_eq!(LicenseStatus::Unavailable.to_string(), "unavailable");
+    }
+}

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
     #[prost(bytes = "bytes", tag = "5")]
     pub response_proof: ::prost::bytes::Bytes,
+    #[prost(enumeration = "TierMutationFailureClass", tag = "6")]
+    pub failure_class: i32,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetLiveEventsRequest {
@@ -1611,6 +1613,35 @@ impl TierMutationPeerState {
         }
     }
 }
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum TierMutationFailureClass {
+    Unspecified = 0,
+    PreDispatchRejected = 1,
+    Ambiguous = 2,
+}
+impl TierMutationFailureClass {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED",
+            Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED",
+            Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified),
+            "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected),
+            "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous),
+            _ => None,
+        }
+    }
+}
 /// Generated client implementations.
 pub mod node_service_client {
     #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]

crates/protos/src/heal_control.rs

@@ -88,6 +88,8 @@ impl From<Priority> for HealChannelPriority {
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct StartCommand {
     disk: Option<String>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    heal_endpoints: Vec<String>,
     bucket: String,
     object_prefix: Option<String>,
     object_version_id: Option<String>,
@@ -113,6 +115,7 @@ impl TryFrom<HealChannelRequest> for StartCommand {
     fn try_from(request: HealChannelRequest) -> Result<Self, Self::Error> {
         Ok(Self {
             disk: request.disk,
+            heal_endpoints: request.heal_endpoints,
             bucket: request.bucket,
             object_prefix: request.object_prefix,
             object_version_id: request.object_version_id,
@@ -146,6 +149,7 @@ impl StartCommand {
         Ok(HealChannelRequest {
             id: request_id,
             disk: self.disk,
+            heal_endpoints: self.heal_endpoints,
             bucket: self.bucket,
             object_prefix: self.object_prefix,
             object_version_id: self.object_version_id,
@@ -632,6 +636,12 @@ mod tests {
         }
     }
 
+    fn replacement_test_request(request_id: String) -> HealChannelRequest {
+        let mut request = test_request(request_id);
+        request.heal_endpoints = vec!["http://node1:9000/drive2".to_string()];
+        request
+    }
+
     fn metadata(byte: u8, epoch: u64) -> RequestMetadata {
         RequestMetadata::new([byte; 16], 1_000, 2_000, epoch)
     }
@@ -639,7 +649,7 @@ mod tests {
     #[test]
     fn round_trips_all_commands_and_results() {
         let request_id = uuid::Uuid::new_v4().to_string();
-        let start = Envelope::start(test_request(request_id), metadata(1, 7)).unwrap();
+        let start = Envelope::start(replacement_test_request(request_id), metadata(1, 7)).unwrap();
         let query = Envelope::query(
             uuid::Uuid::new_v4().to_string(),
             metadata(2, 7),

crates/protos/src/lib.rs

@@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re
 pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
 pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
 pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
+pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
 
 pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
@@ -342,7 +343,16 @@ impl TierMutationRpcPhase {
     }
 }
 
-pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
+// Version 2 required peer Prepare to block new tier-reference creators and
+// drain their in-flight operation leases. Version 3 additionally binds Abort
+// to the canonical Prepare intent so a missing-record Abort can persist an
+// identity-bound tombstone and linearize against a delayed Prepare. Version 4
+// signs a typed failure classification while retaining the exact v3 proof
+// bytes for rolling compatibility.
+pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3;
+pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4;
+pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024;
+pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096;
 
 pub fn canonical_tier_mutation_rpc_body(
     version: u32,
@@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> {
     pub state: i32,
     pub applied: bool,
     pub error_info: Option<&'a str>,
+    pub failure_class: i32,
 }
 
 pub fn canonical_tier_mutation_rpc_response_body(
     input: TierMutationRpcResponseProofInput<'_>,
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
+    const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
+    const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0";
 
     let phase = input.phase.as_wire_str().as_bytes();
     let mutation_id = input.mutation_id.as_bytes();
     let error_info = input.error_info.map(str::as_bytes);
     let error_info_len = error_info.map_or(0, <[u8]>::len);
+    let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION;
+    let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN };
     let mut body = Vec::with_capacity(
-        DOMAIN.len()
+        domain.len()
             + 4
             + 8
             + phase.len()
@@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body(
             + 1
             + 1
             + 8
-            + error_info_len,
+            + error_info_len
+            + if is_v4 { 4 } else { 0 },
     );
-    body.extend_from_slice(DOMAIN);
+    body.extend_from_slice(domain);
     body.extend_from_slice(&input.version.to_be_bytes());
     body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
     body.extend_from_slice(phase);
@@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body(
     if let Some(error_info) = error_info {
         body.extend_from_slice(error_info);
     }
+    if is_v4 {
+        body.extend_from_slice(&input.failure_class.to_be_bytes());
+    }
     Ok(body)
 }
 
@@ -2137,10 +2155,10 @@ mod heal_control_tests {
 #[cfg(test)]
 mod tier_mutation_rpc_tests {
     use super::{
-        TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
-        canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
+        TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase,
+        TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
     };
-    use crate::proto_gen::node_service::TierMutationPeerState;
+    use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState};
     use uuid::uuid;
 
     #[test]
@@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests {
         )
         .expect("small mutation body should encode");
         let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
-        golden.extend_from_slice(&1_u32.to_be_bytes());
+        golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes());
         golden.extend_from_slice(&7_u64.to_be_bytes());
         golden.extend_from_slice(b"prepare");
         golden.extend_from_slice(mutation_id.as_bytes());
@@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests {
 
         assert_ne!(
             baseline,
-            canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
-                .expect("small mutation body should encode")
+            canonical_tier_mutation_rpc_body(
+                TIER_MUTATION_RPC_PROTOCOL_VERSION + 1,
+                TierMutationRpcPhase::Prepare,
+                mutation_id,
+                payload,
+            )
+            .expect("small mutation body should encode")
         );
         assert_ne!(
             baseline,
@@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests {
     }
 
     #[test]
-    fn canonical_tier_mutation_response_binds_request_state_and_error() {
+    fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() {
         let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
         let payload = b"canonical-intent-record";
-        let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
-            version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+        let request = canonical_tier_mutation_rpc_body(
+            TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+        )
+        .expect("v3 request should encode");
+        let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
+        request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
+        request_golden.extend_from_slice(&7_u64.to_be_bytes());
+        request_golden.extend_from_slice(b"prepare");
+        request_golden.extend_from_slice(mutation_id.as_bytes());
+        request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
+        request_golden.extend_from_slice(payload);
+        assert_eq!(request, request_golden);
+
+        let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
+            version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
             phase: TierMutationRpcPhase::Prepare,
             mutation_id,
             canonical_payload: payload,
@@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
+            // v3 must ignore the field so its authenticated bytes stay exact.
+            failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
         })
-        .expect("small mutation response should encode");
+        .expect("v3 response should encode");
+        let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec();
+        response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
+        response_golden.extend_from_slice(&7_u64.to_be_bytes());
+        response_golden.extend_from_slice(b"prepare");
+        response_golden.extend_from_slice(mutation_id.as_bytes());
+        response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
+        response_golden.extend_from_slice(payload);
+        response_golden.push(1);
+        response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes());
+        response_golden.push(1);
+        response_golden.push(0);
+        response_golden.extend_from_slice(&0_u64.to_be_bytes());
+        assert_eq!(response, response_golden);
+    }
+
+    #[test]
+    fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() {
+        let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
+        let payload = b"canonical-intent-record";
+        let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
+            version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            phase: TierMutationRpcPhase::Prepare,
+            mutation_id,
+            canonical_payload: payload,
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: Some("failure"),
+            failure_class: TierMutationFailureClass::Ambiguous as i32,
+        })
+        .expect("small v4 mutation response should encode");
 
         let cases = [
             TierMutationRpcResponseProofInput {
-                version: 2,
+                version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Commit,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: b"tampered-intent-record",
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
                 success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests {
                 mutation_id,
                 canonical_payload: payload,
                 success: false,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
-            },
-            TierMutationRpcResponseProofInput {
-                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
-                phase: TierMutationRpcPhase::Prepare,
-                mutation_id,
-                canonical_payload: payload,
-                success: true,
                 state: TierMutationPeerState::Committed as i32,
-                applied: true,
-                error_info: None,
-            },
-            TierMutationRpcResponseProofInput {
-                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
-                phase: TierMutationRpcPhase::Prepare,
-                mutation_id,
-                canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
                 applied: false,
-                error_info: None,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
                 applied: true,
-                error_info: Some("error"),
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("other failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
             },
         ];
         for case in cases {

crates/protos/src/node.proto

@@ -1068,12 +1068,19 @@ enum TierMutationPeerState {
   TIER_MUTATION_PEER_STATE_ABORTED = 3;
 }
 
+enum TierMutationFailureClass {
+  TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0;
+  TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1;
+  TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2;
+}
+
 message TierMutationControlResponse {
   bool success = 1;
   TierMutationPeerState state = 2;
   bool applied = 3;
   optional string error_info = 4;
   bytes response_proof = 5;
+  TierMutationFailureClass failure_class = 6;
 }
 
 message GetLiveEventsRequest {

crates/replication/src/lib.rs

@@ -59,8 +59,9 @@ pub use mrf::{
     MrfV2Envelope, MrfV2Error, MrfV2Reader, MrfV2Readiness, decode_mrf_file, encode_mrf_file,
 };
 pub use multipart::{
-    ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan,
+    REPLICATION_MAX_SINGLE_PUT_SIZE, ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError,
+    ReplicationMultipartRange, replication_multipart_complete_actual_size, replication_multipart_part_plan,
+    replication_single_put_size_error,
 };
 pub use object::{
     ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag,

crates/replication/src/multipart.rs

@@ -109,15 +109,49 @@ pub fn replication_multipart_complete_actual_size(user_defined: &HashMap<String,
     get_internal_metadata(user_defined, SUFFIX_ACTUAL_SIZE).unwrap_or_default()
 }
 
+/// Largest body S3 accepts on a single `PutObject`. Anything above this has to
+/// be uploaded as multipart; the limit is part of the S3 API, not a RustFS
+/// tunable, so every generic S3 target enforces it.
+pub const REPLICATION_MAX_SINGLE_PUT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
+
+/// Reject a single-`PutObject` replication transfer the target can never accept.
+///
+/// Replication mirrors the object's *source-side storage shape*: an object
+/// written to the source with one `PutObject` replicates with one `PutObject`
+/// whatever its size, and a multipart object replays the source's own part
+/// layout. So a source object larger than [`REPLICATION_MAX_SINGLE_PUT_SIZE`]
+/// that was not written as multipart can never reach a generic S3 target — the
+/// remote rejects it with `EntityTooLarge`, but only after the whole body has
+/// been streamed to it (rustfs#6825).
+///
+/// Returning the failure up front turns an unbounded wasted transfer plus an
+/// opaque remote error into a stated, diagnosable limit. RustFS deliberately
+/// does not re-chunk such an object into multipart on the replication side:
+/// the target's part layout is the source's, and rewriting it would break the
+/// ETag/part identity that heal and delete convergence address.
+pub fn replication_single_put_size_error(is_multipart: bool, transfer_size: i64) -> Option<String> {
+    if is_multipart || transfer_size <= REPLICATION_MAX_SINGLE_PUT_SIZE {
+        return None;
+    }
+    Some(format!(
+        "object of {transfer_size} bytes was not written as multipart on the source and exceeds the \
+         {REPLICATION_MAX_SINGLE_PUT_SIZE} byte single-PutObject limit of an S3 target; \
+         re-upload it with multipart to make it replicable"
+    ))
+}
+
 #[cfg(test)]
 mod tests {
     use super::{
-        ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
-        replication_multipart_complete_actual_size, replication_multipart_part_plan,
+        REPLICATION_MAX_SINGLE_PUT_SIZE, ReplicationMultipartPartInput, ReplicationMultipartPartPlan,
+        ReplicationMultipartPlanError, ReplicationMultipartRange, replication_multipart_complete_actual_size,
+        replication_multipart_part_plan, replication_single_put_size_error,
     };
     use crate::http::{SUFFIX_ACTUAL_SIZE, insert_internal_metadata};
     use std::collections::HashMap;
 
+    const MIB: i64 = 1024 * 1024;
+
     #[test]
     fn multipart_part_plan_builds_range_and_next_offset() {
         assert_eq!(
@@ -219,4 +253,44 @@ mod tests {
         assert_eq!(replication_multipart_complete_actual_size(&user_defined), "123");
         assert!(replication_multipart_complete_actual_size(&HashMap::new()).is_empty());
     }
+
+    #[test]
+    fn single_put_size_guard_admits_transfers_a_target_can_accept() {
+        for size in [
+            0,
+            1,
+            MIB,
+            REPLICATION_MAX_SINGLE_PUT_SIZE - 1,
+            REPLICATION_MAX_SINGLE_PUT_SIZE,
+        ] {
+            assert_eq!(
+                replication_single_put_size_error(false, size),
+                None,
+                "single PUT of {size} bytes is within the S3 limit and must not be rejected"
+            );
+        }
+    }
+
+    #[test]
+    fn single_put_size_guard_rejects_an_oversized_single_put() {
+        let size = REPLICATION_MAX_SINGLE_PUT_SIZE + 1;
+        let err = replication_single_put_size_error(false, size).expect("oversized single PUT must be rejected");
+
+        // The message is the operator's diagnosis: it has to name the actual
+        // size, the limit, and the reason the object is on this route at all.
+        assert!(err.contains(&size.to_string()), "message must name the object size: {err}");
+        assert!(
+            err.contains(&REPLICATION_MAX_SINGLE_PUT_SIZE.to_string()),
+            "message must name the limit: {err}"
+        );
+        assert!(err.contains("multipart"), "message must name the remedy: {err}");
+    }
+
+    #[test]
+    fn single_put_size_guard_never_rejects_the_multipart_route() {
+        // Multipart replays the source part layout, so object size alone says
+        // nothing about whether the target will accept it; the per-part limits
+        // are the target's to enforce.
+        assert_eq!(replication_single_put_size_error(true, REPLICATION_MAX_SINGLE_PUT_SIZE * 1024), None);
+    }
 }

crates/s3select-api/src/input_stream.rs

@@ -326,13 +326,13 @@ fn invalid_gzip_header_error() -> io::Error {
     io::Error::new(io::ErrorKind::InvalidData, CompressionFormat::Gzip.invalid_header_error())
 }
 
-struct BlockingChannelReader {
+pub(crate) struct BlockingChannelReader {
     receiver: mpsc::Receiver<io::Result<Bytes>>,
     current: Bytes,
 }
 
 impl BlockingChannelReader {
-    fn new(receiver: mpsc::Receiver<io::Result<Bytes>>) -> Self {
+    pub(crate) fn new(receiver: mpsc::Receiver<io::Result<Bytes>>) -> Self {
         Self {
             receiver,
             current: Bytes::new(),
@@ -639,7 +639,7 @@ fn error_chain_contains<T: StdError + 'static>(error: &(dyn StdError + 'static))
     find_error_source::<T>(error).is_some()
 }
 
-fn find_error_source<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> {
+pub(crate) fn find_error_source<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> {
     let mut current = Some(error);
     for _ in 0..MAX_ERROR_SOURCE_DEPTH {
         let Some(error) = current else {