Merge branch 'main' into overtrue/fix/lifecycle-rule-validation

* test(odm): keep listing header import test scoped

* fix(build): gate GCS-only migration HTTP helpers

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
 sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
-sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
+sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b

.config/ecstore-required-tests.json

@@ -0,0 +1,72 @@
+{
+  "lane": "ci/test-and-lint",
+  "tests": [
+    {
+      "invariant": "write-quorum",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
+    },
+    {
+      "invariant": "metadata-rollback",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
+    },
+    {
+      "invariant": "stale-writer",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
+    },
+    {
+      "invariant": "range-body",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
+    },
+    {
+      "invariant": "multipart-cancellation",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
+    },
+    {
+      "invariant": "list-uncommitted-version",
+      "suite": "rustfs-filemeta",
+      "name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
+    },
+    {
+      "invariant": "minio-object-fixture",
+      "suite": "rustfs-filemeta",
+      "name": "filemeta::test::parses_real_minio_object_xlmeta"
+    },
+    {
+      "invariant": "corrupt-part-arrays",
+      "suite": "rustfs-filemeta",
+      "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
+    }
+  ],
+  "fixtures": [
+    {
+      "path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
+      "sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
+      "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
+    },
+    {
+      "path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
+      "sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
+      "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
+    },
+    {
+      "path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
+      "sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
+      "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
+    },
+    {
+      "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
+      "sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
+      "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
+    },
+    {
+      "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
+      "sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
+      "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
+    }
+  ]
+}

.config/make/pre-commit.mak

@@ -3,9 +3,10 @@
 .NOTPARALLEL: pre-commit pre-pr dev-check
 
 .PHONY: setup-hooks
-setup-hooks: ## Set up git hooks
+setup-hooks: ## Install the configured pre-commit hooks
 	@echo "🔧 Setting up git hooks..."
-	chmod +x .git/hooks/pre-commit
+	pre-commit validate-config
+	pre-commit install
 	@echo "✅ Git hooks setup complete!"
 
 .PHONY: doc-paths-check

.config/make/tests.mak

@@ -40,6 +40,8 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test

.config/nextest.toml

@@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
 setup = 'ecstore-large-stack'
 
 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
 
 [[profile.default.scripts]]
@@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
 setup = 'ecstore-large-stack'
 
 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
 
 [[profile.ci.scripts]]

.github/actions/quick-checks/action.yml

@@ -0,0 +1,120 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Quick Checks
+description: Run the shared compile-free RustFS quality checks.
+
+runs:
+  using: composite
+  steps:
+    - name: Install quality tools
+      uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+      with:
+        tool: |
+          ripgrep@15.2.0
+          shellcheck@0.11.0
+
+    - name: Install actionlint
+      shell: bash
+      run: |
+        actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
+        curl --fail --location --silent --show-error \
+          --output "$actionlint_dir/actionlint.tar.gz" \
+          https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
+        echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8  $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
+        tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
+        rm "$actionlint_dir/actionlint.tar.gz"
+        echo "$actionlint_dir" >> "$GITHUB_PATH"
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+      with:
+        components: rustfmt
+
+    - name: Check workflow syntax and shell scripts
+      shell: bash
+      run: shellcheck --version && actionlint
+
+    - name: Check code formatting
+      shell: bash
+      run: cargo fmt --all --check
+
+    - name: Check unsafe code allowances
+      shell: bash
+      run: ./scripts/check_unsafe_code_allowances.sh
+
+    - name: Check layered dependencies
+      shell: bash
+      run: ./scripts/check_layer_dependencies.sh
+
+    - name: Check architecture migration rules
+      shell: bash
+      run: ./scripts/check_architecture_migration_rules.sh
+
+    - name: Check logging guardrails
+      shell: bash
+      run: ./scripts/check_logging_guardrails.sh
+
+    - name: Check error other(format!) ratchet
+      shell: bash
+      run: ./scripts/check_error_other_format_ratchet.sh
+
+    - name: Check tokio io-uring feature guard
+      shell: bash
+      run: ./scripts/check_no_tokio_io_uring.sh
+
+    - name: Check extension schema boundaries
+      shell: bash
+      run: ./scripts/check_extension_schema_boundaries.sh
+
+    - name: Check body-cache whitelist guard
+      shell: bash
+      run: ./scripts/check_body_cache_whitelist.sh
+
+    - name: Check s3s footprint ratchet
+      shell: bash
+      run: ./scripts/check_s3s_footprint.sh
+
+    - name: Check cryptographic capability wording
+      shell: bash
+      run: ./scripts/check_fips_wording.sh
+
+    - name: Check no embedded secret material
+      shell: bash
+      run: ./scripts/check_embedded_secrets.sh
+
+    - name: Run script contract tests
+      shell: bash
+      run: make script-tests
+
+    - name: Check test wiring
+      shell: bash
+      run: |
+        python3 ./scripts/check_test_wiring.py --self-test
+        python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+        python3 ./scripts/test_security_workflow.py
+        python3 ./scripts/test_nightly_candidate.py
+        python3 ./scripts/check_test_wiring.py
+
+    - name: Check no planning docs committed
+      shell: bash
+      run: ./scripts/check_no_planning_docs.sh
+
+    - name: Check CI paths stay in sync
+      shell: bash
+      run: ./scripts/check_ci_paths_sync.sh
+
+    - name: Check io_uring lane --lib precondition
+      shell: bash
+      run: ./scripts/check_uring_lane_lib_only.sh

.github/pull_request_template.md

@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
 
 ## Summary of Changes
 <!--
-Briefly explain what changed and why reviewers should accept it.
-Focus on behavior, compatibility, and review-relevant context.
+Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
 -->
 
 ## Verification
 <!--
-List the commands or checks you ran, for example:
-- `make pre-commit`
+Give 1–3 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
 
-Use N/A only when verification is not applicable.
+Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
+
+List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
 -->
 
 ## Impact

.github/workflows/ci-docs-only.yml

@@ -12,24 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Companion to ci.yml for required status checks.
-#
-# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
-# requires a check named "Test and Lint" — without this workflow a docs-only PR
-# would wait on it forever. This workflow triggers on exactly the paths ci.yml
-# ignores and reports success under the same job name. Mixed PRs trigger both
-# workflows and the real check still gates: a required check with any failing
-# run blocks the merge.
-# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
-#
-# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
-# required too (rustfs/backlog#1599). Until that change lands this job is
-# inert; mirroring it first is what lets the ruleset change happen without
-# stranding docs-only PRs on a check nobody reports.
-#
-# Keep the paths list below in sync with the pull_request paths-ignore list
-# in ci.yml, and keep the quick-checks steps below byte-identical to the
-# quick-checks job in ci.yml.
+# Reports the existing required checks for paths excluded by ci.yml.
+# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
+# action to keep validation coverage aligned. Keep this paths list in sync with
+# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
 
 name: Continuous Integration (docs only)
 
@@ -59,19 +45,6 @@ permissions:
   contents: read
 
 jobs:
-  # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
-  # check, ci.yml gates every expensive job behind it, so a mixed PR reports
-  # two check runs with this name: the real one (45-51s) and this companion.
-  # GitHub has no written contract for how it picks between same-named
-  # required check runs ("latest wins" vs "any failure blocks"), so instead of
-  # relying on ordering we make both runs execute the same commands against
-  # the same merge ref — their conclusions are then necessarily identical and
-  # the choice does not matter. Keep these steps byte-identical to the
-  # quick-checks job in ci.yml (a guard script that asserts this, and the paths
-  # sync below, is tracked in rustfs/backlog#1603).
-  #
-  # For a genuinely docs-only PR this adds no strictness (no code changed, so
-  # fmt and the guards always pass) and costs ~50s of ubuntu-latest.
   quick-checks:
     name: Quick Checks
     runs-on: ubuntu-latest
@@ -82,63 +55,8 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-        with:
-          tool: ripgrep@15.2.0
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          components: rustfmt
-
-      - name: Check code formatting
-        run: cargo fmt --all --check
-
-      - name: Check unsafe code allowances
-        run: ./scripts/check_unsafe_code_allowances.sh
-
-      - name: Check layered dependencies
-        run: ./scripts/check_layer_dependencies.sh
-
-      - name: Check architecture migration rules
-        run: ./scripts/check_architecture_migration_rules.sh
-
-      - name: Check logging guardrails
-        run: ./scripts/check_logging_guardrails.sh
-
-      - name: Check tokio io-uring feature guard
-        run: ./scripts/check_no_tokio_io_uring.sh
-
-      - name: Check extension schema boundaries
-        run: ./scripts/check_extension_schema_boundaries.sh
-
-      - name: Check body-cache whitelist guard
-        run: ./scripts/check_body_cache_whitelist.sh
-
-      - name: Check s3s footprint ratchet
-        run: ./scripts/check_s3s_footprint.sh
-
-      - name: Check cryptographic capability wording
-        run: ./scripts/check_fips_wording.sh
-
-      - name: Check no embedded secret material
-        run: ./scripts/check_embedded_secrets.sh
-
-      - name: Check test wiring
-        run: |
-          python3 ./scripts/check_test_wiring.py --self-test
-          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-          python3 ./scripts/check_test_wiring.py
-
-      - name: Check no planning docs committed
-        run: ./scripts/check_no_planning_docs.sh
-
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
+      - name: Run shared quick checks
+        uses: ./.github/actions/quick-checks
 
   test-and-lint:
     name: Test and Lint

.github/workflows/ci.yml

@@ -100,12 +100,7 @@ jobs:
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
-  # Fast, compile-free checks that fail early so contributors get feedback in
-  # ~1 minute instead of waiting for the full test job.
-  #
-  # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
-  # PR, which reports two check runs named "Quick Checks", cannot get one red
-  # and one green. Edit both jobs together.
+  # Fail early with compile-free checks shared with docs-only CI.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -117,66 +112,8 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-        with:
-          tool: ripgrep@15.2.0
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          components: rustfmt
-
-      - name: Check code formatting
-        run: cargo fmt --all --check
-
-      - name: Check unsafe code allowances
-        run: ./scripts/check_unsafe_code_allowances.sh
-
-      - name: Check layered dependencies
-        run: ./scripts/check_layer_dependencies.sh
-
-      - name: Check architecture migration rules
-        run: ./scripts/check_architecture_migration_rules.sh
-
-      - name: Check logging guardrails
-        run: ./scripts/check_logging_guardrails.sh
-
-      - name: Check error other(format!) ratchet
-        run: ./scripts/check_error_other_format_ratchet.sh
-
-      - name: Check tokio io-uring feature guard
-        run: ./scripts/check_no_tokio_io_uring.sh
-
-      - name: Check extension schema boundaries
-        run: ./scripts/check_extension_schema_boundaries.sh
-
-      - name: Check body-cache whitelist guard
-        run: ./scripts/check_body_cache_whitelist.sh
-
-      - name: Check s3s footprint ratchet
-        run: ./scripts/check_s3s_footprint.sh
-
-      - name: Check cryptographic capability wording
-        run: ./scripts/check_fips_wording.sh
-
-      - name: Check no embedded secret material
-        run: ./scripts/check_embedded_secrets.sh
-
-      - name: Check test wiring
-        run: |
-          python3 ./scripts/check_test_wiring.py --self-test
-          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-          python3 ./scripts/check_test_wiring.py
-
-      - name: Check no planning docs committed
-        run: ./scripts/check_no_planning_docs.sh
-
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
+      - name: Run shared quick checks
+        uses: ./.github/actions/quick-checks
 
   test-and-lint:
     name: Test and Lint
@@ -269,6 +206,7 @@ jobs:
           CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
         run: |
           mkdir -p artifacts/test-and-lint
+          rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           set +e
@@ -277,6 +215,12 @@ jobs:
             --status-level all --final-status-level all \
             2>&1 | tee artifacts/test-and-lint/nextest.log
           status=${PIPESTATUS[0]}
+          if [[ "${status}" -eq 0 ]]; then
+            cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
+              > artifacts/test-and-lint/core-test-listing.json \
+              && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
+              && test -s target/nextest/ci/junit.xml || status=$?
+          fi
           {
             echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
             echo "exit_status=${status}"

.github/workflows/e2e-upgrade.yml

@@ -19,7 +19,9 @@ on:
     paths:
       - ".github/workflows/e2e-upgrade.yml"
       - "crates/e2e_test/src/common.rs"
+      - "crates/e2e_test/src/fake_s3_target/**"
       - "crates/e2e_test/src/lib.rs"
+      - "crates/e2e_test/src/replication_extension_test.rs"
       - "crates/e2e_test/src/upgrade_compatibility_test.rs"
       - "crates/ecstore/**"
       - "crates/filemeta/**"
@@ -44,9 +46,9 @@ concurrency:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
-  UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
-  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
-  UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+  UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
+  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
+  UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
 
 jobs:
   upgrade:
@@ -55,14 +57,31 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Direct upgrade from rc.2
+          # The two `_from_rc2_` tests keep their names: they assert
+          # release-independent object contracts and pass unchanged against the
+          # newer pinned source, so renaming them would only churn history and
+          # the CI required-check names. UPGRADE_SOURCE_VERSION above is the
+          # single source of truth for which release they actually run against.
+          - name: Direct upgrade from the previous release
             cache_key: e2e-direct-upgrade
             test: direct_upgrade_from_rc2_preserves_object_contracts
             artifact: direct-upgrade
-          - name: Mixed-version rolling upgrade from rc.2
+          - name: Mixed-version rolling upgrade from the previous release
             cache_key: e2e-mixed-version-upgrade
             test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
             artifact: mixed-version-upgrade
+          - name: Bucket configuration survives the upgrade
+            cache_key: e2e-bucket-config-upgrade
+            test: direct_upgrade_from_previous_release_preserves_bucket_configuration
+            artifact: bucket-config-upgrade
+          - name: Rollback reads current bucket metadata
+            cache_key: e2e-bucket-config-rollback
+            test: rollback_to_previous_release_reads_current_bucket_metadata
+            artifact: bucket-config-rollback
+          - name: ODM configuration recovery after rc.5 rollback
+            cache_key: e2e-odm-config-rollback
+            test: rc5_rollback_requires_restoring_odm_configuration
+            artifact: odm-config-rollback
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:

.github/workflows/nightly-gnu.yml

@@ -166,8 +166,9 @@ jobs:
       # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
       # Skipped when the R2 secrets are not configured (artifact-only mode).
       - name: Upload DEB to Cloudflare R2
-        if: env.R2_ACCESS_KEY_ID != ''
+        id: publish
         env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
             exit 0
           fi
 
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y -qq awscli
-          fi
-
           export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          SOURCE_SHA="$(git rev-parse HEAD)"
+          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+            echo "Checkout SHA does not match the nightly build run" >&2
+            exit 1
+          fi
+          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
+          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
+          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
+
+          # Old AWS CLI models lack conditional PutObject support. Never fall
+          # back to an overwriting upload for a candidate.
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
+            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          "${AWS_CLI}" --version
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
+            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
+          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
+            echo "Published candidate checksum does not match the built package" >&2
+            exit 1
+          fi
+
           R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
 
           echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           # Stable "latest" alias so tests can fetch the newest nightly
           # without knowing today's date.
           echo "📤 Uploading latest alias"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
             --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           echo "✅ R2 upload complete"
 
+          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg source_sha "${SOURCE_SHA}" \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
+            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            > "${CANDIDATE_FILE}"
+          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload nightly candidate manifest
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error
+
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-functional-chain.yml

@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -54,14 +54,26 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 480
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -117,7 +129,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -127,7 +139,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -137,13 +149,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -152,27 +161,99 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
+          ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
+          result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
+
+          steps = {}
+          order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
+          version = None
+          version_node = None
+          verdict = None
+          verdict_detail = ''
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = step_re.match(line)
+                      if m:
+                          n, desc, status = m.group(1), m.group(2), m.group(3)
+                          if n not in steps:
+                              order.append(n)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
+                          continue
+                      m = ver_re.match(line)
+                      if m:
+                          version, version_node = m.group(1), m.group(2)
+                          continue
+                      m = result_re.match(line)
+                      if m and verdict != 'FAIL':
+                          verdict, verdict_detail = m.group(1), m.group(2)
+          except FileNotFoundError:
+              pass
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Step Results\n\n')
+              if version:
+                  node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
+                  out.write(f'- Version under test: **{version}**{node_note}\n')
+              if verdict:
+                  out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
+              out.write('\n')
+              out.write('| Step | Description | Result |\n')
+              out.write('| --- | --- | --- |\n')
+              for n in sorted(order, key=int):
+                  desc, status = steps[n]
+                  out.write(f'| {n} | {desc} | {status} |\n')
+              if not order:
+                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
+          PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${STEPS_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -182,28 +263,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -231,14 +316,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -254,14 +341,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-heal-test*.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}

.github/workflows/rustfs-kms-test.yml

@@ -49,10 +49,28 @@ env:
 jobs:
   kms-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -109,9 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -141,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -156,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -238,28 +214,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -287,14 +267,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -310,14 +292,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-kms.log
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-performance-test.yml

@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -76,22 +75,33 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
   performance-test:
     runs-on: pf-testing
-    # Requirement: a failing benchmark must not fail the workflow;
-    # failures are filed to rustfs/backlog.
-    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -123,7 +133,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -133,7 +143,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -143,7 +153,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -156,17 +166,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -186,7 +194,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -194,7 +201,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -202,6 +209,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -211,8 +220,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          } > "${REPORT_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -231,11 +240,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -263,14 +271,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -286,20 +296,26 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-perf-test*.log
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+          if-no-files-found: error
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-pool-expand-test.yml

@@ -76,9 +76,6 @@ jobs:
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
@@ -380,6 +377,60 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
+          STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md"
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
+          ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
+          result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
+
+          steps = {}
+          order = []
+          version = None
+          version_node = None
+          verdict = None
+          verdict_detail = ''
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = step_re.match(line)
+                      if m:
+                          n, desc, status = m.group(1), m.group(2), m.group(3)
+                          if n not in steps:
+                              order.append(n)
+                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          continue
+                      m = ver_re.match(line)
+                      if m:
+                          version, version_node = m.group(1), m.group(2)
+                          continue
+                      m = result_re.match(line)
+                      if m:
+                          verdict, verdict_detail = m.group(1), m.group(2)
+          except FileNotFoundError:
+              pass
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Step Results\n\n')
+              if version:
+                  node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
+                  out.write(f'- Version under test: **{version}**{node_note}\n')
+              if verdict:
+                  out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
+              out.write('\n')
+              out.write('| Step | Description | Result |\n')
+              out.write('| --- | --- | --- |\n')
+              for n in sorted(order, key=int):
+                  desc, status = steps[n]
+                  out.write(f'| {n} | {desc} | {status} |\n')
+              if not order:
+                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          PY
           {
             echo "# RustFS pool expansion test report"
             echo ""
@@ -389,6 +440,8 @@ jobs:
             echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
             echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
             echo ""
+            cat "${STEPS_TABLE}" || true
+            echo ""
             echo "## Log tail"
             echo '```text'
             tail -n 200 "${LOG_FILE}" || true
@@ -486,17 +539,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}

.github/workflows/rustfs-replication-test.yml

@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -62,12 +61,28 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -116,9 +131,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -141,10 +153,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -166,80 +175,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -249,28 +222,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -298,14 +275,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -321,14 +300,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
@@ -349,13 +329,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-s3-compat-test.yml

@@ -37,10 +37,28 @@ env:
 jobs:
   s3-compat-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -88,9 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -107,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -132,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -218,28 +191,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -267,14 +244,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -290,14 +269,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-s3-compat.log
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-security-test.yml

@@ -74,10 +74,27 @@ env:
 jobs:
   security-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      # Checkout the repository into its own subdirectory. Checking out at
+      # the workspace root would wipe the auto-testing clone above (that is
+      # exactly how run 33934141181 lost rustfs-security-test.sh).
+      - name: Checkout repository (for the OIDC live gate script)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+          path: rustfs-repo
+
+      - name: Initialize security evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
+          printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -98,11 +115,6 @@ jobs:
           echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
           exit 1
 
-      - name: Checkout repository (for the OIDC live gate script)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
       - name: Show environment
         run: |
           uname -a
@@ -135,8 +147,9 @@ jobs:
         id: test
         continue-on-error: true
         env:
-          REPORT_FILE: /tmp/rustfs-security-report.md
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
+          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-security-test.sh
@@ -159,29 +172,48 @@ jobs:
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
-          ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
 
       - name: Generate report
-        if: always()
+        id: report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        env:
+          TEST_OUTCOME: ${{ steps.test.outcome }}
         run: |
           set -euo pipefail
-          if [ ! -f /tmp/rustfs-security-report.md ]; then
-            {
-              echo "# RustFS security test report"
-              echo ""
-              echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-              echo "- Trigger: ${{ github.event_name }}"
-              echo "- Test Step Outcome: failure (suite did not produce a report)"
-            } > /tmp/rustfs-security-report.md
+          RESULT=failure
+          if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
+            RESULT=success
           fi
-          cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
+          {
+            echo "# RustFS security test report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${TEST_OUTCOME}"
+            echo ""
+            # The dashboard prioritizes case rows over the step outcome.
+            # Keep partial case results in the artifact when the suite fails.
+            if [ "${RESULT}" = "success" ]; then
+              cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
+            elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
+              echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
+            else
+              echo "The suite did not produce a non-empty report."
+            fi
+          } > "${SECURITY_ARTIFACTS_DIR}/report.md"
+          cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-security-report.md
+          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
           SUITE: security
         run: |
           set -euo pipefail
@@ -191,17 +223,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -210,8 +247,9 @@ jobs:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'security'
           SUITE_LABEL: 'Security'
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-security-report.md'
+          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
           LOG_FILE: ''
         run: |
           set -euo pipefail
@@ -245,7 +283,7 @@ jobs:
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
             elif [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
@@ -263,14 +301,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-security-test-${{ github.run_id }}
+          name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-security-report.md
-            /tmp/rustfs-security.*/*
-          if-no-files-found: ignore
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)

.github/workflows/rustfs-storage-test.yml

@@ -46,10 +46,28 @@ env:
 jobs:
   storage-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -97,9 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -122,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -147,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -233,28 +206,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -282,14 +259,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -305,14 +284,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-tier-test.yml

@@ -61,9 +61,6 @@ env:
 jobs:
   tier-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
@@ -380,17 +377,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: Verify required tier evidence
         id: evidence_verify

.github/workflows/rustfs-upgrade-test.yml

@@ -18,15 +18,15 @@ on:
   workflow_dispatch:
     inputs:
       from_version:
-        description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: '1.0.0-rc.3'
       from_url:
         description: 'OLD .deb URL. Overrides from_version.'
         required: false
         type: string
       to_version:
-        description: 'NEW RustFS release tag (leave empty for latest nightly)'
+        description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
         required: false
       to_url:
         description: 'NEW .deb URL. Overrides to_version / nightly default.'
@@ -79,10 +79,28 @@ env:
 jobs:
   upgrade-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,9 +160,8 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
-        continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-upgrade-test.sh
@@ -175,13 +192,33 @@ jobs:
           else
             ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
+          # Fail fast with a clear message when a requested release tag has
+          # no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
+          # letting the suite die mid-run on a 404.
+          check_release_asset() {
+            local version="$1" tag asset url
+            [ -n "${version}" ] && [ "${version}" != "null" ] || return 0
+            tag="${version#v}"
+            asset="rustfs_${tag//-/.}_amd64.deb"
+            url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
+            if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
+              echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
+              echo "       ${url}" >&2
+              echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
+              exit 1
+            fi
+            echo "resolved ${tag} -> ${url}"
+          }
+          if [ -z "${FROM_URL}" ]; then
+            check_release_asset "${FROM_VERSION}"
+          fi
+          if [ -z "${TO_URL}" ]; then
+            check_release_asset "${TO_VERSION}"
+          fi
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -202,80 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${MATRIX_TABLE}"
+              echo ""
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -285,28 +289,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -334,14 +342,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -357,14 +367,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)

.github/workflows/scheduled-validation-freshness.yml

@@ -42,6 +42,7 @@ jobs:
       - name: Check latest scheduled runs
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
         run: |
           set +e
           python3 scripts/check_scheduled_validation_freshness.py \

.gitignore

@@ -33,6 +33,7 @@ profile.json
 *.zst
 .secrets
 *.go
+!crates/zip/tests/fixtures/snowball/**/generate/*.go
 *.pb
 *.svg
 deploy/logs/*.log.*

.pre-commit-config.yaml

@@ -3,9 +3,9 @@
 repos:
   - repo: local
     hooks:
-      - id: rustfs-dev-check
-        name: rustfs dev-check
-        entry: make dev-check
+      - id: rustfs-fmt-check
+        name: Rust formatting
+        entry: cargo fmt --all --check
         language: system
         types: [rust]
         pass_filenames: false

CHANGELOG.md

@@ -18,7 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Read paths: an object at or below `policy.inline_max_bytes` (16 MiB by default) is teed to the client and to the local store in a single source read; a larger object or a Range read streams through and a background pull stores the whole object. A HEAD miss is proxied to the source and stores nothing (`policy.head = local_only` disables it). Every source-backed response carries `x-rustfs-on-demand-migration: source`
   - Protections: a per-source circuit breaker, a per-key negative cache, singleflight per key, a concurrency limit and a bounded pull queue shared by the inline and background paths, an optional bandwidth limit, an anti-loop request marker, and the shared outbound-endpoint (SSRF) policy
   - Metrics under `rustfs_on_demand_migration_*` (`requests_total`, `pulled_bytes_total`, `pulled_objects_total`, `pull_failures_total`, `inflight_pulls`, `queue_depth`, `source_latency_seconds_*`, `breaker_state`), mirrored per node by the admin status route
-  - Limitations: listings show only local objects (the source is not merged into `ListObjectsV2`); PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
+  - Listings: `ListObjects` v1 remains local with ordinary key markers. `ListObjectsV2` can merge source objects when `policy.list_through = true`; this is off by default
+  - Upgrade and rollback: finish upgrading every node before enabling ODM. An rc.5 node that writes bucket configuration drops the ODM fields from metadata; neither a later restart nor moving the service out of ECStore recovers them. Before rollback, disable ODM and securely retain the original full configuration and credentials. After every node returns to a compatible version, restore and validate that configuration. Redacted exports cannot replace the credential backup; source-only objects are unavailable through RustFS while ODM is disabled. See the upgrade and rollback section of `docs/operations/on-demand-migration.md`
+  - Optional Google dependencies: default and `full` server builds retain native GCS support. `cargo build -p rustfs --no-default-features --features ftps,webdav` excludes Google SDKs while preserving configuration decoding and redaction; native GCS ODM and tier operations require the `gcs` feature. Do not use that build with existing GCS-tiered data
+  - Limitations: PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
 - **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
   - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
   - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window

CONTRIBUTING.md

@@ -109,24 +109,17 @@ affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
-Git hooks are **not** versioned in this repository, so a fresh clone has no
-active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
-choice is a one-liner that runs `make pre-commit`), you can mark it executable
-with:
+The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
 
 ```bash
 make setup-hooks
 ```
 
-Or manually:
+The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
 
-```bash
-chmod +x .git/hooks/pre-commit
-```
+`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
 
-With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
-applicable scoped checks, and reserve `make pre-pr` for broad cross-module
-changes whose impact cannot be bounded by those checks.
+A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
 
 ### 📝 Formatting Configuration
 
@@ -138,31 +131,11 @@ fn_call_width = 90
 single_line_let_else_max_width = 100
 ```
 
-### 🚫 Commit Prevention
-
-If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
-
-1. **Block the commit** and show clear error messages
-2. **Provide exact commands** to fix the issues
-3. **Guide you through** the resolution process
-
-Example output when formatting fails:
-
-```
-❌ Code formatting check failed!
-💡 Please run 'cargo fmt --all' to format your code before committing.
-
-🔧 Quick fix:
-   cargo fmt --all
-   git add .
-   git commit
-```
-
 ### 🔄 Development Workflow
 
 1. **Make your changes**
 2. **Format your code**: `make fmt` or `cargo fmt --all`
-3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
+3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
 6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -206,11 +179,12 @@ Configure your IDE to:
 #### Pre-commit hook not running?
 
 ```bash
-# Check if hook is executable
-ls -la .git/hooks/pre-commit
-
-# Make it executable if needed
-chmod +x .git/hooks/pre-commit
+pre-commit validate-config
+pre-commit run --all-files
+# Inspect any configured hook manager; do not overwrite it.
+git config --get core.hooksPath
+# Install if no separate hook manager is configured.
+make setup-hooks
 ```
 
 #### Formatting issues?

Cargo.toml

@@ -165,10 +165,10 @@ http-body = "1.1.0"
 http-body-util = "0.1.5"
 minlz = "1.2.3"
 reqwest = "0.13.4"
-rustfs-kafka-async = { version = "1.2.0" }
+rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
-tokio-rustls = { default-features = false, version = "0.26.4" }
+tokio-rustls = { default-features = false, version = "0.26.5" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.19" }
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.1" }
-argon2 = { version = "=0.6.0" }
-blake2 = "=0.11.0"
-chacha20poly1305 = { version = "=0.11.0" }
+aes-gcm = { version = "0.11.1" }
+argon2 = { version = "0.6.0" }
+blake2 = "0.11.0"
+chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.0.0" }
@@ -234,18 +234,22 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
 astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
+# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
+tar-codec = "0.0.14"
+tar-framing = "0.0.14"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.11.0" }
+aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.117.0" }
-aws-sdk-s3 = { default-features = false, version = "1.144.0" }
-aws-sdk-sts = { default-features = false, version = "1.113.0" }
+aws-sdk-kms = { default-features = false, version = "1.118.0" }
+aws-sdk-s3 = { default-features = false, version = "1.145.0" }
+aws-sdk-sts = { default-features = false, version = "1.114.0" }
+aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
-aws-smithy-runtime-api = { version = "1.15.0" }
-aws-smithy-types = { version = "1.6.2" }
+aws-smithy-runtime-api = { version = "1.16.0" }
+aws-smithy-types = { version = "1.6.3" }
 base64-simd = "0.8.0"
 brotli = "9.0.0"
 clap = { version = "4.6.6" }
@@ -339,7 +343,7 @@ windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.18" }
 zip = "8.6.0"
-zstd = "0.13.3"
+zstd = "0.14.0"
 
 # Observability and Metrics
 metrics = "0.24.6"
@@ -359,15 +363,16 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.63.1" }
+russh = { version = "0.63.2" }
 russh-sftp = "2.4.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.5.2" }
-hotpath = { version = "0.24.0", default-features = false }
+rustfs-mimalloc = { version = "0.5.3" }
+# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
+hotpath = { version = "=0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

crates/config/README.md

@@ -130,6 +130,21 @@ Scanner cycle budget controls:
   - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
   - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
 
+## Remote tier timeout environment variables
+
+- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
+  - remote tier TCP connect timeout.
+  - default is `10`.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
+- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
+  - remote tier request timeout through response headers.
+  - default is `86400` so large transition uploads keep a production-safe budget.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
+- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
+  - maximum idle time between remote tier response-body chunks.
+  - default is `60`; the timer resets only when non-empty body data keeps progressing.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
+
 ## Drive timeout environment variables
 
 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`

crates/config/src/constants/object.rs

@@ -137,6 +137,28 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
 const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
 
+/// Environment variable for remote tier TCP connect timeout in seconds.
+pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
+/// Default remote tier TCP connect timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+/// Environment variable for the remote tier request timeout in seconds.
+///
+/// This bounds upload/download request progress through response headers. The
+/// default is intentionally large so multi-TiB transition uploads keep their
+/// previous production budget while black-hole remotes no longer wait forever.
+pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
+/// Default remote tier request timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
+
+/// Environment variable for remote tier response-body idle timeout in seconds.
+///
+/// The timer is re-armed on every non-empty response-body chunk, so slow but
+/// progressing remotes can continue while silent response bodies are cancelled.
+pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
+/// Default remote tier response-body idle timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
+
 /// Request the object-transaction fencing contract used by storage-owned
 /// cleanup receipts and lock-window optimizations.
 ///
@@ -812,6 +834,16 @@ mod remote_version_state_tests {
         );
     }
 
+    #[test]
+    fn remote_tier_timeout_env_names_are_stable() {
+        assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
+        assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
+        assert_eq!(
+            super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+            "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
+        );
+    }
+
     #[test]
     fn data_movement_part_checksum_gate_uses_stable_environment_names() {
         assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");

crates/e2e_test/src/on_demand_migration/concurrency_test.rs

@@ -20,9 +20,10 @@
 //! journal (`count_requests`) carries the assertion in every one of them.
 
 use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
-use crate::fake_s3_target::Operation;
+use crate::fake_s3_target::{FaultAction, Operation};
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use bytes::Bytes;
+use futures::{StreamExt, TryStreamExt};
 use std::time::Duration;
 
 type TestResult = Result<(), BoxError>;
@@ -145,14 +146,38 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
     .await?;
 
     let body = payload(128 * 1024);
+    let blocker = "queue/blocker.bin";
+    env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
+    // The one-chunk range completes immediately; its full background pull
+    // occupies the only slot while the remaining requests fill the queue.
+    env.source.inject_for_key(
+        Operation::GetObject,
+        blocker,
+        FaultAction::SlowSendBody {
+            chunk_bytes: 1024,
+            delay: Duration::from_millis(100),
+        },
+        2,
+    );
+    let response = env
+        .raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
+        .await?;
+    assert_eq!(response.status, 206);
+    assert_eq!(response.body, body.slice(0..1024));
+    env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
+
     let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
     let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
     env.seed_source(SOURCE_BUCKET, &seeds);
 
-    let responses: Vec<RawResponse> = futures::future::try_join_all(
+    // Bound source connections below the fixture's limit while still
+    // submitting all 100 requests to the eight-slot background queue.
+    let responses: Vec<RawResponse> = futures::stream::iter(
         keys.iter()
             .map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
     )
+    .buffered(16)
+    .try_collect()
     .await?;
     for (key, response) in keys.iter().zip(&responses) {
         assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
@@ -168,6 +193,15 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         .wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
         .await?;
     assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
+    let queue_full = usize::try_from(queue_full)?;
+    assert!(queue_full <= REQUESTS);
+    env.wait_for_status_counter(
+        bucket,
+        "/counters/pulled_objects_total/background",
+        u64::try_from(REQUESTS + 1 - queue_full)?,
+        SETTLE,
+    )
+    .await?;
 
     let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
     assert!(
@@ -175,9 +209,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         "every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
     );
     let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
-    assert!(
-        dropped > 0,
-        "the overflowed keys are the ones with no backfill GET, but every key got one"
-    );
+    assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
     Ok(())
 }

crates/e2e_test/src/on_demand_migration/list_through_test.rs

@@ -265,16 +265,13 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
     let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
     assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
 
-    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
-    let rejected = env
-        .raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
-        .await?;
-    assert_eq!(
-        rejected.status,
-        400,
-        "a bumped token version is a client error: {}",
-        String::from_utf8_lossy(&rejected.body)
-    );
+    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
+    assert_ne!(tampered, token, "the test must change the token version");
+    let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
+    let rejected = env.raw_list_objects_v2(bucket, &query).await?;
+    let error_body = String::from_utf8_lossy(&rejected.body);
+    assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
+    assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
     Ok(())
 }
 

crates/e2e_test/src/replication_extension_test.rs

@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
     Ok(())
 }
 
+#[tokio::test]
+async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
+    init_logging();
+
+    // Keep compilation outside the scenario timeout. Recovery itself waits
+    // for the production 30-second lightweight retry tick.
+    let _rustfs_binary = rustfs_binary_path();
+
+    match timeout(Duration::from_secs(150), async {
+        let mut site_env = replication_fast_env();
+        site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+
+        let mut site_a_env = RustFSTestEnvironment::new().await?;
+        site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut site_b_env = RustFSTestEnvironment::new().await?;
+        site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
+
+        let site_a_client = site_a_env.create_s3_client();
+        let site_b_client = site_b_env.create_s3_client();
+        let bucket = "site-repl-peer-outage";
+        let key = "after-recovery.txt";
+        let payload = b"site replication recovered the missed bucket".to_vec();
+
+        let add_status = site_replication_add(
+            &site_a_env,
+            &[
+                PeerSite {
+                    name: "outage-site-a".to_string(),
+                    endpoint: site_a_env.url.clone(),
+                    access_key: site_a_env.access_key.clone(),
+                    secret_key: site_a_env.secret_key.clone(),
+                    ..Default::default()
+                },
+                PeerSite {
+                    name: "outage-site-b".to_string(),
+                    endpoint: site_b_env.url.clone(),
+                    access_key: site_b_env.access_key.clone(),
+                    secret_key: site_b_env.secret_key.clone(),
+                    ..Default::default()
+                },
+            ],
+        )
+        .await?;
+        assert!(add_status.success, "unexpected site add result: {add_status:?}");
+        wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        site_b_env.stop_server();
+        site_a_client.create_bucket().bucket(bucket).send().await?;
+        site_a_client.head_bucket().bucket(bucket).send().await?;
+
+        let queued = site_replication_info(&site_a_env)
+            .await?
+            .retry_stats
+            .ok_or("peer outage did not persist a site replication retry event")?;
+        assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
+
+        site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
+        let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
+        loop {
+            let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
+            let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
+            if bucket_recovered && queue_empty {
+                break;
+            }
+            if tokio::time::Instant::now() >= recovery_deadline {
+                return Err(format!(
+                    "site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
+                )
+                .into());
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        site_a_client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from(payload.clone()))
+            .send()
+            .await?;
+        assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
+
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
+    }
+}
+
 /// Re-applying a site's own replication config must not disable the peer's reverse direction.
 ///
 /// `PutBucketReplication` broadcasts the config to every peer — the console's replication

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,21 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use crate::common::{
+    RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
+};
+use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
+use crate::replication_extension_test::{
+    LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
+    BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
+    ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
+    ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
+    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
 };
+use http::{Method, StatusCode};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use tokio::task::JoinSet;
 use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -40,6 +53,32 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
 // comfortably covers that window plus CI scheduling jitter.
 const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
+// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
+const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
+const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
+const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
+const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
+const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
+const ROLLBACK_BUCKET: &str = "rollback-config-data";
+const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
+const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
+const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
+const LIFECYCLE_PREFIX: &str = "logs/";
+const LIFECYCLE_DAYS: i32 = 30;
+const BUCKET_TAG_KEY: &str = "owner";
+const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
+const OBJECT_LOCK_DAYS: i32 = 1;
+// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
+// authoritative; the quota test uses the same 30s budget.
+const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
+// Quota admission fails closed while a freshly started server has neither
+// authoritative usage nor a persisted degraded baseline for the bucket
+// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
+// window. It is a restart property, not an upgrade property — the same window
+// opens on the very first start — so the write assertions ride it out instead
+// of treating it as an upgrade failure.
+const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
+
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
         .map(PathBuf::from)
@@ -240,6 +279,93 @@ async fn exercise_mixed_cluster(
     Ok(())
 }
 
+/// Pins the published old writer's limitation and the supported recovery
+/// procedure. This is not a promise that mixed-version ODM is supported.
+/// Replace the loss assertion when ODM gains independent persistence;
+/// preserving configuration across rc.5 writes is then an improvement.
+#[tokio::test]
+#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
+async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let version = tokio::process::Command::new(&previous_binary)
+        .arg("--version")
+        .output()
+        .await?;
+    assert!(version.status.success(), "previous binary must report its version");
+    assert!(
+        String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
+        "this compatibility scenario requires the published rc.5 writer"
+    );
+    let mut env = OdmTestEnv::start().await?;
+    let bucket = "odm-rc5-rollback";
+    let source_bucket = "odm-rc5-source";
+    env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
+    env.seed_source(
+        source_bucket,
+        &[SeedObject::new(
+            "source-only",
+            bytes::Bytes::from_static(b"source read after recovery"),
+        )],
+    );
+    env.rustfs.create_test_bucket(bucket).await?;
+    let saved_config = env.fake_source_spec(source_bucket);
+    assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
+    let before = env.get_config(bucket).await?;
+    assert_eq!(before.status, 200);
+    let expected_config = before
+        .json()?
+        .get("config")
+        .cloned()
+        .ok_or("configuration response omitted config")?;
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key("local")
+        .body(ByteStream::from_static(b"local data survives rollback"))
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restarted = env.get_config(bucket).await?;
+    assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
+    assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
+
+    restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(
+            Tagging::builder()
+                .tag_set(Tag::builder().key("writer").value("rc5").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let missing = env.get_config(bucket).await?;
+    assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
+    assert!(missing.body.contains("NoSuchConfiguration"));
+    assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
+
+    assert_eq!(
+        env.configure_source(bucket, &saved_config).await?.status,
+        200,
+        "restore from saved full configuration"
+    );
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restored = env.get_config(bucket).await?;
+    assert_eq!(restored.status, 200, "restored ODM configuration persists");
+    assert_eq!(restored.json()?.get("config"), Some(&expected_config));
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        read_object(&env.client, bucket, "source-only", None).await?.1,
+        b"source read after recovery"
+    );
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -429,3 +555,653 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
 
     Ok(())
 }
+
+/// Child-process environment shared by both bucket-configuration scenarios.
+///
+/// The replication target is an in-process fake bound to `127.0.0.1`, which
+/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
+/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
+/// server's outbound health check.
+fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
+    let mut env = vec![
+        (SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
+        ("NO_PROXY", "127.0.0.1,localhost"),
+        ("HTTP_PROXY", ""),
+        ("HTTPS_PROXY", ""),
+        // Shorten the scanner cycle so the bucket's usage becomes authoritative
+        // in seconds; both `set-bucket-quota` and quota admission block on it.
+        ("RUSTFS_SCANNER_CYCLE", "1"),
+        ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
+    ];
+    env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env.extend(replication_fast_env());
+    env
+}
+
+/// Restart `env` in place on the same data directory using an explicit binary.
+///
+/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
+/// the workspace build, which is the upgrade direction only. The rollback
+/// scenario needs the reverse: stop the current build and bring the pinned
+/// previous release up on the metadata that build just wrote.
+async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
+    env.stop_server();
+    env.start_rustfs_server_from_binary(binary, vec![], server_env).await
+}
+
+async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
+    let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
+    loop {
+        let (status, response) =
+            admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
+        if status.is_success() {
+            return Ok(());
+        }
+        if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
+            return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
+/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
+///
+/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
+/// never ends, is a genuine regression and surfaces as an error.
+async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
+    let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
+    loop {
+        let result = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(body))
+            .send()
+            .await;
+        let error = match result {
+            Ok(_) => return Ok(()),
+            Err(error) => error,
+        };
+        let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
+        if !retryable || Instant::now() >= deadline {
+            return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
+    }
+    let quota: serde_json::Value = serde_json::from_str(&response)?;
+    Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
+}
+
+/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
+///
+/// Returns an error for any non-200, because rustfs#7172 made this endpoint
+/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
+/// An upgrade that misreads a blob written by the previous release therefore
+/// shows up here as an error, and a silently dropped target shows up as an
+/// empty list — the caller must distinguish the two.
+async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
+    let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
+    }
+    Ok(serde_json::from_str(&response)?)
+}
+
+/// Assert that `bucket` still carries exactly the replication target `arn`.
+async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
+    let targets = list_remote_targets(env, bucket).await?;
+    assert_eq!(
+        targets.len(),
+        1,
+        "{context}: list-remote-targets must still report the single configured target, got {targets:?}"
+    );
+    assert_eq!(
+        targets[0].get("arn").and_then(serde_json::Value::as_str),
+        Some(arn),
+        "{context}: the target ARN changed across the restart: {targets:?}"
+    );
+    Ok(())
+}
+
+/// Configure a replication target on `bucket` pointing at the in-process fake,
+/// then attach an enabled replication rule for it. Returns the target ARN.
+async fn configure_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target: &FakeS3Target,
+    target_bucket: &str,
+) -> Result<String, BoxError> {
+    let arn = set_replication_target_with_options(
+        env,
+        bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(env, bucket, &arn).await?;
+    Ok(arn)
+}
+
+async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
+    let configuration = ServerSideEncryptionConfiguration::builder()
+        .rules(
+            ServerSideEncryptionRule::builder()
+                .apply_server_side_encryption_by_default(
+                    ServerSideEncryptionByDefault::builder()
+                        .sse_algorithm(ServerSideEncryption::Aes256)
+                        .build()?,
+                )
+                .build(),
+        )
+        .build()?;
+    client
+        .put_bucket_encryption()
+        .bucket(bucket)
+        .server_side_encryption_configuration(configuration)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let response = client.get_bucket_encryption().bucket(bucket).send().await?;
+    let rules = response
+        .server_side_encryption_configuration()
+        .ok_or("GetBucketEncryption omitted the configuration")?
+        .rules();
+    assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
+    assert_eq!(
+        rules[0]
+            .apply_server_side_encryption_by_default()
+            .map(ServerSideEncryptionByDefault::sse_algorithm),
+        Some(&ServerSideEncryption::Aes256),
+        "{context}: the default encryption algorithm changed"
+    );
+    Ok(())
+}
+
+async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
+    let tagging = Tagging::builder()
+        .tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
+        .build()?;
+    client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
+    Ok(())
+}
+
+async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
+    let tag_set = tags.tag_set();
+    assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
+    assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
+    assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
+    Ok(())
+}
+
+async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
+    assert_eq!(
+        versioning.status(),
+        Some(&BucketVersioningStatus::Enabled),
+        "{context}: versioning is no longer Enabled on {bucket}"
+    );
+    Ok(())
+}
+
+fn bucket_policy_document(bucket: &str) -> serde_json::Value {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Sid": "UpgradePublicRead",
+            "Effect": "Allow",
+            "Principal": { "AWS": ["*"] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
+        }]
+    })
+}
+
+/// `GET .../on-demand-migration/{bucket}/status`.
+///
+/// The migration module defaults on from rustfs#7089, so a bucket that never
+/// configured a source must still answer `configured: false` rather than
+/// engaging the migration path.
+async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
+    let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
+    );
+    let body: serde_json::Value = serde_json::from_str(&response)?;
+    assert_eq!(
+        body.get("configured"),
+        Some(&serde_json::Value::Bool(false)),
+        "a bucket upgraded from the previous release must not look migration-configured: {body}"
+    );
+    Ok(())
+}
+
+/// A GET for a key that was never written must be a plain `NoSuchKey`.
+///
+/// With the migration module on by default this is the cheap proof that an
+/// unconfigured bucket never consults a source: any migration engagement would
+/// surface as a different status or error code here.
+async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
+    let error = client
+        .get_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("a key that was never written must not be readable");
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "a missing key must stay a 404 on a bucket with no migration configuration"
+    );
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("NoSuchKey"),
+        "a missing key must stay NoSuchKey on a bucket with no migration configuration"
+    );
+    Ok(())
+}
+
+/// Bucket configuration written by the pinned previous release must survive an
+/// upgrade to the current build unchanged, and must keep working.
+///
+/// This pins the three on-disk surfaces the on-demand-migration series moved:
+///
+/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
+///   every configuration read below decodes a 44-key blob on 46-key code.
+/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
+///   public-access-block / quota blob "present but unreadable" instead of
+///   silently defaulting, and made `list-remote-targets` fail closed on it. A
+///   replication target configured by the old release must therefore still be
+///   *listed*, not dropped and not an error.
+/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
+///   encryption configuration cannot be read, so a misparsed SSE config would
+///   turn every PUT to that bucket into a 500.
+///
+/// Not covered on purpose: on-demand-migration configuration itself, which the
+/// previous release has no public API for — the reverse direction is asserted
+/// instead (an upgraded bucket reports `configured: false`).
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    // In-process: the fake target outlives both server processes, so the
+    // replication target stays reachable across the upgrade.
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
+        .await?;
+    let old_client = env.create_s3_client();
+
+    env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
+    env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
+    env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
+    old_client
+        .create_bucket()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    // Plain bucket: policy, tags, lifecycle, quota.
+    let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
+    old_client
+        .put_bucket_policy()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .policy(policy.to_string())
+        .send()
+        .await?;
+    put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
+    old_client
+        .put_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .lifecycle_configuration(
+            BucketLifecycleConfiguration::builder()
+                .rules(
+                    LifecycleRule::builder()
+                        .id(LIFECYCLE_RULE_ID)
+                        .status(ExpirationStatus::Enabled)
+                        .filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
+                        .expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
+                        .build()?,
+                )
+                .build()?,
+        )
+        .send()
+        .await?;
+    set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
+
+    // Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
+    // public access block, both of which rustfs#7172 now fails closed on.
+    put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
+    old_client
+        .put_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .block_public_acls(true)
+                .ignore_public_acls(true)
+                .block_public_policy(true)
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // Replicated bucket: versioning, a validated remote target, a rule.
+    enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
+    let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
+
+    // Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
+    old_client
+        .put_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_configuration(
+            ObjectLockConfiguration::builder()
+                .object_lock_enabled(ObjectLockEnabled::Enabled)
+                .rule(
+                    ObjectLockRule::builder()
+                        .default_retention(
+                            DefaultRetention::builder()
+                                .mode(ObjectLockRetentionMode::Governance)
+                                .days(OBJECT_LOCK_DAYS)
+                                .build(),
+                        )
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let plain_key = "plain/written-by-previous";
+    let plain_bytes = b"plain object written by the previous RustFS release";
+    put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
+
+    let encrypted_key = "encrypted/written-by-previous";
+    let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
+    old_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(encrypted_key)
+        .body(ByteStream::from_static(encrypted_bytes))
+        .send()
+        .await?;
+    assert_eq!(
+        read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
+            .await?
+            .0,
+        Some(ServerSideEncryption::Aes256),
+        "the previous release must apply the bucket default encryption it just accepted"
+    );
+
+    // The multipart object lives in the default-encrypted bucket so the
+    // upgraded build has to reassemble parts *and* re-derive the object key.
+    let multipart_key = "encrypted/multipart-written-by-previous";
+    let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
+
+    let versioned_key = "versioned/written-by-previous";
+    let versioned_bytes = b"versioned object written by the previous RustFS release";
+    let versioned_id = old_client
+        .put_object()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .key(versioned_key)
+        .body(ByteStream::from_static(versioned_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    env.restart_server_preserving_data(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    // Every configuration must read back unchanged on the upgraded build.
+    let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
+    let upgraded_policy: serde_json::Value =
+        serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
+    assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
+    assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
+
+    let lifecycle = new_client
+        .get_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .send()
+        .await?;
+    let rules = lifecycle.rules();
+    assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
+    assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
+    assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
+    assert_eq!(
+        rules[0].expiration().and_then(LifecycleExpiration::days),
+        Some(LIFECYCLE_DAYS),
+        "the lifecycle expiration changed across the upgrade"
+    );
+
+    assert_eq!(
+        get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
+        Some(BUCKET_QUOTA_BYTES),
+        "the bucket quota changed across the upgrade"
+    );
+
+    assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
+    let public_access_block = new_client
+        .get_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .send()
+        .await?;
+    let public_access_block = public_access_block
+        .public_access_block_configuration()
+        .ok_or("GetPublicAccessBlock omitted the configuration")?;
+    assert_eq!(public_access_block.block_public_acls(), Some(true));
+    assert_eq!(public_access_block.ignore_public_acls(), Some(true));
+    assert_eq!(public_access_block.block_public_policy(), Some(true));
+    assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
+
+    assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
+    // rustfs#7172: neither an empty list nor an error is acceptable here.
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
+    let replication = new_client
+        .get_bucket_replication()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .send()
+        .await?;
+    let replication_rules = replication
+        .replication_configuration()
+        .ok_or("GetBucketReplication omitted the configuration")?
+        .rules();
+    assert_eq!(
+        replication_rules.len(),
+        1,
+        "the replication rule count changed across the upgrade: {replication_rules:?}"
+    );
+    assert_eq!(
+        replication_rules[0].destination().map(|destination| destination.bucket()),
+        Some(target_arn.as_str()),
+        "the replication rule no longer points at the configured target"
+    );
+
+    let object_lock = new_client
+        .get_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .send()
+        .await?;
+    let object_lock = object_lock
+        .object_lock_configuration()
+        .ok_or("GetObjectLockConfiguration omitted the configuration")?;
+    assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
+    let retention = object_lock
+        .rule()
+        .and_then(ObjectLockRule::default_retention)
+        .ok_or("the object lock configuration lost its default retention")?;
+    assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
+    assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
+
+    // rustfs#7183: a PUT into the default-encrypted bucket must still succeed
+    // and still come back encrypted.
+    let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
+    let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
+    new_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(post_upgrade_encrypted_key)
+        .body(ByteStream::from_static(post_upgrade_encrypted_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "a PUT after the upgrade lost the bucket default encryption"
+    );
+    assert_eq!(body, post_upgrade_encrypted_bytes);
+
+    let post_upgrade_plain_key = "plain/written-after-upgrade";
+    let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
+    put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
+    assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
+    assert_eq!(body, post_upgrade_plain_bytes);
+
+    // Every object written by the previous release reads back byte-identical.
+    assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, encrypted_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
+    assert_eq!(
+        read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
+            .await?
+            .1,
+        versioned_bytes
+    );
+
+    // rustfs#7089: the migration module is on by default, but a bucket that
+    // never configured a source behaves exactly as before.
+    assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
+    assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
+
+    replication_target.shutdown().await;
+    Ok(())
+}
+
+/// Rolling back to the pinned previous release must still read the bucket
+/// metadata the current build wrote.
+///
+/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
+/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
+/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
+/// has to skip those two unknown keys instead of failing the whole blob. If it
+/// did not, every configuration read below would come back empty or error and
+/// the rollback would silently discard the bucket's configuration.
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_with_env(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    env.create_test_bucket(ROLLBACK_BUCKET).await?;
+    enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
+    put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
+    put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
+    let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
+
+    let single_key = "rollback/single";
+    let single_bytes = b"single-part object written by the current RustFS build";
+    let single_version = new_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(single_key)
+        .body(ByteStream::from_static(single_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "rollback/multipart";
+    let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
+
+    restart_from_binary(&mut env, &previous_binary, &server_env).await?;
+    let old_client = env.create_s3_client();
+
+    assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
+
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, single_bytes);
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
+
+    // A PUT on the rolled-back release must still honour the encryption
+    // configuration it decoded out of the current build's metadata blob.
+    let post_rollback_key = "rollback/written-after-rollback";
+    let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
+    old_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(post_rollback_key)
+        .body(ByteStream::from_static(post_rollback_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "the rolled-back release lost the bucket default encryption"
+    );
+    assert_eq!(body, post_rollback_bytes);
+
+    replication_target.shutdown().await;
+    Ok(())
+}

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
 # Compiles the controlled list-objects namespace-journal chaos injector into a
 # production binary (it is always available to tests). Off by default so the
 # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
@@ -212,8 +213,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
-google-cloud-storage = { workspace = true }
-google-cloud-auth = { workspace = true }
+google-cloud-storage = { workspace = true, optional = true }
+google-cloud-auth = { workspace = true, optional = true }
 faster-hex = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
@@ -244,6 +245,7 @@ windows-sys = { workspace = true, features = [
 windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
 
 [dev-dependencies]
+aws-smithy-async.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
 criterion = { workspace = true, features = ["html_reports"] }
 temp-env = { workspace = true, features = ["async_closure"] }

crates/ecstore/src/api/mod.rs

@@ -89,8 +89,9 @@ pub mod bucket {
         #[allow(clippy::module_inception)]
         pub mod lifecycle {
             pub use crate::bucket::lifecycle::lifecycle::{
-                Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
-                TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
+                Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
+                ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
+                object_opts_from_object_info,
             };
         }
 
@@ -146,66 +147,23 @@ pub mod bucket {
         };
     }
 
-    pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
-            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
-            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
-            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
-            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
-            SourceLatencySnapshot, source_client_spec,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-            OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
-            SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
-            PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
-            WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
-            idle_guarded_body,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
-            ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
-            SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
-        };
-        pub mod backfill {
-            pub use crate::bucket::on_demand_migration::backfill::{
-                BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
-                BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
-                BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
-                BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
-                BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
-                StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
-                read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
-            };
-        }
-        pub mod source_client {
-            pub use crate::bucket::on_demand_migration::source_client::{
-                SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
-                SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
-                resolve_path_style,
-            };
-        }
-    }
-
     pub mod metadata_sys {
-        #[cfg(feature = "test-util")]
-        pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
         pub use crate::bucket::metadata_sys::{
-            BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
-            acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
-            delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
-            get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
-            get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
-            get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
-            get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
-            get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
-            remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
-            update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
+            BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
+            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
+            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
+            get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
+            get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
+            get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
+            get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
+            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
+            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
+            update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
+            update_quota_if_incarnation, update_under_transaction_lock,
         };
+        #[cfg(feature = "test-util")]
+        pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
     }
 
     pub mod migration {
@@ -248,7 +206,14 @@ pub mod bucket {
     pub mod remote_s3_client {
         pub use crate::bucket::remote_s3_client::{
             PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
-            validate_remote_endpoint,
+            build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
+        };
+    }
+
+    pub mod sealed_credentials {
+        pub use crate::bucket::sealed_credentials::{
+            CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError,
+            SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret,
         };
     }
 
@@ -472,9 +437,11 @@ pub mod notification {
     #[cfg(any(test, feature = "test-util"))]
     pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
     pub use crate::services::notification_sys::{
-        ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
-        acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
-        new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
+        ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
+        NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
+        acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
+        legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
+        scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -485,9 +452,9 @@ pub mod object {
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
         SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
-        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
-        unregister_object_mutation_hook,
+        ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
+        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
+        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -551,6 +518,12 @@ pub mod set_disk {
     pub mod test_util {
         pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
+
+        /// Keep a namespace commit pending until the returned owner is dropped.
+        #[must_use]
+        pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync {
+            store.ctx.begin_namespace_commit()
+        }
     }
 }
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -59,7 +59,7 @@ use rustfs_utils::http::{
     insert_header,
 };
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::error::Error;
 use std::fmt;
 use std::str::FromStr as _;
@@ -376,6 +376,11 @@ pub struct BucketTargetSys {
     /// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
     ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
     pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
+    /// Buckets whose persisted `bucket-targets.json` exists but cannot be
+    /// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
+    /// alongside `targets_map`, and read before it so an unreadable
+    /// configuration surfaces as a typed error instead of an empty target set.
+    unreadable_targets: Arc<RwLock<HashSet<String>>>,
     pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
     target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
     pub hc_client: Arc<HttpClient>,
@@ -419,6 +424,7 @@ impl BucketTargetSys {
             arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
             ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
             targets_map: Arc::new(RwLock::new(HashMap::new())),
+            unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
             h_mutex: Arc::new(RwLock::new(HashMap::new())),
             target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
             hc_client: Arc::new(build_health_check_client()),
@@ -628,30 +634,40 @@ impl BucketTargetSys {
         health_map.clone()
     }
 
-    pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> {
+    /// Targets of one bucket, or of every bucket when `bucket` is empty.
+    ///
+    /// A bucket that simply has no targets yields an empty list; a bucket
+    /// whose persisted configuration cannot be decoded is an error, so an
+    /// admin listing reports the fault instead of an empty list that reads as
+    /// "replication is not configured" (rustfs/backlog#2282).
+    pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result<Vec<BucketTarget>, BucketTargetError> {
         let health_stats = self.target_health_stats().await;
         let mut targets = Vec::new();
 
         if !bucket.is_empty() {
-            if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await {
-                for mut target in bucket_targets.targets {
-                    if arn_type.is_empty() || target.target_type.to_string() == arn_type {
-                        if let Some(health) = health_stats.get(&target.arn) {
-                            target.total_downtime = health.offline_duration;
-                            target.online = health.online;
-                            target.last_online = health.last_online;
-                            target.latency = target::LatencyStat {
-                                curr: health.latency.curr,
-                                avg: health.latency.avg,
-                                max: health.latency.peak,
-                            };
-                            target.offline_count = health.offline_count;
+            match self.list_bucket_targets(bucket).await {
+                Ok(bucket_targets) => {
+                    for mut target in bucket_targets.targets {
+                        if arn_type.is_empty() || target.target_type.to_string() == arn_type {
+                            if let Some(health) = health_stats.get(&target.arn) {
+                                target.total_downtime = health.offline_duration;
+                                target.online = health.online;
+                                target.last_online = health.last_online;
+                                target.latency = target::LatencyStat {
+                                    curr: health.latency.curr,
+                                    avg: health.latency.avg,
+                                    max: health.latency.peak,
+                                };
+                                target.offline_count = health.offline_count;
+                            }
+                            targets.push(target);
                         }
-                        targets.push(target);
                     }
                 }
+                Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {}
+                Err(err) => return Err(err),
             }
-            return targets;
+            return Ok(targets);
         }
 
         let targets_map = self.targets_map.read().await;
@@ -674,10 +690,16 @@ impl BucketTargetSys {
             }
         }
 
-        targets
+        Ok(targets)
     }
 
     pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
+        if self.unreadable_targets.read().await.contains(bucket) {
+            return Err(BucketTargetError::BucketRemoteTargetsUnreadable {
+                bucket: bucket.to_string(),
+            });
+        }
+
         let targets_map = self.targets_map.read().await;
         if let Some(targets) = targets_map.get(bucket) {
             Ok(BucketTargets {
@@ -690,13 +712,30 @@ impl BucketTargetSys {
         }
     }
 
+    /// Record that this bucket's persisted targets configuration exists but
+    /// cannot be decoded (rustfs/backlog#2282).
+    ///
+    /// Any snapshot published from an earlier readable load is deliberately
+    /// left in place: withdrawing it would produce exactly the silent "no
+    /// targets configured" state this marker exists to prevent. The marker is
+    /// cleared by the next successful publish, which is what makes a repaired
+    /// configuration take effect without a restart.
+    pub async fn mark_targets_unreadable(&self, bucket: &str) {
+        let update_mutex = self.target_update_mutex(bucket).await;
+        let _update_guard = update_mutex.lock().await;
+
+        self.unreadable_targets.write().await.insert(bucket.to_string());
+    }
+
     pub async fn delete(&self, bucket: &str) {
         let update_mutex = self.target_update_mutex(bucket).await;
         let _update_guard = update_mutex.lock().await;
 
-        // Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
-        // then ssec_passthrough_map (always last; also taken standalone by the
-        // capability accessors).
+        // Lock order: unreadable_targets, then targets_map, then
+        // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
+        // (always last; also taken standalone by the capability accessors).
+        self.unreadable_targets.write().await.remove(bucket);
+
         let mut targets_map = self.targets_map.write().await;
         let mut arn_remotes_map = self.arn_remotes_map.write().await;
         let mut health_map = self.target_h_mutex.write().await;
@@ -1093,6 +1132,11 @@ impl BucketTargetSys {
     /// Keeping persisted-config reads under the same mutex prevents a stale
     /// reload from overwriting a concurrent credential rotation.
     async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
+        // Reaching here means the persisted configuration decoded, so the
+        // unreadable marker (if any) is stale. Cleared before the maps below
+        // so `unreadable_targets` stays the outermost of this module's locks.
+        self.unreadable_targets.write().await.remove(bucket);
+
         let mut clients = Vec::new();
         if let Some(new_targets) = targets {
             for target in &new_targets.targets {
@@ -1100,9 +1144,9 @@ impl BucketTargetSys {
             }
         }
 
-        // Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
-        // then ssec_passthrough_map (always last; also taken standalone by the
-        // capability accessors).
+        // Lock order: unreadable_targets (above), then targets_map, then
+        // arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
+        // (always last; also taken standalone by the capability accessors).
         let mut targets_map = self.targets_map.write().await;
         let mut arn_remotes_map = self.arn_remotes_map.write().await;
         let mut health_map = self.target_h_mutex.write().await;
@@ -1161,6 +1205,11 @@ impl BucketTargetSys {
     }
 
     pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
+        if meta.bucket_targets_unreadable() {
+            self.mark_targets_unreadable(bucket).await;
+            return;
+        }
+
         let Some(config) = &meta.bucket_target_config else {
             return;
         };
@@ -2276,6 +2325,13 @@ pub enum BucketTargetError {
     BucketRemoteTargetNotFound {
         bucket: String,
     },
+    /// The bucket's persisted targets configuration exists but cannot be
+    /// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the
+    /// bucket genuinely has no targets: callers must not degrade this one to
+    /// an empty target set (rustfs/backlog#2282).
+    BucketRemoteTargetsUnreadable {
+        bucket: String,
+    },
     BucketRemoteArnTypeInvalid {
         bucket: String,
     },
@@ -2309,6 +2365,9 @@ impl fmt::Display for BucketTargetError {
             BucketTargetError::BucketRemoteTargetNotFound { bucket } => {
                 write!(f, "Remote target not found for bucket: {bucket}")
             }
+            BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => {
+                write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}")
+            }
             BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => {
                 write!(f, "Invalid ARN type for bucket: {bucket}")
             }
@@ -3256,7 +3315,7 @@ mod tests {
             }],
         );
 
-        let targets = sys.list_targets("", "").await;
+        let targets = sys.list_targets("", "").await.expect("listing every bucket's targets");
 
         assert_eq!(targets.len(), 1);
         assert!(!targets[0].online);

crates/ecstore/src/bucket/lifecycle/core.rs

@@ -15,9 +15,9 @@
 use crate::object_api::ObjectInfo;
 
 pub use rustfs_lifecycle::{
-    Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
-    TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
-    expiration_action_has_valid_target,
+    Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
+    RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
+    expected_expiry_time, expiration_action_has_valid_target,
 };
 
 pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -25,6 +25,7 @@ use super::{
     manual_transition_job, tier_delete_journal, transition_transaction,
 };
 use crate::error::{Error, Result};
+use crate::services::tier::tier_probe_intent;
 
 pub(crate) const ILM_META_PREFIX: &str = "ilm";
 const ILM_META_OBJECT_PREFIX: &str = "ilm/";
@@ -35,6 +36,7 @@ pub(crate) enum DurableIlmRecordKind {
     TierDeleteJournal,
     TierDeleteDispatchManifest,
     TransitionTransaction,
+    TierProbeIntent,
     ManualTransitionJob,
     ManualTransitionScope,
     ManualTransitionTask,
@@ -73,6 +75,12 @@ pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = Durable
     max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE,
     kind: DurableIlmRecordKind::TransitionTransaction,
 };
+pub(crate) const TIER_PROBE_INTENT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-probe-intent",
+    prefix: tier_probe_intent::TIER_PROBE_INTENT_RECORD_PREFIX,
+    max_record_size: tier_probe_intent::MAX_TIER_PROBE_INTENT_SIZE,
+    kind: DurableIlmRecordKind::TierProbeIntent,
+};
 pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
     name: "manual-transition-job",
     prefix: "ilm/manual-transition/jobs",
@@ -98,11 +106,12 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
     TIER_DELETE_JOURNAL_V6_NAMESPACE,
     TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
     TRANSITION_TRANSACTION_NAMESPACE,
+    TIER_PROBE_INTENT_NAMESPACE,
     MANUAL_TRANSITION_JOB_NAMESPACE,
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
     MANUAL_TRANSITION_TASK_NAMESPACE,
@@ -200,6 +209,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         revision: u64,
         state: transition_transaction::TransitionTransactionState,
     },
+    TierProbeIntent {
+        content_sha256: String,
+        identity_sha256: String,
+        remote_version_sha256: String,
+        remote_version_known: bool,
+        owner_fence_sha256: String,
+        revision: u64,
+        state: tier_probe_intent::TierProbeIntentState,
+    },
     ManualTransitionJob {
         content_sha256: String,
         identity_sha256: String,
@@ -232,6 +250,7 @@ impl DurableIlmRecordCheckpoint {
             | Self::TierDeleteDispatchManifest { content_sha256, .. }
             | Self::TierDeleteDispatchParent { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
+            | Self::TierProbeIntent { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
             | Self::ManualTransitionTask { content_sha256 }
@@ -421,6 +440,32 @@ impl DurableIlmRecordCheckpoint {
                         .is_some_and(|expected_revision| *next_revision == expected_revision)
                     && (!previous_remote_version_known || previous_remote_version == next_remote_version)
             }
+            (
+                Self::TierProbeIntent {
+                    identity_sha256: previous_identity,
+                    remote_version_sha256: previous_remote_version,
+                    remote_version_known: previous_remote_version_known,
+                    owner_fence_sha256: previous_owner_fence,
+                    revision: previous_revision,
+                    state: previous_state,
+                    ..
+                },
+                Self::TierProbeIntent {
+                    identity_sha256: next_identity,
+                    remote_version_sha256: next_remote_version,
+                    owner_fence_sha256: next_owner_fence,
+                    revision: next_revision,
+                    state: next_state,
+                    ..
+                },
+            ) => {
+                previous_identity == next_identity
+                    && previous_owner_fence == next_owner_fence
+                    && next_revision
+                        .checked_sub(*previous_revision)
+                        .is_some_and(|distance| distance == 1 && tier_probe_state_reaches(*previous_state, *next_state, distance))
+                    && (!previous_remote_version_known || previous_remote_version == next_remote_version)
+            }
             (
                 Self::ManualTransitionJob {
                     content_sha256: previous_content,
@@ -500,6 +545,14 @@ impl DurableIlmRecordCheckpoint {
     /// after the exact terminal ETag and terminal receipt were committed, to
     /// purge older object versions exposed by that deletion.
     pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
+        if let Self::TierProbeIntent { state, .. } = terminal
+            && !matches!(
+                state,
+                tier_probe_intent::TierProbeIntentState::AbortedNoRemote | tier_probe_intent::TierProbeIntentState::Completed
+            )
+        {
+            return false;
+        }
         if self == terminal || self.validate_successor(terminal).is_ok() {
             return true;
         }
@@ -568,6 +621,37 @@ impl DurableIlmRecordCheckpoint {
                         }
                     })
             }
+            (
+                Self::TierProbeIntent {
+                    identity_sha256: previous_identity,
+                    remote_version_sha256: previous_remote_version,
+                    remote_version_known: previous_remote_version_known,
+                    owner_fence_sha256: previous_owner_fence,
+                    revision: previous_revision,
+                    state: previous_state,
+                    ..
+                },
+                Self::TierProbeIntent {
+                    identity_sha256: terminal_identity,
+                    remote_version_sha256: terminal_remote_version,
+                    owner_fence_sha256: terminal_owner_fence,
+                    revision: terminal_revision,
+                    state: terminal_state,
+                    ..
+                },
+            ) => {
+                previous_identity == terminal_identity
+                    && previous_owner_fence == terminal_owner_fence
+                    && matches!(
+                        terminal_state,
+                        tier_probe_intent::TierProbeIntentState::AbortedNoRemote
+                            | tier_probe_intent::TierProbeIntentState::Completed
+                    )
+                    && terminal_revision
+                        .checked_sub(*previous_revision)
+                        .is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
+                    && (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
+            }
             _ => false,
         }
     }
@@ -606,6 +690,23 @@ fn transition_state_distance(
     }
 }
 
+fn tier_probe_state_reaches(
+    from: tier_probe_intent::TierProbeIntentState,
+    to: tier_probe_intent::TierProbeIntentState,
+    revision_distance: u64,
+) -> bool {
+    use tier_probe_intent::TierProbeIntentState::{AbortedNoRemote, CleanupPending, Completed, UploadOutcomeUnknown, Uploaded};
+
+    match (from, to) {
+        (UploadOutcomeUnknown, Uploaded | CleanupPending | AbortedNoRemote) => revision_distance == 1,
+        (UploadOutcomeUnknown, Completed) => matches!(revision_distance, 2 | 3),
+        (Uploaded, CleanupPending) => revision_distance == 1,
+        (Uploaded, Completed) => revision_distance == 2,
+        (CleanupPending, Completed) => revision_distance == 1,
+        _ => false,
+    }
+}
+
 fn manual_job_state_reaches(
     from: manual_transition_job::ManualTransitionJobState,
     to: manual_transition_job::ManualTransitionJobState,
@@ -1082,6 +1183,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 },
             )
         }
+        DurableIlmRecordKind::TierProbeIntent => {
+            let probe_id = tier_probe_intent::tier_probe_intent_id_from_record_object_name(path)
+                .map_err(|err| Error::other(err.to_string()))?;
+            let intent =
+                tier_probe_intent::TierProbeIntent::decode(probe_id, data).map_err(|err| Error::other(err.to_string()))?;
+            let canonical =
+                tier_probe_intent::tier_probe_intent_record_object_name(probe_id).map_err(|err| Error::other(err.to_string()))?;
+            if canonical != path {
+                return Err(Error::other("tier probe intent path is not canonical"));
+            }
+            let identity_sha256 = checkpoint_hash(&(
+                intent.probe_id,
+                &intent.operation,
+                &intent.tier_name,
+                intent.destination_id,
+                &intent.probe_object,
+                &intent.creator_id,
+                intent.creator_epoch,
+                intent.created_at_unix_nanos,
+            ))?;
+            let remote_version_sha256 = checkpoint_hash(&intent.remote_version)?;
+            let owner_fence_sha256 = checkpoint_hash(&intent.owner)?;
+            (
+                "probe_id",
+                probe_id.to_string(),
+                DurableIlmRecordCheckpoint::TierProbeIntent {
+                    content_sha256,
+                    identity_sha256,
+                    remote_version_sha256,
+                    remote_version_known: !intent.remote_version.is_unknown(),
+                    owner_fence_sha256,
+                    revision: intent.revision,
+                    state: intent.state,
+                },
+            )
+        }
         DurableIlmRecordKind::ManualTransitionJob => {
             let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
                 .map_err(|err| Error::other(err.to_string()))?;
@@ -1237,6 +1374,102 @@ mod tests {
         }
     }
 
+    fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
+        let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
+        tier_probe_intent::TierProbeIntent {
+            probe_id,
+            revision: 1,
+            state: tier_probe_intent::TierProbeIntentState::UploadOutcomeUnknown,
+            operation: tier_probe_intent::TierProbeOperationIdentity::Verify {
+                config_etag: "config-etag".to_string(),
+                backend_identity: [1; 32],
+            },
+            tier_name: "COLD-A".to_string(),
+            destination_id: [1; 32],
+            probe_object: tier_probe_intent::tier_probe_object_name(probe_id),
+            creator_id: "node-a".to_string(),
+            creator_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture creator epoch should parse"),
+            created_at_unix_nanos: 1_780_000_000_000_000_000,
+            owner: tier_probe_intent::TierProbeOwnerFence {
+                owner_id: "node-a".to_string(),
+                owner_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture owner epoch should parse"),
+                not_after_unix_nanos: 1_780_000_900_000_000_000,
+            },
+            remote_version: tier_probe_intent::TierProbeRemoteVersion::default(),
+        }
+    }
+
+    fn tier_probe_checkpoint(intent: &tier_probe_intent::TierProbeIntent) -> DurableIlmRecordCheckpoint {
+        let path =
+            tier_probe_intent::tier_probe_intent_record_object_name(intent.probe_id).expect("tier probe path should build");
+        let encoded = intent.encode().expect("tier probe intent should encode");
+        let namespace = classify_durable_ilm_record(&path)
+            .expect("tier probe namespace should classify")
+            .expect("tier probe intent should be durable");
+        assert_eq!(namespace, &TIER_PROBE_INTENT_NAMESPACE);
+        validate_durable_ilm_record(&path, &encoded)
+            .expect("tier probe intent should validate")
+            .checkpoint
+    }
+
+    #[test]
+    fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
+        let initial_intent = tier_probe_intent_fixture();
+        let initial = tier_probe_checkpoint(&initial_intent);
+
+        let mut uploaded_intent = initial_intent;
+        uploaded_intent
+            .advance(
+                tier_probe_intent::TierProbeIntentState::Uploaded,
+                tier_probe_intent::TierProbeRemoteVersion::versioned("opaque-v1"),
+            )
+            .expect("uploaded state should advance");
+        let uploaded = tier_probe_checkpoint(&uploaded_intent);
+        initial
+            .validate_successor(&uploaded)
+            .expect("durable receipt may adopt the exact uploaded generation");
+
+        let mut cleanup_intent = uploaded_intent.clone();
+        cleanup_intent
+            .advance(
+                tier_probe_intent::TierProbeIntentState::CleanupPending,
+                uploaded_intent.remote_version.clone(),
+            )
+            .expect("cleanup state should advance");
+        let cleanup = tier_probe_checkpoint(&cleanup_intent);
+        uploaded
+            .validate_successor(&cleanup)
+            .expect("durable receipt may adopt the exact cleanup generation");
+
+        let mut completed_intent = cleanup_intent.clone();
+        completed_intent
+            .advance(tier_probe_intent::TierProbeIntentState::Completed, cleanup_intent.remote_version.clone())
+            .expect("completed state should advance");
+        let completed = tier_probe_checkpoint(&completed_intent);
+        cleanup
+            .validate_successor(&completed)
+            .expect("durable receipt may adopt the exact terminal generation");
+        assert!(
+            initial.is_predecessor_of_terminal(&completed),
+            "terminal cleanup must recognize the full acknowledged-PUT path"
+        );
+        assert!(
+            initial.validate_successor(&completed).is_err(),
+            "ordinary receipt advancement must not skip intermediate generations"
+        );
+        assert!(
+            !initial.is_predecessor_of_terminal(&uploaded),
+            "a nonterminal generation must not be accepted as terminal proof"
+        );
+
+        let mut rebound = uploaded_intent;
+        rebound.owner.owner_epoch = Uuid::new_v4();
+        assert!(
+            rebound.encode().is_err(),
+            "dormant v1 must reject owner takeover before producing a checkpoint"
+        );
+    }
+
     #[test]
     fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
         use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()
@@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent(
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
 
         let mut opts = ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             no_lock: true,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(observed_etag),
@@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(etag.to_string()),
                 ..Default::default()
@@ -3780,6 +3783,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()
@@ -3869,6 +3873,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_match: Some(etag),
                         ..Default::default()
@@ -3893,6 +3898,7 @@ where
                 data.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use super::runtime_boundary as runtime_sources;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
@@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
 });
 
 #[cfg(test)]
-static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
-    std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
-> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
+type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
+
+#[cfg(test)]
+static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
+    std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
 
 #[derive(Debug)]
 struct RemoteDeleteBreaker {
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
     fn prune(&mut self, now: Instant) {
         while let Some(ts) = self.failures.front().copied() {
             if now.duration_since(ts) > self.window {
-                self.failures.pop_front();
+                let _ = self.failures.pop_front();
             } else {
                 break;
             }
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
         return false;
     }
 
-    if let Some(source) = err.get_ref() {
-        if error_chain_contains_signer_header_marker(source) {
-            return true;
-        }
+    if let Some(source) = err.get_ref()
+        && error_chain_contains_signer_header_marker(source)
+    {
+        return true;
     }
 
     let message = err.to_string().to_ascii_lowercase();
@@ -205,7 +205,7 @@ impl ObjSweeper {
 
     #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
     pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
-        self.version_id = vid.clone();
+        self.version_id = vid;
         self
     }
 
@@ -219,7 +219,7 @@ impl ObjSweeper {
     #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
     pub fn get_opts(&self) -> lifecycle::ObjectOpts {
         let mut opts = ObjectOpts {
-            version_id: self.version_id.clone(),
+            version_id: self.version_id,
             versioned: self.versioned,
             version_suspended: self.suspended,
             ..Default::default()
@@ -388,8 +388,8 @@ impl Jentry {
 impl ExpiryOp for Jentry {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.tier_name).as_bytes());
-        hasher.update(format!("{}", self.obj_name).as_bytes());
+        hasher.update(self.tier_name.as_bytes());
+        hasher.update(self.obj_name.as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
 
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
     tier_name: &str,
     tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
 ) -> Result<(), std::io::Error> {
-    let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
+    let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
         .await
         .map_err(std::io::Error::other)?;
     delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
         data.clone(),
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(etag),
                 ..Default::default()

crates/ecstore/src/bucket/metadata.rs

@@ -477,28 +477,29 @@ impl BucketMetadata {
         !self.table_bucket_config_json.is_empty()
     }
 
-    /// Parsed per-bucket durability override, if a valid one is stored.
+    /// `bucket-targets.json` is stored for this bucket but this build cannot
+    /// decode it.
     ///
-    /// Absent/empty/unparsable payloads all mean "no override" (the bucket
-    /// follows the global durability mode); a parse failure is logged so a
-    /// corrupted entry cannot silently change fsync behavior.
-    /// Parsed on-demand migration config, if one is stored.
-    ///
-    /// `Ok(None)` means no config (absent or cleared). A stored payload that
-    /// does not parse is an error, never a default: the runtime must not
-    /// pull from a source it cannot describe.
-    pub fn on_demand_migration_config(
-        &self,
-    ) -> std::result::Result<
-        Option<super::on_demand_migration::OnDemandMigrationConfig>,
-        super::on_demand_migration::OnDemandMigrationConfigError,
-    > {
-        if self.on_demand_migration_config_json.is_empty() {
-            return Ok(None);
-        }
-        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
+    /// Keeps "no replication targets configured" and "the target
+    /// configuration cannot be read" apart, the same distinction the
+    /// `fabricated` marker draws for the bucket metadata as a whole. Only
+    /// meaningful after [`Self::parse_all_configs`] has run; readers must fail
+    /// closed on `true` instead of serving an empty target set.
+    pub fn bucket_targets_unreadable(&self) -> bool {
+        !self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
     }
 
+    /// Opaque application-owned configuration with its persisted update time.
+    /// Empty bytes mean absent or cleared; decoding belongs to the consumer.
+    pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
+        (!self.on_demand_migration_config_json.is_empty()).then_some((
+            self.on_demand_migration_config_json.as_slice(),
+            self.on_demand_migration_config_updated_at,
+        ))
+    }
+
+    /// Parsed per-bucket durability override, if a valid one is stored.
+    /// Invalid payloads follow the global mode after logging a parse failure.
     pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
         if self.durability_config_json.is_empty() {
             return None;
@@ -904,13 +905,6 @@ impl BucketMetadata {
                 self.durability_config_updated_at = updated;
             }
             BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
-                // Structural check only (shape, unknown fields); the
-                // deployment-relative rules run in the admin handler with a
-                // `ValidationContext`. A blob this build cannot read must not
-                // be persisted for every later reader to trip over.
-                if !data.is_empty() {
-                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
-                }
                 self.on_demand_migration_config_json = data;
                 self.on_demand_migration_config_updated_at = updated;
             }
@@ -964,7 +958,32 @@ impl BucketMetadata {
         Ok(())
     }
 
-    fn parse_all_configs(&mut self) -> Result<()> {
+    /// Decode every stored sub-configuration into its typed field.
+    ///
+    /// A decode failure never fails the whole load: this runs on every bucket
+    /// metadata read, including startup and peer reload, so one bucket's
+    /// corrupt sub-configuration must not make the bucket — or the node —
+    /// unloadable. Instead the failure is *retained*: the raw bytes stay
+    /// untouched and the typed field stays `None`, so `!raw.is_empty() &&
+    /// typed.is_none()` is the durable "exists but cannot be read" signal that
+    /// each accessor keys off. Which accessors must fail closed on it:
+    ///
+    /// | Config | Verdict |
+    /// |---|---|
+    /// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
+    /// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
+    /// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
+    /// | replication | Fails closed in `get_replication_config`. |
+    /// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
+    /// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
+    /// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
+    /// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
+    /// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
+    /// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
+    /// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
+    /// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
+    /// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
+    pub(super) fn parse_all_configs(&mut self) -> Result<()> {
         if let Err(e) = self.parse_policy_config() {
             tracing::warn!(
                 event = "bucket_metadata_parse_failed",
@@ -1088,20 +1107,26 @@ impl BucketMetadata {
                 "Failed to parse bucket metadata config"
             );
         }
+        // A stored targets blob that cannot be decoded must not collapse into
+        // the empty target set: that is indistinguishable from "no replication
+        // configured", so replication stops and no caller ever sees an error
+        // (rustfs/backlog#2282). Leaving the typed field `None` while the raw
+        // bytes stay non-empty is the retained parse failure every targets
+        // reader keys off; the bytes are preserved so the configuration is
+        // still recoverable.
+        self.bucket_target_config = None;
         if !self.bucket_targets_config_json.is_empty() {
-            if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
-                .map(|t| self.bucket_target_config = Some(t))
-            {
-                tracing::warn!(
+            match serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) {
+                Ok(targets) => self.bucket_target_config = Some(targets),
+                Err(e) => tracing::error!(
                     event = "bucket_metadata_parse_failed",
                     component = "ecstore",
                     subsystem = "bucket_metadata",
                     bucket = %self.name,
                     config = "bucket_targets",
                     error = %e,
-                    "Failed to parse bucket metadata config"
-                );
-                self.bucket_target_config = Some(BucketTargets::default());
+                    "Bucket replication targets are unreadable; replication for this bucket fails closed"
+                ),
             }
         } else {
             self.bucket_target_config = Some(BucketTargets::default());
@@ -1535,6 +1560,117 @@ mod test {
         assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket");
     }
 
+    /// rustfs/backlog#2282: a stored targets blob this build cannot decode
+    /// must not become the empty target set, and must stay distinguishable
+    /// from a bucket that never configured a target.
+    #[test]
+    fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() {
+        let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec();
+        let mut corrupt = BucketMetadata::new("corrupt-targets");
+        corrupt.bucket_targets_config_json = truncated.clone();
+
+        corrupt
+            .parse_all_configs()
+            .expect("one unreadable sub-config must not fail the whole metadata load");
+
+        assert!(
+            corrupt.bucket_target_config.is_none(),
+            "an undecodable targets blob must not produce a target set at all"
+        );
+        assert!(corrupt.bucket_targets_unreadable());
+        assert_eq!(
+            corrupt.bucket_targets_config_json, truncated,
+            "the raw bytes must survive so the configuration stays recoverable"
+        );
+
+        // The genuinely-absent case is unchanged, and the two now diverge.
+        let mut absent = BucketMetadata::new("no-targets");
+        absent.parse_all_configs().expect("absent targets parse");
+        assert!(
+            absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty),
+            "a bucket that configured no target still reads as an empty target set"
+        );
+        assert!(!absent.bucket_targets_unreadable());
+    }
+
+    /// `Credentials` carries no struct-level `serde(default)`, so one target
+    /// missing `secretKey` is a hard parse error for the whole document. That
+    /// must surface as "unreadable", never as "no targets configured".
+    #[test]
+    fn bucket_targets_missing_secret_key_are_unreadable_not_empty() {
+        let mut bm = BucketMetadata::new("missing-secret-key");
+        bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec();
+
+        bm.parse_all_configs()
+            .expect("a rejected targets document must not fail the whole metadata load");
+
+        assert!(
+            bm.bucket_targets_unreadable(),
+            "a targets document rejected for a missing secretKey is unreadable, not empty"
+        );
+        assert!(bm.bucket_target_config.is_none());
+    }
+
+    /// The invariant every branch of `parse_all_configs` shares: a stored but
+    /// undecodable payload keeps its raw bytes and leaves the typed field
+    /// `None`, so no branch fabricates a value. What a reader may then do with
+    /// that state is decided per config; see the table on `parse_all_configs`.
+    #[test]
+    fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() {
+        let malformed_xml = b"<not-a-valid-document".to_vec();
+        let malformed_json = b"{not-json".to_vec();
+
+        let mut bm = BucketMetadata::new("all-configs-malformed");
+        bm.policy_config_json = malformed_json.clone();
+        bm.quota_config_json = malformed_json.clone();
+        bm.bucket_targets_config_json = malformed_json.clone();
+        bm.notification_config_xml = malformed_xml.clone();
+        bm.lifecycle_config_xml = malformed_xml.clone();
+        bm.object_lock_config_xml = malformed_xml.clone();
+        bm.versioning_config_xml = malformed_xml.clone();
+        bm.encryption_config_xml = malformed_xml.clone();
+        bm.tagging_config_xml = malformed_xml.clone();
+        bm.replication_config_xml = malformed_xml.clone();
+        bm.cors_config_xml = malformed_xml.clone();
+        bm.logging_config_xml = malformed_xml.clone();
+        bm.website_config_xml = malformed_xml.clone();
+        bm.accelerate_config_xml = malformed_xml.clone();
+        bm.request_payment_config_xml = malformed_xml.clone();
+        bm.public_access_block_config_xml = malformed_xml.clone();
+        // `bucket_acl_config_json` is only checked for UTF-8, so only invalid
+        // UTF-8 exercises its failure branch.
+        bm.bucket_acl_config_json = vec![0xff, 0xfe];
+
+        bm.parse_all_configs()
+            .expect("a bucket whose every config is corrupt must still load its metadata");
+
+        let cleared: [(&str, bool); 17] = [
+            ("policy", bm.policy_config.is_none()),
+            ("quota", bm.quota_config.is_none()),
+            ("bucket_targets", bm.bucket_target_config.is_none()),
+            ("notification", bm.notification_config.is_none()),
+            ("lifecycle", bm.lifecycle_config.is_none()),
+            ("object_lock", bm.object_lock_config.is_none()),
+            ("versioning", bm.versioning_config.is_none()),
+            ("encryption", bm.sse_config.is_none()),
+            ("tagging", bm.tagging_config.is_none()),
+            ("replication", bm.replication_config.is_none()),
+            ("cors", bm.cors_config.is_none()),
+            ("logging", bm.logging_config.is_none()),
+            ("website", bm.website_config.is_none()),
+            ("accelerate", bm.accelerate_config.is_none()),
+            ("request_payment", bm.request_payment_config.is_none()),
+            ("public_access_block", bm.public_access_block_config.is_none()),
+            ("bucket_acl", bm.bucket_acl_config.is_none()),
+        ];
+        for (config, is_cleared) in cleared {
+            assert!(is_cleared, "{config}: a corrupt payload must not be replaced by a default");
+        }
+
+        assert_eq!(bm.bucket_targets_config_json, malformed_json, "raw bytes are retained");
+        assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
+    }
+
     #[test]
     fn lifecycle_update_config_clears_parsed_config_on_delete() {
         let mut bm = BucketMetadata::new("test-bucket");
@@ -1824,51 +1960,30 @@ mod test {
 
     const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
-    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
-    /// extension entry that round-trips through `update_config` and the
-    /// msgpack codec, clears on delete, and never parses corruption into a
-    /// default.
+    /// The metadata codec preserves application-owned bytes and timestamps.
     #[test]
     fn on_demand_migration_config_round_trips_and_tracks_updates() {
-        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
-
         let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
-
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+        assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("valid config is accepted");
-        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
-
+            .expect("opaque config is accepted");
+        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
+        assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
         let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
         assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(
-            back.on_demand_migration_config_updated_at.unix_timestamp(),
-            bm.on_demand_migration_config_updated_at.unix_timestamp()
-        );
-        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
-
-        // A blob this build cannot read is rejected at the write boundary
-        // rather than persisted for every reader to trip over.
-        let before = bm.on_demand_migration_config_json.clone();
-        assert!(
-            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
-                .is_err()
-        );
-        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
-
-        // Delete clears the entry.
-        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
         assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
         assert!(bm.on_demand_migration_config_updated_at >= stamped);
-
-        // Corruption that bypassed `update_config` (disk, another writer)
-        // is a typed error, never a default.
-        bm.on_demand_migration_config_json = b"not-json".to_vec();
-        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
+        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
+            .unwrap();
+        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
+        assert_eq!(
+            back.on_demand_migration_config_json, b"not-json",
+            "metadata must not reinterpret application bytes"
+        );
     }
 
     /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
@@ -1880,7 +1995,7 @@ mod test {
         let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
         assert!(bm.on_demand_migration_config_json.is_empty());
         assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
 
         bm.default_timestamps();
         assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");

crates/ecstore/src/bucket/metadata_sys.rs

@@ -19,7 +19,6 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
-use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
@@ -49,6 +48,11 @@ use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};
 use uuid::Uuid;
 
+/// Opaque bucket configuration notifications for application-owned services.
+/// `None` withdraws a configuration; consumers validate nonempty bytes.
+pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
+pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
+
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
 
 #[cfg(any(test, feature = "test-util"))]
@@ -360,6 +364,16 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
 }
 
 async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
+    if bm.bucket_targets_unreadable() {
+        // "The configuration cannot be read" is not "no targets configured".
+        // Publishing an empty snapshot here is what silently stopped
+        // replication (rustfs/backlog#2282): mark the bucket instead, so every
+        // targets reader gets a typed error, and leave any snapshot from an
+        // earlier readable load in place rather than withdrawing it.
+        BucketTargetSys::get().mark_targets_unreadable(bucket).await;
+        return;
+    }
+
     BucketTargetSys::get()
         .update_all_targets(bucket, bm.bucket_target_config.as_ref())
         .await;
@@ -385,39 +399,21 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
-/// Publish the bucket's on-demand migration config (or its absence) to the
-/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
-///
-/// Called from the same five cache-install paths as
-/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
-/// published as `None`: the runtime must stop pulling for that bucket rather
-/// than keep an older config or guess.
+/// Publish application-owned bytes on every cache install path.
 fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
-        return;
-    };
-    match bm.on_demand_migration_config() {
-        Ok(config) => hook(bucket, config.as_ref()),
-        Err(err) => {
-            warn!(
-                event = "bucket_metadata_parse_failed",
-                component = "ecstore",
-                subsystem = "bucket_metadata",
-                bucket = %bucket,
-                config = "on_demand_migration",
-                error = %err,
-                "Failed to parse bucket metadata config"
-            );
-            hook(bucket, None);
-        }
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(
+            bucket,
+            super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
+            bm.on_demand_migration_config()
+                .map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
+        );
     }
 }
 
-/// Withdraw a bucket's on-demand migration config when its metadata leaves
-/// the cache.
 fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
-        hook(bucket, None);
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
     }
 }
 
@@ -645,6 +641,12 @@ pub struct BucketMetadataMutationGuard {
 }
 
 impl BucketMetadataMutationGuard {
+    /// Returns the storage-verified identity while both incarnation fences remain valid.
+    pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
+        self.ensure_valid(&self.bucket)?;
+        Ok((&self.bucket, self.incarnation_id))
+    }
+
     fn ensure_valid(&self, bucket: &str) -> Result<()> {
         if self.bucket != bucket {
             return Err(Error::other("bucket metadata mutation guard does not match bucket"));
@@ -664,6 +666,29 @@ async fn acquire_config_write_guard_for_incarnation(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
     expected_incarnation_id: Option<Uuid>,
+) -> Result<BucketMetadataMutationGuard> {
+    acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
+}
+
+/// Scanner probes must not create an incarnation to make a capability available.
+pub async fn acquire_scanner_bucket_incarnation_fence(
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<BucketMetadataMutationGuard> {
+    super::utils::check_valid_bucket_name(bucket)?;
+    let sys = get_bucket_metadata_sys()?;
+    if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
+        return Err(Error::other("scanner bucket incarnation owner does not match"));
+    }
+    acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
+}
+
+async fn acquire_config_write_guard_with_migration(
+    sys: Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    expected_incarnation_id: Option<Uuid>,
+    migrate: bool,
 ) -> Result<BucketMetadataMutationGuard> {
     let metadata_sys = sys.read().await.clone();
     let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
@@ -671,13 +696,15 @@ async fn acquire_config_write_guard_for_incarnation(
     // Legacy buckets are migrated while the lifecycle fence prevents a
     // same-name replacement. The second read under the write transaction is
     // the CAS source of truth for the actual rewrite.
-    await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
-        bucket,
-        "bucket config incarnation migration",
-        metadata_sys.get_bucket_incarnation_id(bucket),
-    )
-    .await?;
+    if migrate {
+        await_bucket_namespace_operation(
+            Some(&lifecycle_guard),
+            bucket,
+            "bucket config incarnation migration",
+            metadata_sys.get_bucket_incarnation_id(bucket),
+        )
+        .await?;
+    }
     let transaction_guard = await_bucket_namespace_operation(
         Some(&lifecycle_guard),
         bucket,
@@ -1008,15 +1035,21 @@ pub async fn get_durability_config(
 }
 
 /// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. A stored payload that does not parse
-/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
+pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
     bucket_meta_sys.get_on_demand_migration_config(bucket).await
 }
 
+/// Resolve opaque configuration from the store's own metadata system.
+pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+    let sys = bucket_metadata_sys_of(&api.ctx)?;
+    let lock = sys.read().await;
+    lock.get_on_demand_migration_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2118,7 +2151,9 @@ impl BucketMetadataSys {
     pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.public_access_block_config {
+        if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
+            Err(Error::other("persisted bucket public access block configuration is invalid"))
+        } else if let Some(config) = &bm.public_access_block_config {
             Ok((config.clone(), bm.public_access_block_config_updated_at))
         } else {
             Err(Error::ConfigNotFound)
@@ -2429,7 +2464,9 @@ impl BucketMetadataSys {
     pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.sse_config {
+        if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
+            Err(Error::other("persisted bucket encryption configuration is invalid"))
+        } else if let Some(config) = &bm.sse_config {
             Ok((config.clone(), bm.encryption_config_updated_at))
         } else {
             Err(Error::ConfigNotFound)
@@ -2500,7 +2537,9 @@ impl BucketMetadataSys {
     pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.quota_config {
+        if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() {
+            Err(Error::other("persisted bucket quota configuration is invalid"))
+        } else if let Some(config) = &bm.quota_config {
             Ok((config.clone(), bm.quota_config_updated_at))
         } else {
             Err(Error::ConfigNotFound)
@@ -2522,7 +2561,9 @@ impl BucketMetadataSys {
     pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.bucket_target_config {
+        if bm.bucket_targets_unreadable() {
+            Err(Error::other("persisted bucket replication target configuration is invalid"))
+        } else if let Some(config) = &bm.bucket_target_config {
             Ok(config.clone())
         } else {
             Err(Error::ConfigNotFound)
@@ -2530,29 +2571,27 @@ impl BucketMetadataSys {
     }
 
     /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(
-        &self,
-        bucket: &str,
-    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+    pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        let config = bm.on_demand_migration_config().map_err(Error::other)?;
-        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
+        Ok(bm
+            .on_demand_migration_config()
+            .map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
     }
 }
 
 /// Test-only fixture shared with sibling modules (e.g. the quota checker
 /// tests): a 4-disk `ECStore` on an isolated instance context, so tests
 /// exercising the metadata system never touch ambient process state.
-#[cfg(test)]
-pub(crate) mod test_support {
+#[cfg(any(test, feature = "test-util"))]
+pub mod test_support {
     use super::*;
     use crate::disk::endpoint::Endpoint;
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::store::init_local_disks_with_instance_ctx;
 
-    pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+    pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
         let mut dirs = Vec::with_capacity(4);
         let mut endpoints = Vec::with_capacity(4);
         for disk_idx in 0..4 {
@@ -2593,6 +2632,7 @@ pub(crate) mod test_support {
 mod tests {
     use super::test_support::isolated_store_over_temp_disks;
     use super::*;
+    use crate::bucket::bucket_target_sys::BucketTargetError;
     use crate::bucket::metadata::{
         BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
         BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
@@ -2788,6 +2828,36 @@ mod tests {
         );
     }
 
+    /// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor
+    /// whose configuration grants something — plaintext storage, anonymous
+    /// access, capacity, replication targets — reports a corrupt payload as
+    /// invalid rather than as absent, because "absent" is what grants it.
+    #[tokio::test]
+    async fn malformed_permissive_configs_are_not_reported_as_absent() {
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let bucket = "malformed-permissive-config";
+        let mut metadata = BucketMetadata::new(bucket);
+        metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration".to_vec();
+        metadata.public_access_block_config_xml = b"<PublicAccessBlockConfiguration".to_vec();
+        metadata.quota_config_json = b"{not-json".to_vec();
+        metadata.bucket_targets_config_json = b"{not-json".to_vec();
+        metadata
+            .parse_all_configs()
+            .expect("a corrupt sub-config must not fail the load");
+        sys.set(bucket.to_string(), Arc::new(metadata)).await;
+
+        for (config, result) in [
+            ("encryption", sys.get_sse_config(bucket).await.err()),
+            ("public access block", sys.get_public_access_block_config(bucket).await.err()),
+            ("quota", sys.get_quota_config(bucket).await.err()),
+            ("bucket targets", sys.get_bucket_targets_config(bucket).await.err()),
+        ] {
+            let err = result.unwrap_or_else(|| panic!("malformed {config} metadata must not read as a value"));
+            assert_ne!(err, Error::ConfigNotFound, "malformed {config} metadata must not be reported as absent");
+        }
+    }
+
     #[tokio::test]
     async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() {
         use std::sync::atomic::Ordering;
@@ -3127,6 +3197,82 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
+        let bucket = "scoped-ack-legacy";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
+        }
+        let mut metadata = BucketMetadata::new(bucket);
+        metadata.bucket_incarnation_id = Uuid::nil();
+        sys.read()
+            .await
+            .persist_and_set(metadata)
+            .await
+            .expect("persist legacy metadata");
+        assert!(
+            acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
+                .await
+                .is_err()
+        );
+        assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
+        assert!(
+            sys.read()
+                .await
+                .get_config_from_disk(bucket)
+                .await
+                .expect("read metadata")
+                .bucket_incarnation_id
+                .is_nil()
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
+        let bucket = "scoped-ack-recreated";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
+        let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
+            .await
+            .expect("trusted incarnation fence");
+        assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
+        drop(guard);
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete bucket");
+        assert!(
+            acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
+                .await
+                .is_err()
+        );
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
+        assert_ne!(old, new);
+        assert!(
+            acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
+                .await
+                .is_err()
+        );
+        assert!(
+            acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
+                .await
+                .is_ok()
+        );
+    }
+
     #[tokio::test]
     async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4066,6 +4212,114 @@ mod tests {
         target_sys.delete(bucket).await;
     }
 
+    /// rustfs/backlog#2282: an unreadable `bucket-targets.json` reaches every
+    /// targets reader as a typed error; it neither withdraws a snapshot a
+    /// previous readable load published, nor collapses into the "no targets
+    /// configured" state that a bucket with an absent configuration reports.
+    #[tokio::test]
+    #[serial]
+    async fn unreadable_bucket_targets_fail_closed_and_stay_distinct_from_absent() {
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let target_sys = BucketTargetSys::get();
+        let unreadable = "targets-unreadable";
+        let absent = "targets-absent";
+        target_sys.delete(unreadable).await;
+        target_sys.delete(absent).await;
+
+        // A readable load publishes this bucket's targets.
+        let mut readable = BucketMetadata::new(unreadable);
+        readable.bucket_target_config = Some(BucketTargets {
+            targets: vec![target(unreadable, "live")],
+        });
+        sync_bucket_target_sys(unreadable, &readable).await;
+        assert_eq!(
+            target_sys
+                .list_bucket_targets(unreadable)
+                .await
+                .expect("readable targets publish")
+                .targets
+                .len(),
+            1
+        );
+
+        // The same bucket reloaded with a blob that cannot be decoded.
+        let mut corrupt = BucketMetadata::new(unreadable);
+        corrupt.bucket_targets_config_json = br#"{"targets":[{"endpoint":"#.to_vec();
+        corrupt
+            .parse_all_configs()
+            .expect("an unreadable targets blob must not fail the metadata load");
+        sys.set(unreadable.to_string(), Arc::new(corrupt)).await;
+
+        assert!(
+            matches!(
+                target_sys.list_bucket_targets(unreadable).await,
+                Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
+            ),
+            "an unreadable configuration must not read as an empty or a missing target set"
+        );
+        assert!(
+            target_sys.list_targets(unreadable, "").await.is_err(),
+            "the admin listing must surface the fault instead of an empty list"
+        );
+        let err = sys
+            .get_bucket_targets_config(unreadable)
+            .await
+            .expect_err("an unreadable targets configuration must not read as a value");
+        assert_ne!(err, Error::ConfigNotFound, "unreadable must not be reported as absent");
+
+        // A bucket that never configured a target keeps its previous behavior.
+        let mut no_targets = BucketMetadata::new(absent);
+        no_targets.parse_all_configs().expect("absent targets parse");
+        sys.set(absent.to_string(), Arc::new(no_targets)).await;
+        assert!(
+            matches!(
+                target_sys.list_bucket_targets(absent).await,
+                Err(BucketTargetError::BucketRemoteTargetNotFound { .. })
+            ),
+            "an absent configuration must still report as a missing target set"
+        );
+        assert!(
+            target_sys
+                .list_targets(absent, "")
+                .await
+                .expect("an absent configuration lists no targets")
+                .is_empty()
+        );
+        assert!(
+            sys.get_bucket_targets_config(absent)
+                .await
+                .expect("an absent targets configuration still reads as an empty set")
+                .is_empty(),
+            "the absent path must keep returning an empty target set, exactly as before"
+        );
+
+        // One bucket's unreadable configuration does not reach another bucket.
+        assert!(!matches!(
+            target_sys.list_bucket_targets(absent).await,
+            Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
+        ));
+
+        // A repaired configuration takes effect on the next load, no restart.
+        let mut repaired = BucketMetadata::new(unreadable);
+        repaired.bucket_target_config = Some(BucketTargets {
+            targets: vec![target(unreadable, "repaired")],
+        });
+        sync_bucket_target_sys(unreadable, &repaired).await;
+        assert_eq!(
+            target_sys
+                .list_bucket_targets(unreadable)
+                .await
+                .expect("a repaired configuration clears the unreadable marker")
+                .targets
+                .len(),
+            1
+        );
+
+        target_sys.delete(unreadable).await;
+        target_sys.delete(absent).await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
@@ -4121,19 +4375,26 @@ mod tests {
 
     const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
+    type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
+    type RecordedOdmHookCall = (String, RecordedOdmConfig);
+
     /// Every `(bucket, config)` the recording hook has seen. Tests filter by
     /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
+    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
 
     fn install_recording_odm_hook() {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
-            Box::new(|bucket, config| {
-                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
+        BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
+            Box::new(|bucket, config_file, config| {
+                assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
+                ODM_HOOK_CALLS.lock().unwrap().push((
+                    bucket.to_string(),
+                    config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
+                ));
             })
         });
     }
 
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
+    fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
         ODM_HOOK_CALLS
             .lock()
             .unwrap()
@@ -4143,54 +4404,6 @@ mod tests {
             .collect()
     }
 
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
-
     /// rustfs/backlog#2148: the publish hook fires on every path that
     /// installs bucket metadata into the cache (set, initial load, peer
     /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
@@ -4204,15 +4417,22 @@ mod tests {
         for dir in &dirs {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
         }
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+
+        let incarnation = Uuid::new_v4();
         let expect_publish = |before: usize, label: &str| {
             let calls = odm_hook_calls(bucket);
             assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
+            assert_eq!(
+                calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
+                Some(ODM_JSON),
+                "{label} must publish the stored bytes"
+            );
+            assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
         };
 
         // set (via persist_new_and_set, which installs through `set`).
         let mut bm = BucketMetadata::new(bucket);
+        bm.bucket_incarnation_id = incarnation;
         bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
             .unwrap();
         let writer = BucketMetadataSys::new(ecstore.clone());
@@ -4254,14 +4474,18 @@ mod tests {
         assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
         assert_eq!(calls.last().unwrap(), &None);
 
-        // A corrupt payload is withdrawn, never published as a config.
+        // Opaque bytes reach the application even if they are not valid JSON.
         let mut corrupt = BucketMetadata::new(bucket);
         corrupt.on_demand_migration_config_json = b"not-json".to_vec();
         let before = odm_hook_calls(bucket).len();
         lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
         let calls = odm_hook_calls(bucket);
         assert_eq!(calls.len(), before + 1);
-        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
+        assert_eq!(
+            calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
+            Some(b"not-json".as_slice()),
+            "the application validates opaque config bytes"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/bucket/mod.rs

@@ -26,11 +26,11 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
-pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;
 pub mod replication;
+pub mod sealed_credentials;
 pub mod tagging;
 pub mod target;
 pub mod utils;

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec {
 
 #[derive(Debug, thiserror::Error)]
 pub enum RemoteS3ClientError {
+    #[error("the {0} backend is not included in this build")]
+    BackendNotCompiled(&'static str),
     #[error("remote endpoint requires credentials")]
     MissingCredentials,
     #[error("{0}")]
@@ -281,9 +283,7 @@ impl Intercept for UserAgentSuffixInterceptor {
 
 /// Builds the SDK config for `spec` without finalizing it, so callers can add
 /// interceptors or (in tests) swap the HTTP client before `build()`.
-pub(crate) async fn build_remote_s3_config(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
+pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
     let Some(credentials) = &spec.credentials else {
         return Err(RemoteS3ClientError::MissingCredentials);
     };
@@ -523,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
     Ok(())
 }
 
-pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
+pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
     validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
 }
 
@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use aws_smithy_async::time::TimeSource;
     use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
     use std::sync::Mutex;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 
     fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
         RemoteS3EndpointSpec {
@@ -824,6 +825,174 @@ mod tests {
         );
     }
 
+    #[derive(Clone, Debug)]
+    struct ClockSkewTimeSource(Arc<AtomicU64>);
+
+    impl TimeSource for ClockSkewTimeSource {
+        fn now(&self) -> SystemTime {
+            SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
+        }
+    }
+
+    #[derive(Clone, Debug)]
+    struct ClockSkewConnector {
+        request_headers: RecordedHeaders,
+        error_code: &'static str,
+        skew_seconds: i64,
+        clock: ClockSkewTimeSource,
+    }
+
+    fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
+        headers
+            .iter()
+            .find(|(key, _)| key.eq_ignore_ascii_case(name))
+            .map(|(_, value)| value.as_str())
+            .unwrap_or_else(|| panic!("signed request must contain {name}"))
+    }
+
+    fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
+        chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
+            .expect("SDK signing timestamp must use the SigV4 format")
+    }
+
+    impl SmithyHttpConnector for ClockSkewConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
+            assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
+            headers.push(
+                request
+                    .headers()
+                    .iter()
+                    .map(|(key, value)| (key.to_string(), value.to_string()))
+                    .collect(),
+            );
+            let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+                + chrono::Duration::seconds(self.skew_seconds);
+            let (status, body) = if headers.len() == 1 {
+                (
+                    403,
+                    format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
+                )
+            } else {
+                (200, String::new())
+            };
+            let response = http::Response::builder()
+                .status(status)
+                .header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
+                .header("content-type", "application/xml")
+                .header("content-length", body.len())
+                .body(SdkBody::from(body))
+                .expect("clock skew fixture response");
+            HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
+        }
+    }
+
+    async fn clock_skew_client(
+        error_code: &'static str,
+        skew_seconds: i64,
+        retry: RemoteS3RetryPolicy,
+    ) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
+        let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
+        let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
+        let connector = SharedHttpConnector::new(ClockSkewConnector {
+            request_headers: Arc::clone(&headers),
+            error_code,
+            skew_seconds,
+            clock: clock.clone(),
+        });
+        let mut spec = spec("s3.example.com", true);
+        spec.retry = retry;
+        let config = build_remote_s3_config(&spec)
+            .await
+            .expect("clock skew fixture uses the production outbound configuration")
+            .http_client(http_client_fn(move |_settings, _components| connector.clone()))
+            .time_source(clock.clone())
+            .build();
+        (S3Client::from_conf(config), headers, clock)
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
+        for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
+            for skew_seconds in [-600, 600] {
+                let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
+                let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
+                client
+                    .get_object()
+                    .bucket("bucket")
+                    .key("object")
+                    .send()
+                    .await
+                    .expect("clock skew GET must retry successfully");
+                assert_eq!(
+                    headers.lock().expect("captured requests").len(),
+                    2,
+                    "{error_code}: GET needs exactly one retry"
+                );
+                clock.0.fetch_add(17, Ordering::SeqCst);
+                // SDK signing time is independent of Tokio's retry/scheduler clock.
+                tokio::time::advance(Duration::from_secs(61)).await;
+                client
+                    .head_bucket()
+                    .bucket("bucket")
+                    .send()
+                    .await
+                    .expect("subsequent HEAD must use the client's cached skew");
+                let headers = headers.lock().expect("captured signed requests");
+                assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
+                assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
+                assert_eq!(
+                    signing_time(&headers[1]),
+                    initial + chrono::Duration::seconds(skew_seconds),
+                    "{error_code}: retry must apply the measured offset exactly"
+                );
+                assert_eq!(
+                    signing_time(&headers[2]),
+                    initial + chrono::Duration::seconds(skew_seconds + 17),
+                    "{error_code}: the next operation must apply cached skew to the advanced signing clock"
+                );
+                let signature = |index: usize| {
+                    recorded_header(&headers[index], "authorization")
+                        .rsplit_once("Signature=")
+                        .expect("SigV4 authorization contains a signature")
+                        .1
+                };
+                assert_ne!(
+                    signature(0),
+                    signature(1),
+                    "{error_code}: retry must be signed again after adjusting its date"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn remote_s3_clock_skew_respects_one_attempt_policy() {
+        use aws_smithy_types::error::metadata::ProvideErrorMetadata;
+
+        for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
+            for retry in [
+                RemoteS3RetryPolicy::Disabled,
+                RemoteS3RetryPolicy::Standard { max_attempts: 1 },
+            ] {
+                let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
+                let error = client
+                    .get_object()
+                    .bucket("bucket")
+                    .key("object")
+                    .send()
+                    .await
+                    .expect_err("clock skew must not override the caller's one-attempt budget");
+                assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
+                assert_eq!(
+                    headers.lock().expect("captured requests").len(),
+                    1,
+                    "{error_code}: {retry:?} must send exactly one request"
+                );
+            }
+        }
+    }
+
     #[test]
     fn path_style_auto_and_path_force_path_style() {
         assert!(PathStyle::Auto.force_path_style());

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -46,7 +46,7 @@ use super::replication_storage_boundary::{
     HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
     ReplicationStorage,
 };
-use super::replication_target_boundary::{ReplicationTargetStore, replication_object_is_ssec_encrypted};
+use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
 use super::replication_versioning_boundary::ReplicationVersioningStore;
 use super::runtime_boundary as runtime_sources;
 use futures_util::stream::{self, StreamExt};
@@ -3084,6 +3084,23 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
 
     let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await {
         Ok(targets) => Some(targets),
+        // A bucket whose persisted target configuration cannot be decoded has
+        // an unknown target set, not an empty one: scheduling against `None`
+        // here would drop every heal for it without a trace
+        // (rustfs/backlog#2282). Report it missed so the object is retried
+        // once the configuration is readable again.
+        Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => {
+            warn!(
+                event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION,
+                bucket,
+                reason = "target_config_unreadable",
+                "Bucket replication targets are unreadable; replication heal queue fails closed"
+            );
+
+            return ReplicationQueueAdmission::Missed;
+        }
         Err(err) => {
             debug!(
                 event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -15,7 +15,8 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
+pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError;
+use crate::bucket::bucket_target_sys::BucketTargetSys;
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
 use http::HeaderMap;

crates/ecstore/src/bucket/sealed_credentials.rs

@@ -0,0 +1,357 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Sealed remote credentials shared by the three stores that persist one
+//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote
+//! tiers (`tier-config.bin`) and on-demand migration sources
+//! (`on-demand-migration.json`).
+//!
+//! The design record is `docs/architecture/remote-credential-sealing-adr.md`.
+//! What this module owns: the versioned envelope, the encryption context that
+//! binds a ciphertext to the record owning it, the sealer registration point,
+//! and the fail-closed error type. What it deliberately does not own: any KMS
+//! call (ECStore does not depend on `rustfs-kms`; the binary installs a
+//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event
+//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and
+//! any decision about which stored field a consumer writes.
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+use std::fmt;
+use std::sync::{Arc, OnceLock};
+
+/// Envelope format this build writes. A reader accepts only versions it
+/// knows; an unknown version is a typed error, never a fallback.
+pub const SEALED_CREDENTIAL_VERSION: u8 = 1;
+
+/// Which store a sealed value belongs to. Part of the encryption context, so
+/// a ciphertext cannot be replayed into a different store.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum SealedCredentialStore {
+    /// `bucket-targets.json` (replication and other bucket targets).
+    BucketTargets,
+    /// `tier-config.bin` (remote tiers).
+    TierConfig,
+    /// `on-demand-migration.json` (migration sources).
+    OnDemandMigration,
+}
+
+impl SealedCredentialStore {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            SealedCredentialStore::BucketTargets => "bucket-targets",
+            SealedCredentialStore::TierConfig => "tier-config",
+            SealedCredentialStore::OnDemandMigration => "on-demand-migration",
+        }
+    }
+}
+
+/// Identity of the record a secret belongs to: the store, its owner (bucket
+/// name, tier name, or target ARN) and the field name. Rendered into the KMS
+/// encryption context so a ciphertext moved between buckets, tiers or fields
+/// fails to decrypt instead of silently authorizing a different remote.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealScope {
+    pub store: SealedCredentialStore,
+    pub owner: String,
+    pub field: &'static str,
+}
+
+impl SealScope {
+    pub fn new(store: SealedCredentialStore, owner: impl Into<String>, field: &'static str) -> Self {
+        Self {
+            store,
+            owner: owner.into(),
+            field,
+        }
+    }
+
+    /// The encryption context handed to the sealer. Keys are stable: they are
+    /// part of the on-disk contract, because a ciphertext only decrypts under
+    /// the same context.
+    pub fn encryption_context(&self) -> BTreeMap<String, String> {
+        BTreeMap::from([
+            ("rustfs:store".to_string(), self.store.as_str().to_string()),
+            ("rustfs:owner".to_string(), self.owner.clone()),
+            ("rustfs:field".to_string(), self.field.to_string()),
+        ])
+    }
+}
+
+/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value
+/// is not a secret, but it is noise in a log line and an operator reading one
+/// should see the key it is wrapped under, not the bytes.
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SealedCredential {
+    /// Envelope version; see [`SEALED_CREDENTIAL_VERSION`].
+    pub v: u8,
+    /// KMS master key id the data key is wrapped under.
+    pub key_id: String,
+    /// Master key version, when the backend reports one. Carried so the KMS
+    /// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell
+    /// stale envelopes apart; nothing here rotates on its own.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub key_version: Option<String>,
+    /// Algorithm label reported by the sealer, for forensics and migration.
+    pub alg: String,
+    /// Ciphertext blob as produced by the sealer, base64 (standard, padded)
+    /// in the JSON stores and raw inside the tier msgpack payload.
+    pub ct: String,
+}
+
+impl fmt::Debug for SealedCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SealedCredential")
+            .field("v", &self.v)
+            .field("key_id", &self.key_id)
+            .field("key_version", &self.key_version)
+            .field("alg", &self.alg)
+            .field("ct", &format_args!("<{} bytes sealed>", self.ct.len()))
+            .finish()
+    }
+}
+
+impl SealedCredential {
+    /// Rejects an envelope this build cannot read. Called before every
+    /// unseal so an unknown version fails here rather than inside a backend.
+    pub fn check_version(&self) -> Result<(), SealedCredentialError> {
+        if self.v == SEALED_CREDENTIAL_VERSION {
+            Ok(())
+        } else {
+            Err(SealedCredentialError::UnsupportedVersion(self.v))
+        }
+    }
+}
+
+/// Why a seal or unseal did not produce a usable value. Every variant is
+/// terminal for the record that carried it: a caller reports the remote as
+/// unusable, and never substitutes a default or empty credential.
+#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum SealedCredentialError {
+    /// No sealer is installed: KMS is not configured, or the process has not
+    /// finished startup. Reading a sealed record is impossible here.
+    #[error("no credential sealer is installed")]
+    NoSealer,
+    /// The stored envelope is from a newer (or otherwise unknown) format.
+    #[error("unsupported sealed credential envelope version {0}")]
+    UnsupportedVersion(u8),
+    /// The stored bytes are not a well-formed envelope.
+    #[error("malformed sealed credential: {0}")]
+    Malformed(String),
+    /// The sealer refused: wrong encryption context, missing key, revoked
+    /// access, or a failed authentication tag.
+    #[error("sealed credential could not be unsealed: {0}")]
+    Kms(String),
+}
+
+/// The KMS-backed half, installed by the binary at startup.
+#[async_trait]
+pub trait CredentialSealer: Send + Sync + 'static {
+    /// Wraps `plaintext` under the scope's encryption context.
+    async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError>;
+
+    /// Unwraps a stored envelope. Must fail when the envelope was sealed
+    /// under a different scope.
+    async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError>;
+}
+
+static CREDENTIAL_SEALER: OnceLock<Arc<dyn CredentialSealer>> = OnceLock::new();
+
+/// Installs the process-wide sealer. Returns `false` when one is already
+/// installed, matching the other ECStore hooks.
+pub fn install_credential_sealer(sealer: Arc<dyn CredentialSealer>) -> bool {
+    CREDENTIAL_SEALER.set(sealer).is_ok()
+}
+
+/// The installed sealer, or `None` when KMS is not wired. Callers that only
+/// need to know whether sealing is possible use this; callers that must have
+/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error.
+pub fn credential_sealer() -> Option<Arc<dyn CredentialSealer>> {
+    CREDENTIAL_SEALER.get().cloned()
+}
+
+/// Seals one secret field. Fails closed: without a sealer the caller must
+/// reject the write rather than persist the secret in clear text after the
+/// operator asked for sealing.
+pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.seal(plaintext, scope).await
+}
+
+/// Unseals one secret field, rejecting an unknown envelope version first.
+pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+    sealed.check_version()?;
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.unseal(sealed, scope).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use parking_lot::Mutex;
+    use std::collections::BTreeMap;
+
+    fn encode_context(context: &BTreeMap<String, String>) -> String {
+        let ordered = context.iter().collect::<BTreeMap<_, _>>();
+        serde_json::to_string(&ordered).expect("context serializes")
+    }
+
+    /// Stands in for the KMS-backed sealer: records the context it was called
+    /// with, and refuses a ciphertext presented under a different one.
+    #[derive(Default)]
+    struct FakeSealer {
+        sealed_contexts: Mutex<Vec<BTreeMap<String, String>>>,
+    }
+
+    #[async_trait]
+    impl CredentialSealer for FakeSealer {
+        async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+            let context = scope.encryption_context();
+            self.sealed_contexts.lock().push(context.clone());
+            let mut bound = encode_context(&context);
+            bound.push('|');
+            bound.push_str(plaintext);
+            Ok(SealedCredential {
+                v: SEALED_CREDENTIAL_VERSION,
+                key_id: "key-1".to_string(),
+                key_version: Some("3".to_string()),
+                alg: "AES-256-GCM".to_string(),
+                ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()),
+            })
+        }
+
+        async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+            let raw = base64_simd::STANDARD
+                .decode_to_vec(sealed.ct.as_bytes())
+                .map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let expected = encode_context(&scope.encryption_context());
+            bound
+                .strip_prefix(&expected)
+                .and_then(|rest| rest.strip_prefix('|'))
+                .map(str::to_string)
+                .ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string()))
+        }
+    }
+
+    fn scope(owner: &str) -> SealScope {
+        SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key")
+    }
+
+    #[tokio::test]
+    async fn seal_round_trips_and_binds_the_scope() {
+        let sealer = Arc::new(FakeSealer::default());
+        let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal");
+        assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION);
+        assert_eq!(sealed.key_version.as_deref(), Some("3"));
+        assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret");
+
+        // The same ciphertext under another bucket must not unseal.
+        let err = sealer
+            .unseal(&sealed, &scope("other-bucket"))
+            .await
+            .expect_err("a ciphertext must not move between owners");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        // Nor under another field of the same record.
+        let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token");
+        let err = sealer
+            .unseal(&sealed, &other_field)
+            .await
+            .expect_err("a ciphertext must not move between fields");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        let contexts = sealer.sealed_contexts.lock();
+        assert_eq!(contexts.len(), 1);
+        assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration");
+        assert_eq!(contexts[0]["rustfs:owner"], "photos");
+        assert_eq!(contexts[0]["rustfs:field"], "secret_key");
+    }
+
+    #[tokio::test]
+    async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() {
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION + 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            sealed.check_version().expect_err("a newer envelope must not be read"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+        // The global helper reports the version, not "no sealer", even in a
+        // process where none is installed.
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+    }
+
+    #[tokio::test]
+    async fn without_a_sealer_both_directions_fail_closed() {
+        // This test binary installs no sealer, so the global helpers must
+        // report NoSealer rather than fall back to clear text.
+        assert!(credential_sealer().is_none(), "no sealer is installed in unit tests");
+        assert_eq!(
+            seal_secret("super-secret", &scope("photos")).await.expect_err("seal"),
+            SealedCredentialError::NoSealer
+        );
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"),
+            SealedCredentialError::NoSealer
+        );
+    }
+
+    #[test]
+    fn debug_and_serde_keep_the_on_disk_shape_stable() {
+        let sealed = SealedCredential {
+            v: 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        // key_version is omitted when absent, so an envelope from a backend
+        // without version history stays compact.
+        assert_eq!(
+            serde_json::to_string(&sealed).expect("serialize"),
+            r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#
+        );
+        let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#)
+            .expect("an envelope without key_version parses");
+        assert_eq!(parsed, sealed);
+
+        let rendered = format!("{sealed:?}");
+        assert!(rendered.contains("key-1"), "{rendered}");
+        assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}");
+    }
+
+    #[test]
+    fn a_malformed_envelope_is_a_typed_error() {
+        let err = serde_json::from_str::<SealedCredential>(r#"{"v":1,"key_id":"key-1"}"#)
+            .map_err(|err| SealedCredentialError::Malformed(err.to_string()))
+            .expect_err("a truncated envelope must not parse");
+        assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}");
+    }
+}

crates/ecstore/src/cluster/rpc/client.rs

@@ -30,6 +30,7 @@ use rustfs_protos::{
     ChannelClass, create_new_channel, get_channel_for_class,
     proto_gen::node_service::{
         heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
+        scanner_control_service_client::ScannerControlServiceClient,
         tier_mutation_control_service_client::TierMutationControlServiceClient,
     },
 };
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
     node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
 }
 
+pub(crate) async fn scanner_control_time_out_client(
+    addr: &str,
+    interceptor: TonicInterceptor,
+) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
+    let interceptor = interceptor.with_rpc_audience(addr)?;
+    let channel = match runtime_sources::cached_node_channel(addr).await {
+        Some(channel) => channel,
+        None => create_new_channel(addr)
+            .await
+            .map_err(|err| crate::error::Error::other(err.to_string()))?,
+    };
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
+    let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
+    Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
+        .max_decoding_message_size(limit)
+        .max_encoding_message_size(limit))
+}
+
 pub async fn heal_control_time_out_client(
     addr: &str,
     interceptor: TonicInterceptor,

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2050,6 +2050,53 @@ impl PeerRestClient {
         .await
     }
 
+    /// Probe only: scoped ACK production requires a durable per-bucket proof.
+    pub async fn scanner_scoped_dirty_usage_capability(
+        &self,
+        owner_id: String,
+        instance_id: String,
+        entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
+    ) -> Result<bool> {
+        use rustfs_protos::scoped_dirty_usage::*;
+        let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
+            challenge: Uuid::new_v4().as_bytes().to_vec().into(),
+            protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
+            owner_id,
+            instance_id,
+            scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
+            probe_only: true,
+            entries,
+        };
+        let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
+        self.finalize_result(
+            async {
+                let mut client = super::client::scanner_control_time_out_client(
+                    &self.grid_host,
+                    TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
+                )
+                .await?;
+                let mut request = Request::new(payload.clone());
+                set_tonic_canonical_body_digest(&mut request, &canonical)?;
+                let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
+                let body = canonical_scoped_dirty_usage_response(&canonical, &response)
+                    .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
+                verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
+                if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
+                    || response.owner_id != payload.owner_id
+                    || response.instance_id != payload.instance_id
+                    || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
+                    || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
+                    || response.cleared != 0
+                {
+                    return Err(Error::other("scoped dirty usage capability response does not match request"));
+                }
+                Ok(response.supported)
+            }
+            .await,
+        )
+        .await
+    }
+
     pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
         let result = self
             .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)

crates/ecstore/src/core/pools.rs

@@ -5493,6 +5493,7 @@ where
     fence.ensure_held()?;
     let mut opts = ObjectOptions {
         max_parity: true,
+        write_completion: crate::object_api::WriteCompletion::TailDrained,
         no_lock: true,
         http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
         ..Default::default()
@@ -14412,6 +14413,7 @@ impl ECStore {
                 encoded.clone(),
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_none_match: Some("*".to_string()),
                         ..Default::default()
@@ -14566,6 +14568,7 @@ impl ECStore {
                 encoded,
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(http_preconditions),
                     ..Default::default()
                 },
@@ -14957,6 +14960,7 @@ impl ECStore {
                 encoded,
                 &ObjectOptions {
                     max_parity: true,
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
                     http_preconditions: Some(HTTPPreconditions {
                         if_match: Some(etag),
                         ..Default::default()

crates/ecstore/src/disk/disk_store.rs

@@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         dst_path: &str,
         external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<RenameDataResp> {
+        self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
+            .await
+            .result
+    }
+}
+
+impl LocalDiskWrapper {
+    pub(in crate::disk) async fn rename_data_observed(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> super::RenameDataObservation {
         let operation = self.clone();
         let src_volume = src_volume.to_owned();
         let src_path = src_path.to_owned();
@@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         } else {
             get_max_timeout_duration()
         };
-        run_owned_mutation(external_guard, move || async move {
-            operation
+        let observed = run_owned_mutation(external_guard, move || async move {
+            let mut preflight_rejection = None;
+            let result = operation
                 .track_disk_health_mutation(
                     "rename_data",
                     DiskMetricMutation::Write,
                     || async {
-                        operation
-                            .disk
-                            .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
-                            .await
+                        // Preserve the former DiskAPI future's single boxing boundary.
+                        let observed =
+                            Box::pin(
+                                operation
+                                    .disk
+                                    .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
+                            )
+                            .await;
+                        preflight_rejection = observed.preflight_rejection;
+                        observed.result
                     },
                     timeout_duration,
                 )
-                .await
+                .await;
+            // Health tracking must observe the real disk error, not an Ok tuple.
+            Ok(super::RenameDataObservation {
+                result,
+                preflight_rejection,
+            })
         })
-        .await
+        .await;
+        observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
     }
 }
 
@@ -2588,6 +2617,46 @@ mod tests {
         assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
     }
 
+    #[tokio::test]
+    async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
+        for source_exists in [false, true] {
+            for guarded in [false, true] {
+                let dir = tempfile::tempdir().expect("temp dir should be created");
+                let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
+                    .expect("endpoint should parse");
+                let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+                if source_exists {
+                    disk.make_volume("source").await.expect("source volume should exist");
+                }
+                let wrapper = LocalDiskWrapper::new(disk, false);
+                let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+                let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
+                let mut file_info = FileInfo::new("object", 1, 0);
+                file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
+                file_info.erasure.index = 1;
+                let observed = wrapper
+                    .rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
+                    .await;
+                assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
+                assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
+                let snapshot = wrapper.metrics_snapshot();
+                assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
+                assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
+                assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
+
+                wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                let observed = wrapper
+                    .rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
+                    .await;
+                assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
+                assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
+                let snapshot = wrapper.metrics_snapshot();
+                assert_eq!(snapshot.total_errors_availability, 1);
+                assert_eq!(snapshot.total_writes, 0);
+            }
+        }
+    }
+
     #[tokio::test]
     async fn local_disk_health_wrapper_counts_returned_availability_errors() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/mod.rs

@@ -75,6 +75,25 @@ use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
 
+/// Local preflight evidence stays outside DiskAPI and the RPC response format.
+pub(crate) struct RenameDataObservation {
+    pub(crate) result: Result<RenameDataResp>,
+    preflight_rejection: Option<local::LocalRenamePreflightRejection>,
+}
+
+impl RenameDataObservation {
+    fn unknown(result: Result<RenameDataResp>) -> Self {
+        Self {
+            result,
+            preflight_rejection: None,
+        }
+    }
+
+    pub(crate) fn rejected_before_publication(&self) -> bool {
+        self.result.is_err() && self.preflight_rejection.is_some()
+    }
+}
+
 const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
 pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
 
@@ -711,6 +730,36 @@ impl Disk {
             .await
     }
 
+    pub(crate) async fn rename_data_borrowed_with_fence_observed(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        scanner_publication_lease_token: Option<Uuid>,
+    ) -> RenameDataObservation {
+        match self {
+            Disk::Local(local_disk) => {
+                local_disk
+                    .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
+                    .await
+            }
+            Disk::Remote(remote_disk) => RenameDataObservation::unknown(
+                remote_disk
+                    .rename_data_borrowed_with_fence(
+                        src_volume,
+                        src_path,
+                        fi,
+                        dst_volume,
+                        dst_path,
+                        scanner_publication_lease_token,
+                    )
+                    .await,
+            ),
+        }
+    }
+
     pub(crate) async fn rename_data_borrowed_with_fence(
         &self,
         src_volume: &str,

crates/ecstore/src/object_api/types.rs

@@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink {
     }
 }
 
+/// Internal PUT completion boundary; this does not change fsync or write quorum.
+#[doc(hidden)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub enum WriteCompletion {
+    /// Return at write quorum when the commit owner can retain its guards.
+    #[default]
+    Quorum,
+    /// Drain the rename fan-out before returning. Minority failures still heal
+    /// after a successful quorum commit; this does not require every disk to succeed.
+    TailDrained,
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -896,6 +908,10 @@ pub struct ObjectOptions {
     /// Persisted bucket incarnation observed before authorization.
     pub expected_bucket_incarnation_id: Option<Uuid>,
     pub no_lock: bool,
+    /// Control-plane writers that immediately read or CAS the same namespace
+    /// key use TailDrained without changing namespace lock ownership.
+    #[doc(hidden)]
+    pub write_completion: WriteCompletion,
     /// True when an upper layer already holds the object read lock before
     /// forwarding a no_lock read to the set layer.
     pub metadata_cache_safe: bool,
@@ -940,6 +956,9 @@ pub struct ObjectOptions {
     pub preserve_etag: Option<String>,
     pub metadata_chg: bool,
     pub http_preconditions: Option<HTTPPreconditions>,
+    /// Internal create-only writes may also preserve an acknowledged deletion.
+    /// Evaluated with `http_preconditions` under the namespace commit lock.
+    pub preserve_delete_marker: bool,
 
     pub delete_replication: Option<ReplicationState>,
     pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,

crates/ecstore/src/runtime/instance.rs

@@ -78,6 +78,21 @@ pub(crate) struct ScannerPublicationLeaseEntry {
     pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
 }
 
+pub(crate) struct NamespaceCommitGuard {
+    ctx: Arc<InstanceContext>,
+    counted: bool,
+}
+
+impl Drop for NamespaceCommitGuard {
+    fn drop(&mut self) {
+        if self.counted {
+            // Publish the new generation before a zero-pending publication probe.
+            self.ctx.advance_namespace_commit_generation();
+            self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
+        }
+    }
+}
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -209,9 +224,13 @@ pub struct InstanceContext {
     /// Last storage-owned movement snapshot observed under the operation
     /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
     scanner_publication_state: AtomicU8,
+    namespace_commits: AtomicU64,
+    namespace_commit_generation: AtomicU64,
     /// Resolves object-encryption material at the application boundary.
     object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
+    #[cfg(test)]
+    suppress_tier_delete_journal_recovery: bool,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
 }
@@ -256,8 +275,12 @@ impl InstanceContext {
             data_movement_generation_exhausted: AtomicBool::new(false),
             data_movement_generation_notify: Arc::new(Notify::new()),
             scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
+            namespace_commits: AtomicU64::new(0),
+            namespace_commit_generation: AtomicU64::new(0),
             object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
+            #[cfg(test)]
+            suppress_tier_delete_journal_recovery: false,
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
         }
@@ -385,6 +408,36 @@ impl InstanceContext {
             && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
     }
 
+    pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
+        let counted = self
+            .namespace_commits
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
+            .is_ok();
+        if counted {
+            self.advance_namespace_commit_generation();
+        } else {
+            self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
+        }
+        Arc::new(NamespaceCommitGuard {
+            ctx: Arc::clone(self),
+            counted,
+        })
+    }
+
+    fn advance_namespace_commit_generation(&self) {
+        let _ = self
+            .namespace_commit_generation
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
+    }
+
+    pub(crate) fn namespace_commit_generation(&self) -> u64 {
+        self.namespace_commit_generation.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn namespace_commits_pending(&self) -> bool {
+        self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
+    }
+
     pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
         self.scanner_publication_state.store(
             if blocked {
@@ -640,12 +693,21 @@ impl InstanceContext {
     }
 
     pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
+        #[cfg(test)]
+        if self.suppress_tier_delete_journal_recovery {
+            return false;
+        }
         self.tier_delete_journal_recovery_stores
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner)
             .insert(store_id)
     }
 
+    #[cfg(test)]
+    pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
+        self.suppress_tier_delete_journal_recovery = true;
+    }
+
     pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
         self.transition_transaction_recovery_stores
             .lock()
@@ -756,6 +818,50 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
+        let first = Arc::new(InstanceContext::new());
+        let other = Arc::new(InstanceContext::new());
+        first.set_scanner_publication_state(false);
+        other.set_scanner_publication_state(false);
+        assert!(first.scanner_publication_state_allowed());
+        let one = first.begin_namespace_commit();
+        let shared_owner = Arc::clone(&one);
+        let two = first.begin_namespace_commit();
+        assert!(first.namespace_commits_pending());
+        assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
+        assert_eq!(first.namespace_commit_generation(), 2);
+        assert!(!other.namespace_commits_pending());
+        assert_eq!(other.namespace_commit_generation(), 0);
+        assert!(other.scanner_publication_state_allowed());
+        drop(one);
+        assert_eq!(first.namespace_commit_generation(), 2);
+        drop(shared_owner);
+        assert!(first.namespace_commits_pending());
+        assert_eq!(first.namespace_commit_generation(), 3);
+        drop(two);
+        assert!(!first.namespace_commits_pending());
+        assert_eq!(first.namespace_commit_generation(), 4);
+        assert!(first.scanner_publication_state_allowed());
+    }
+
+    #[test]
+    fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
+        for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
+            let ctx = Arc::new(InstanceContext::new());
+            ctx.set_scanner_publication_state(false);
+            ctx.namespace_commits.store(count, Ordering::Release);
+            ctx.namespace_commit_generation.store(generation, Ordering::Release);
+            let guard = ctx.begin_namespace_commit();
+            assert!(ctx.namespace_commits_pending());
+            assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
+            drop(guard);
+            assert!(ctx.namespace_commits_pending());
+            assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
+            assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
+        }
+    }
+
     // The SetupType inputs must derive the exact (is_erasure,
     // is_dist_erasure, is_erasure_sd) triples that the original three
     // process-global erasure bools produced via update_erasure_type().
@@ -1073,6 +1179,12 @@ mod tests {
         assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
         assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
         assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
+
+        let mut manual_ctx = InstanceContext::new();
+        manual_ctx.suppress_tier_delete_journal_recovery_for_test();
+        assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
+        assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
+        assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
     }
 
     #[test]

crates/ecstore/src/services/notification_sys.rs

@@ -62,12 +62,27 @@ const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
 const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
 const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
+// Keep this synchronized with the version served by node_service. Including
+// the local member in the minimum prevents an older coordinator from
+// self-authorizing a policy implemented only by newer remote peers.
+const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
+/// Version 5 is reserved for a fleet whose every metadata writer preserves
+/// explicit transition version state and destination identity, and implements
+/// conditional per-generation `xl.meta` writes with strong readback. The node
+/// service must not advertise this version until the conditional writer from
+/// rustfs/backlog#684 is available.
+const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
 type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
 
 fn cross_pool_fence_policy_results(
     peer_epochs: BTreeMap<String, Uuid>,
     minimum_version: u32,
-) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
+) -> (
+    CrossPoolFencePolicyResult,
+    CrossPoolFencePolicyResult,
+    CrossPoolFencePolicyResult,
+    CrossPoolFencePolicyResult,
+) {
     let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
         Ok(peer_epochs.clone())
     } else {
@@ -78,7 +93,18 @@ fn cross_pool_fence_policy_results(
     } else {
         Err(Error::other("decommission target fence policy capability version is unsupported"))
     };
-    (Ok(peer_epochs), journal_result, decommission_target_fence_result)
+    let legacy_transition_state_reconcile_result =
+        if minimum_version >= LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION {
+            Ok(peer_epochs.clone())
+        } else {
+            Err(Error::other("legacy transition state reconcile policy capability version is unsupported"))
+        };
+    (
+        Ok(peer_epochs),
+        journal_result,
+        decommission_target_fence_result,
+        legacy_transition_state_reconcile_result,
+    )
 }
 
 #[derive(Clone, Debug)]
@@ -252,10 +278,21 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
     _permit: FleetCapabilityProofPermit,
 }
 
+/// Effect-window authority for one legacy transition-state reconciliation.
+///
+/// The token intentionally cannot be cloned. Its permit keeps the admitted
+/// fleet generation alive until the caller finishes the final strong
+/// readback, while revocation makes every later validation fail immediately.
+pub struct LegacyTransitionStateReconcileFleetProofToken {
+    token: FleetCapabilityProofToken,
+    _permit: FleetCapabilityProofPermit,
+}
+
 static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
+static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -274,6 +311,10 @@ fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<Fl
     DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
+fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
+    LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
+}
+
 fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
     if let Some(proof) = state.proof.take() {
         proof.generation.revoke();
@@ -444,6 +485,125 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
     stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
 }
 
+/// Acquire one non-cloneable authority that must span the complete reconcile
+/// effect window, including its final strong readback.
+pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let proof = {
+        let state = legacy_transition_state_reconcile_fleet_proof_slot()
+            .read()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, expected_topology, Instant::now())?
+    };
+    let observed_peer_epochs = observe_legacy_transition_state_reconcile_fleet(expected_topology).await?;
+    let state = legacy_transition_state_reconcile_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
+        &state,
+        &proof,
+        expected_topology,
+        &observed_peer_epochs,
+        Instant::now(),
+    )
+    .then_some(proof)
+}
+
+fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
+    state: &FleetCapabilityProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<LegacyTransitionStateReconcileFleetProofToken> {
+    let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
+    let permit = state.proof.as_ref()?.generation.try_acquire()?;
+    Some(LegacyTransitionStateReconcileFleetProofToken { token, _permit: permit })
+}
+
+async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
+    let notification_sys = get_global_notification_sys()?;
+    let (peer_epochs, minimum_version) = timeout(
+        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+        notification_sys.probe_cross_pool_fence_fleet(expected_topology),
+    )
+    .await
+    .ok()?
+    .ok()?;
+    let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peer_epochs, minimum_version);
+    reconcile_result.ok()
+}
+
+/// Revalidate the exact fleet generation captured by a reconcile token with a
+/// fresh synchronous observation. Callers must await this before each
+/// conditional metadata write and after the final strong readback.
+pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
+    proof: &LegacyTransitionStateReconcileFleetProofToken,
+) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    legacy_transition_state_reconcile_fleet_proof_matches_with_observer(
+        legacy_transition_state_reconcile_fleet_proof_slot(),
+        proof,
+        expected_topology,
+        || observe_legacy_transition_state_reconcile_fleet(expected_topology),
+    )
+    .await
+}
+
+async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
+    slot: &std::sync::RwLock<FleetCapabilityProofState>,
+    proof: &LegacyTransitionStateReconcileFleetProofToken,
+    expected_topology: &str,
+    observe: F,
+) -> bool
+where
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = Option<BTreeMap<String, Uuid>>>,
+{
+    {
+        let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
+        if !legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
+            return false;
+        }
+    }
+    let Some(observed_peer_epochs) = observe().await else {
+        return false;
+    };
+    let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
+    legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
+        &state,
+        proof,
+        expected_topology,
+        &observed_peer_epochs,
+        Instant::now(),
+    )
+}
+
+fn legacy_transition_state_reconcile_fleet_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &LegacyTransitionStateReconcileFleetProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof._permit.generation.is_accepting()
+        && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
+        && state
+            .proof
+            .as_ref()
+            .is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
+}
+
+fn legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
+    state: &FleetCapabilityProofState,
+    proof: &LegacyTransitionStateReconcileFleetProofToken,
+    expected_topology: &str,
+    observed_peer_epochs: &BTreeMap<String, Uuid>,
+    now: Instant,
+) -> bool {
+    legacy_transition_state_reconcile_fleet_proof_matches_at(state, proof, expected_topology, now)
+        && proof.token.peer_epochs.as_ref() == observed_peer_epochs
+}
+
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
     let state = tier_delete_journal_fleet_proof_slot()
@@ -766,6 +926,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 cross_pool_fence_fleet_proof_slot(),
                 tier_delete_journal_fleet_proof_slot(),
                 decommission_target_fence_fleet_proof_slot(),
+                legacy_transition_state_reconcile_fleet_proof_slot(),
             ] {
                 mark_fleet_capability_topology_conflict(slot);
             }
@@ -798,11 +959,12 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
                 None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
             };
-            let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
+            let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
                 Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
                 Err(err) => {
                     let message = err.to_string();
                     (
+                        Err(Error::other(message.clone())),
                         Err(Error::other(message.clone())),
                         Err(Error::other(message.clone())),
                         Err(Error::other(message)),
@@ -818,6 +980,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
                 revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
                 revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
+                revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -880,6 +1043,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                     "notification capability probe"
                 );
             }
+            if !topology_conflict
+                && let Some(err) = publish_fleet_capability_probe_result(
+                    legacy_transition_state_reconcile_fleet_proof_slot(),
+                    &topology_fingerprint,
+                    reconcile_result,
+                    Instant::now(),
+                )
+            {
+                debug!(
+                    event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    capability = "legacy_transition_state_reconcile_v1",
+                    state = "failed_closed",
+                    error = %err,
+                    "notification capability probe"
+                );
+            }
             sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
         }
     });
@@ -959,7 +1140,7 @@ impl NotificationSys {
             client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
         });
         let mut peer_epochs = BTreeMap::new();
-        let mut minimum_version = u32::MAX;
+        let mut minimum_version = LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION;
         for result in join_all(probes).await {
             let (peer, version, epoch) = result?;
             if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
@@ -968,11 +1149,6 @@ impl NotificationSys {
             minimum_version = minimum_version.min(version);
             insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
         }
-        // A single-node deployment has no remote member to lower the local
-        // policy version advertised by this binary.
-        if minimum_version == u32::MAX {
-            minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
-        }
         Ok((peer_epochs, minimum_version))
     }
 }
@@ -3190,20 +3366,36 @@ mod tests {
     #[test]
     fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
         let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
-        let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
+        let (generic_v2, journal_v2, decommission_v2, reconcile_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
         assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
         assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
         assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
+        assert!(reconcile_v2.is_err(), "v2 cannot authorize legacy transition-state reconciliation");
 
-        let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
+        let (generic_v3, journal_v3, decommission_v3, reconcile_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
         assert!(generic_v3.is_ok());
         assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
         assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
+        assert!(reconcile_v3.is_err());
 
-        let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
+        let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
+            cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
         assert!(generic_v4.is_ok());
         assert!(journal_v4.is_ok());
         assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
+        assert!(
+            reconcile_v4.is_err(),
+            "the current local policy lacks the conditional xl.meta writer required by reconcile"
+        );
+
+        let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
+        assert!(generic_v5.is_ok());
+        assert!(journal_v5.is_ok());
+        assert!(decommission_v5.is_ok());
+        assert!(
+            reconcile_v5.is_ok(),
+            "only an all-v5 fleet preserves destination identity and conditional reconcile writes"
+        );
     }
 
     #[test]
@@ -3458,6 +3650,234 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_transition_state_reconcile_admits_only_compatible_single_and_multi_node_fleets() {
+        let now = Instant::now();
+        for peers in [
+            BTreeMap::new(),
+            BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
+        ] {
+            let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+            let (_, _, _, result) =
+                cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+            assert!(publish_fleet_capability_probe_result(&slot, "topology-a", result, now).is_none());
+
+            let admitted = {
+                let state = slot.read().expect("reconcile proof slot should not poison");
+                acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                    .expect("an all-compatible fleet should admit reconciliation")
+            };
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            assert!(legacy_transition_state_reconcile_fleet_proof_matches_at(
+                &state,
+                &admitted,
+                "topology-a",
+                now,
+            ));
+        }
+    }
+
+    #[test]
+    fn legacy_transition_state_reconcile_restart_drains_concurrent_effect_windows() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let (_, _, _, original_result) =
+            cross_pool_fence_policy_results(original_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
+
+        let (first, second) = {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            (
+                acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                    .expect("the first reconcile writer should be admitted"),
+                acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                    .expect("the second reconcile writer should be admitted"),
+            )
+        };
+
+        let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let (_, _, _, restarted_result) =
+            cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        let blocked =
+            publish_fleet_capability_probe_result(&slot, "topology-a", restarted_result, now + Duration::from_millis(1))
+                .expect("a restarted member must revoke the old generation and wait for both writers");
+        assert!(blocked.to_string().contains("previous generation to drain"));
+        {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            assert!(state.proof.is_none());
+            assert!(state.draining_generation.is_some());
+            assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
+                &state,
+                &first,
+                "topology-a",
+                now + Duration::from_millis(1),
+            ));
+            assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
+                &state,
+                &second,
+                "topology-a",
+                now + Duration::from_millis(1),
+            ));
+        }
+
+        drop(first);
+        let (_, _, _, still_blocked_result) =
+            cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", still_blocked_result, now + Duration::from_millis(2),)
+                .is_some(),
+            "one remaining writer must keep the successor generation closed"
+        );
+
+        drop(second);
+        let (_, _, _, admitted_result) =
+            cross_pool_fence_policy_results(restarted_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", admitted_result, now + Duration::from_millis(3),)
+                .is_none(),
+            "the restarted generation may publish only after every old writer drains"
+        );
+    }
+
+    #[test]
+    fn legacy_transition_state_reconcile_fresh_observation_closes_the_polling_window() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let (_, _, _, original_result) =
+            cross_pool_fence_policy_results(original_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
+        let admitted = {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                .expect("the original fleet should admit reconciliation")
+        };
+
+        let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let state = slot.read().expect("reconcile proof slot should not poison");
+        assert!(
+            legacy_transition_state_reconcile_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+            "the periodic cache has not observed the restart yet"
+        );
+        assert!(!legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
+            &state,
+            &admitted,
+            "topology-a",
+            &restarted_peers,
+            now,
+        ));
+
+        let (_, _, _, downgraded) = cross_pool_fence_policy_results(original_peers, 4);
+        assert!(
+            downgraded.is_err(),
+            "a synchronous observation of a downgraded peer must fail before any cached proof can authorize a write"
+        );
+    }
+
+    #[tokio::test]
+    async fn legacy_transition_state_reconcile_invalid_token_skips_fleet_observation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
+        let admitted = {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                .expect("the original fleet should admit reconciliation")
+        };
+        revoke_fleet_capability_proof(&slot);
+
+        assert!(
+            !legacy_transition_state_reconcile_fleet_proof_matches_with_observer(&slot, &admitted, "topology-a", || async {
+                panic!("an invalid local generation must not trigger a fleet observation");
+            },)
+            .await
+        );
+    }
+
+    #[test]
+    fn legacy_transition_state_reconcile_membership_and_topology_changes_revoke_authority() {
+        let now = Instant::now();
+        for replacement in [
+            BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
+            BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]),
+        ] {
+            let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+            let original = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+            assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
+            let admitted = {
+                let state = slot.read().expect("reconcile proof slot should not poison");
+                acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                    .expect("the original fleet should admit reconciliation")
+            };
+
+            assert!(
+                publish_fleet_capability_probe_result(&slot, "topology-a", Ok(replacement), now + Duration::from_millis(1),)
+                    .is_some(),
+                "membership or process-epoch replacement must wait for the admitted writer"
+            );
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
+                &state,
+                &admitted,
+                "topology-a",
+                now + Duration::from_millis(1),
+            ));
+        }
+
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(BTreeMap::new()), now).is_none());
+        let admitted = {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                .expect("the original topology should admit reconciliation")
+        };
+        mark_fleet_capability_topology_conflict(&slot);
+        let state = slot.read().expect("reconcile proof slot should not poison");
+        assert!(state.topology_conflict);
+        assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
+            &state,
+            &admitted,
+            "topology-a",
+            now,
+        ));
+    }
+
+    #[test]
+    fn legacy_transition_state_reconcile_capability_downgrade_fails_closed() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let (_, _, _, compatible_result) =
+            cross_pool_fence_policy_results(peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", compatible_result, now).is_none());
+        let admitted = {
+            let state = slot.read().expect("reconcile proof slot should not poison");
+            acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
+                .expect("v5 should admit reconciliation")
+        };
+
+        let (_, _, _, downgraded_result) =
+            cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION - 1);
+        let err = publish_fleet_capability_probe_result(&slot, "topology-a", downgraded_result, now + Duration::from_millis(1))
+            .expect("a v4 member must revoke reconcile authority");
+        assert!(err.to_string().contains("reconcile policy capability version is unsupported"));
+        let state = slot.read().expect("reconcile proof slot should not poison");
+        assert!(state.proof.is_none());
+        assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
+            &state,
+            &admitted,
+            "topology-a",
+            now + Duration::from_millis(1),
+        ));
+        assert!(
+            acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now + Duration::from_millis(1),)
+                .is_none(),
+            "a downgraded fleet must remain inspect-only"
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
         let now = Instant::now();
@@ -3539,6 +3959,57 @@ mod tests {
         assert!(err.to_string().contains("incomplete"));
     }
 
+    #[tokio::test]
+    async fn legacy_transition_state_reconcile_probe_rejects_missing_or_unreachable_members() {
+        let missing = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+        let missing_err = missing
+            .probe_cross_pool_fence_fleet("topology-a")
+            .await
+            .expect_err("a missing member slot must prevent reconcile capability proof");
+        assert!(missing_err.to_string().contains("incomplete"));
+
+        let unreachable = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: vec![None, None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+        let unreachable_err = unreachable
+            .probe_cross_pool_fence_fleet("topology-a")
+            .await
+            .expect_err("an unreachable member must prevent reconcile capability proof");
+        assert!(unreachable_err.to_string().contains("unreachable"));
+    }
+
+    #[tokio::test]
+    async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
+        let notification_sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![None],
+            peer_topology_hosts: Vec::new(),
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+        let (peers, minimum_version) = notification_sys
+            .probe_cross_pool_fence_fleet("topology-a")
+            .await
+            .expect("a single-node capability probe should complete");
+        assert!(peers.is_empty());
+        assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
+        let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
+        assert!(
+            reconcile_result.is_err(),
+            "the current node must not self-authorize reconcile before the conditional writer lands"
+        );
+    }
+
     fn build_props(endpoint: &str) -> ServerProperties {
         ServerProperties {
             endpoint: endpoint.to_string(),

crates/ecstore/src/services/tier/mod.rs

@@ -21,9 +21,11 @@ pub mod tier_gen;
 pub mod tier_handlers;
 pub(crate) mod tier_mutation_intent;
 pub mod tier_mutation_peer;
+pub(crate) mod tier_probe_intent;
 pub mod warm_backend;
 pub mod warm_backend_aliyun;
 pub mod warm_backend_azure;
+#[cfg(feature = "gcs")]
 pub mod warm_backend_gcs;
 pub mod warm_backend_huaweicloud;
 pub mod warm_backend_minio;

crates/ecstore/src/services/tier/test_util.rs

@@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend {
         Ok(version)
     }
 
-    async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
         self.precondition().await?;
         let barrier = self.inner.get_barrier.lock().await.take();
         if let Some(barrier) = barrier {
@@ -719,6 +719,9 @@ impl WarmBackend for MockWarmBackend {
         let Some(stored) = objects.get(object) else {
             return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
         };
+        if !rv.is_empty() && stored.remote_version_id != rv {
+            return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
+        }
         let bytes = &stored.bytes;
 
         let start = opts.start_offset.max(0) as usize;

crates/ecstore/src/services/tier/tier_admin.rs

@@ -15,28 +15,204 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 
-#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[derive(Serialize, Deserialize, Default, Clone)]
+#[serde(default)]
+pub struct TierServicePrincipalAuth {
+    #[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
+    pub tenant_id: String,
+    #[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
+    pub client_id: String,
+    #[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
+    pub client_secret: String,
+}
+
+impl TierServicePrincipalAuth {
+    pub(crate) fn is_empty(&self) -> bool {
+        self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
+    }
+}
+
+impl std::fmt::Debug for TierServicePrincipalAuth {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierServicePrincipalAuth")
+            .field("tenant_id", &self.tenant_id)
+            .field("client_id", &self.client_id)
+            .field("client_secret", &"REDACTED")
+            .finish()
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Clone)]
 #[serde(default)]
 pub struct TierCreds {
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "access", alias = "accessKey")]
     pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secret", alias = "secretKey")]
     pub secret_key: String,
 
-    #[serde(rename = "awsRole")]
+    #[serde(rename = "awsrole", alias = "awsRole")]
     pub aws_role: bool,
-    #[serde(rename = "awsRoleWebIdentityTokenFile")]
+    #[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
     pub aws_role_web_identity_token_file: String,
-    #[serde(rename = "awsRoleArn")]
+    #[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
     pub aws_role_arn: String,
 
-    //azsp: ServicePrincipalAuth,
+    #[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
+    pub azure_service_principal: TierServicePrincipalAuth,
 
-    //#[serde(rename = "credsJson")]
+    #[serde(
+        rename = "creds",
+        alias = "credsJson",
+        alias = "credsJSON",
+        alias = "creds_json",
+        default,
+        skip_serializing_if = "Vec::is_empty",
+        with = "base64_bytes"
+    )]
     pub creds_json: Vec<u8>,
 }
+
+impl std::fmt::Debug for TierCreds {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierCreds")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &"REDACTED")
+            .field("aws_role", &self.aws_role)
+            .field(
+                "aws_role_web_identity_token_file",
+                &(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
+            )
+            .field("aws_role_arn", &self.aws_role_arn)
+            .field("azure_service_principal", &self.azure_service_principal)
+            .field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
+            .finish()
+    }
+}
+
+mod base64_bytes {
+    use super::*;
+
+    pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
+    }
+
+    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum EncodedBytes {
+            Base64(String),
+            Legacy(Vec<u8>),
+        }
+
+        match EncodedBytes::deserialize(deserializer)? {
+            EncodedBytes::Base64(value) => base64_simd::STANDARD
+                .decode_to_vec(value.as_bytes())
+                .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
+                .map_err(de::Error::custom),
+            EncodedBytes::Legacy(value) => Ok(value),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
+        let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "access": "access",
+            "secret": "secret",
+            "awsrole": false,
+            "creds": encoded,
+        }))
+        .expect("madmin tier credentials should decode");
+
+        assert_eq!(creds.access_key, "access");
+        assert_eq!(creds.secret_key, "secret");
+        assert_eq!(creds.creds_json.as_slice(), service_account);
+
+        let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
+        assert_eq!(wire["access"], "access");
+        assert_eq!(wire["secret"], "secret");
+        assert_eq!(wire["creds"], encoded);
+        assert!(wire.get("accessKey").is_none());
+        assert!(wire.get("secretKey").is_none());
+
+        let legacy: TierCreds = serde_json::from_value(serde_json::json!({
+            "accessKey": "legacy-access",
+            "secretKey": "legacy-secret",
+            "credsJson": service_account,
+        }))
+        .expect("the former RustFS field names and byte-array encoding should remain readable");
+        assert_eq!(legacy.access_key, "legacy-access");
+        assert_eq!(legacy.secret_key, "legacy-secret");
+        assert_eq!(legacy.creds_json.as_slice(), service_account);
+    }
+
+    #[test]
+    fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
+        for encoder in [
+            base64_simd::STANDARD,
+            base64_simd::STANDARD_NO_PAD,
+            base64_simd::URL_SAFE,
+            base64_simd::URL_SAFE_NO_PAD,
+        ] {
+            let encoded = encoder.encode_to_string(service_account);
+            let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
+                .expect("all supported madmin base64 forms should decode");
+            assert_eq!(creds.creds_json, service_account);
+        }
+    }
+
+    #[test]
+    fn tier_creds_debug_redacts_secret_payloads() {
+        let creds = TierCreds {
+            access_key: "access".to_string(),
+            secret_key: "tier-secret-value".to_string(),
+            aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
+            creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
+            ..Default::default()
+        };
+
+        let rendered = format!("{creds:?}");
+        assert!(!rendered.contains("tier-secret-value"));
+        assert!(!rendered.contains("/var/run/private-token"));
+        assert!(!rendered.contains("gcs-private-key-value"));
+    }
+
+    #[test]
+    fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "azSP": {
+                "TenantID": "tenant",
+                "ClientID": "client",
+                "ClientSecret": "service-principal-secret"
+            }
+        }))
+        .expect("canonical madmin azure service principal credentials should decode");
+
+        assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
+        assert_eq!(creds.azure_service_principal.client_id, "client");
+        assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
+        let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
+        assert_eq!(wire["azSP"]["TenantID"], "tenant");
+        assert_eq!(wire["azSP"]["ClientID"], "client");
+        assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
+        assert!(!format!("{creds:?}").contains("service-principal-secret"));
+    }
+}

crates/ecstore/src/services/tier/tier_config.rs

@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
 pub enum TierType {
     #[default]
     Unsupported,
-    #[serde(rename = "s3")]
+    #[serde(rename = "s3", alias = "S3")]
     S3,
     #[serde(rename = "wasabi")]
     Wasabi,
@@ -58,7 +58,7 @@ pub enum TierType {
     Huaweicloud,
     #[serde(rename = "azure")]
     Azure,
-    #[serde(rename = "gcs")]
+    #[serde(rename = "gcs", alias = "GCS")]
     GCS,
     #[serde(rename = "r2")]
     R2,
@@ -138,16 +138,18 @@ impl TierType {
     }
 }
 
-#[derive(Default, Debug, Serialize, Deserialize)]
+pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
+
+#[derive(Default, Serialize, Deserialize)]
 #[serde(default)]
 pub struct TierConfig {
     #[serde(skip)]
     pub version: String,
-    #[serde(rename = "type")]
+    #[serde(rename = "type", alias = "Type")]
     pub tier_type: TierType,
-    #[serde(skip)]
+    #[serde(rename = "Name", alias = "name", skip_serializing)]
     pub name: String,
-    #[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
     pub s3: Option<TierS3>,
     #[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
     pub wasabi: Option<TierWasabi>,
@@ -159,7 +161,7 @@ pub struct TierConfig {
     pub huaweicloud: Option<TierHuaweicloud>,
     #[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
     pub azure: Option<TierAzure>,
-    #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
     pub gcs: Option<TierGCS>,
     #[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
     pub r2: Option<TierR2>,
@@ -170,109 +172,91 @@ pub struct TierConfig {
 }
 
 impl Clone for TierConfig {
-    fn clone(&self) -> TierConfig {
-        let mut s3 = None;
-        let mut wasabi = None;
-        let mut r = None;
-        let mut compatible_backend = None;
-        let mut aliyun = None;
-        let mut tencent = None;
-        let mut huaweicloud = None;
-        let mut azure = None;
-        let mut gcs = None;
-        let mut r2 = None;
-        match self.tier_type {
-            TierType::S3 => {
-                if let Some(s3_) = self.s3.as_ref() {
-                    let mut s3_clone = s3_.clone();
-                    s3_clone.secret_key = "REDACTED".to_string();
-                    s3 = Some(s3_clone);
-                }
-            }
-            TierType::Wasabi => {
-                if let Some(wasabi_) = self.wasabi.as_ref() {
-                    let mut wasabi_clone = wasabi_.clone();
-                    wasabi_clone.secret_key = "REDACTED".to_string();
-                    wasabi = Some(wasabi_clone);
-                }
-            }
-            TierType::RustFS => {
-                if let Some(r_) = self.rustfs.as_ref() {
-                    let mut r_clone = r_.clone();
-                    r_clone.secret_key = "REDACTED".to_string();
-                    r = Some(r_clone);
-                }
-            }
-            TierType::MinIO => {
-                if let Some(compatible_backend_) = self.minio.as_ref() {
-                    let mut compatible_backend_clone = compatible_backend_.clone();
-                    compatible_backend_clone.secret_key = "REDACTED".to_string();
-                    compatible_backend = Some(compatible_backend_clone);
-                }
-            }
-            TierType::Aliyun => {
-                if let Some(aliyun_) = self.aliyun.as_ref() {
-                    let mut aliyun_clone = aliyun_.clone();
-                    aliyun_clone.secret_key = "REDACTED".to_string();
-                    aliyun = Some(aliyun_clone);
-                }
-            }
-            TierType::Tencent => {
-                if let Some(tencent_) = self.tencent.as_ref() {
-                    let mut tencent_clone = tencent_.clone();
-                    tencent_clone.secret_key = "REDACTED".to_string();
-                    tencent = Some(tencent_clone);
-                }
-            }
-            TierType::Huaweicloud => {
-                if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
-                    let mut huaweicloud_clone = huaweicloud_.clone();
-                    huaweicloud_clone.secret_key = "REDACTED".to_string();
-                    huaweicloud = Some(huaweicloud_clone);
-                }
-            }
-            TierType::Azure => {
-                if let Some(azure_) = self.azure.as_ref() {
-                    let mut azure_clone = azure_.clone();
-                    azure_clone.secret_key = "REDACTED".to_string();
-                    azure = Some(azure_clone);
-                }
-            }
-            TierType::GCS => {
-                if let Some(gcs_) = self.gcs.as_ref() {
-                    let mut gcs_clone = gcs_.clone();
-                    gcs_clone.creds = "REDACTED".to_string();
-                    gcs = Some(gcs_clone);
-                }
-            }
-            TierType::R2 => {
-                if let Some(r2_) = self.r2.as_ref() {
-                    let mut r2_clone = r2_.clone();
-                    r2_clone.secret_key = "REDACTED".to_string();
-                    r2 = Some(r2_clone);
-                }
-            }
-            _ => (),
-        }
-        TierConfig {
-            version: self.version.clone(),
-            tier_type: self.tier_type.clone(),
-            name: self.name.clone(),
-            s3,
-            wasabi,
-            rustfs: r,
-            minio: compatible_backend,
-            aliyun,
-            tencent,
-            huaweicloud,
-            azure,
-            gcs,
-            r2,
-        }
+    fn clone(&self) -> Self {
+        self.redacted()
     }
 }
 
 impl TierConfig {
+    pub(crate) fn redacted(&self) -> Self {
+        let mut redacted = Self {
+            version: self.version.clone(),
+            tier_type: self.tier_type.clone(),
+            name: self.name.clone(),
+            ..Default::default()
+        };
+        match self.tier_type {
+            TierType::S3 => {
+                redacted.s3 = self.s3.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.aws_role_web_identity_token_file.is_empty() {
+                        backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::Wasabi => {
+                redacted.wasabi = self.wasabi.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::RustFS => {
+                redacted.rustfs = self.rustfs.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::MinIO => {
+                redacted.minio = self.minio.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Aliyun => {
+                redacted.aliyun = self.aliyun.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Tencent => {
+                redacted.tencent = self.tencent.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Huaweicloud => {
+                redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Azure => {
+                redacted.azure = self.azure.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.sp_auth.client_secret.is_empty() {
+                        backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::GCS => {
+                redacted.gcs = self.gcs.clone().map(|mut backend| {
+                    backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::R2 => {
+                redacted.r2 = self.r2.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Unsupported => {}
+        }
+        redacted
+    }
+
     pub(crate) fn clone_with_credentials(&self) -> Self {
         Self {
             version: self.version.clone(),
@@ -372,31 +356,61 @@ impl TierConfig {
     }
 }
 
+impl std::fmt::Debug for TierConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let redacted = self.redacted();
+        f.debug_struct("TierConfig")
+            .field("version", &redacted.version)
+            .field("tier_type", &redacted.tier_type)
+            .field("name", &redacted.name)
+            .field("s3", &redacted.s3)
+            .field("wasabi", &redacted.wasabi)
+            .field("aliyun", &redacted.aliyun)
+            .field("tencent", &redacted.tencent)
+            .field("huaweicloud", &redacted.huaweicloud)
+            .field("azure", &redacted.azure)
+            .field("gcs", &redacted.gcs)
+            .field("r2", &redacted.r2)
+            .field("rustfs", &redacted.rustfs)
+            .field("minio", &redacted.minio)
+            .finish()
+    }
+}
+
 //type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
 
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierS3 {
+    #[serde(alias = "Name")]
     pub name: String,
+    #[serde(alias = "Endpoint")]
     pub endpoint: String,
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "accessKey", alias = "AccessKey")]
     pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secretKey", alias = "SecretKey")]
     pub secret_key: String,
+    #[serde(alias = "Bucket")]
     pub bucket: String,
+    #[serde(alias = "Prefix")]
     pub prefix: String,
+    #[serde(alias = "Region")]
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
     pub aws_role: bool,
-    #[serde(skip)]
+    #[serde(
+        rename = "AWSRoleWebIdentityTokenFile",
+        alias = "awsRoleWebIdentityTokenFile",
+        skip_serializing
+    )]
     pub aws_role_web_identity_token_file: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
     pub aws_role_arn: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
     pub aws_role_session_name: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
     pub aws_role_duration_seconds: i32,
 }
 
@@ -623,8 +637,11 @@ pub struct TierHuaweicloud {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct ServicePrincipalAuth {
+    #[serde(alias = "TenantID")]
     pub tenant_id: String,
+    #[serde(alias = "ClientID")]
     pub client_id: String,
+    #[serde(alias = "ClientSecret")]
     pub client_secret: String,
 }
 
@@ -640,9 +657,9 @@ pub struct TierAzure {
     pub bucket: String,
     pub prefix: String,
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
-    #[serde(rename = "spAuth")]
+    #[serde(rename = "spAuth", alias = "SPAuth")]
     pub sp_auth: ServicePrincipalAuth,
 }
 
@@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierGCS {
+    #[serde(alias = "Name")]
     pub name: String,
+    #[serde(alias = "Endpoint")]
     pub endpoint: String,
-    #[serde(rename = "creds")]
+    #[serde(rename = "creds", alias = "Creds")]
     pub creds: String,
+    #[serde(alias = "Bucket")]
     pub bucket: String,
+    #[serde(alias = "Prefix")]
     pub prefix: String,
+    #[serde(alias = "Region")]
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
 }
 
@@ -725,6 +747,43 @@ pub struct TierR2 {
 mod tests {
     use super::*;
 
+    #[test]
+    fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
+        let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
+        let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
+        assert!(matches!(s3, TierType::S3));
+        assert!(matches!(gcs, TierType::GCS));
+        assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
+        assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
+    }
+
+    #[test]
+    fn azure_service_principal_accepts_canonical_madmin_field_names() {
+        for field in ["TenantID", "ClientID", "ClientSecret"] {
+            let mut sp_auth = serde_json::Map::new();
+            sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
+            let config: TierConfig = serde_json::from_value(serde_json::json!({
+                "type": "azure",
+                "Name": "COLD-AZURE",
+                "azure": {
+                    "name": "COLD-AZURE",
+                    "endpoint": "https://azure.example.invalid",
+                    "accessKey": "account",
+                    "secretKey": "key",
+                    "bucket": "archive",
+                    "SPAuth": sp_auth
+                }
+            }))
+            .expect("mixed RustFS/madmin Azure payload should decode");
+            let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
+
+            assert!(
+                !sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
+                "canonical {field} must not be silently discarded"
+            );
+        }
+    }
+
     fn wasabi_config() -> TierWasabi {
         TierWasabi {
             name: "COLD-WASABI".to_string(),
@@ -838,9 +897,14 @@ mod tests {
         let config = TierConfig {
             tier_type: TierType::Wasabi,
             wasabi: Some(wasabi_config()),
+            rustfs: Some(TierRustFS {
+                access_key: "inactive-access".to_string(),
+                secret_key: "inactive-secret".to_string(),
+                ..Default::default()
+            }),
             ..Default::default()
         };
-        let redacted = config.clone();
+        let redacted = config.redacted();
         assert_eq!(
             redacted
                 .wasabi
@@ -849,21 +913,46 @@ mod tests {
                 .secret_key,
             "REDACTED"
         );
+        assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
+        let cloned = config.clone();
+        assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
+        assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
+        let preserved = config.clone_with_credentials();
         assert_eq!(
-            config
-                .clone_with_credentials()
+            preserved
                 .wasabi
                 .as_ref()
-                .expect("credential-bearing Wasabi payload should remain")
+                .expect("credential-bearing Wasabi snapshot should remain")
                 .secret_key,
             "secret"
         );
+        assert_eq!(
+            preserved
+                .rustfs
+                .expect("credential-bearing snapshots should preserve inactive provider data")
+                .secret_key,
+            "inactive-secret"
+        );
 
         let mut debug_config = wasabi_config();
         debug_config.secret_key = "wasabi-debug-secret-value".to_string();
         let debug = format!("{debug_config:?}");
         assert!(debug.contains("REDACTED"));
         assert!(!debug.contains("wasabi-debug-secret-value"));
+
+        let debug = format!(
+            "{:?}",
+            TierConfig {
+                tier_type: TierType::RustFS,
+                rustfs: Some(TierRustFS {
+                    secret_key: "rustfs-debug-secret-value".to_string(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }
+        );
+        assert!(debug.contains("REDACTED"));
+        assert!(!debug.contains("rustfs-debug-secret-value"));
     }
 
     #[test]
@@ -894,7 +983,7 @@ mod tests {
         assert_eq!(encoded, expected);
         let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
         assert!(matches!(decoded.tier_type, TierType::Wasabi));
-        let redacted = config.clone();
+        let redacted = config.redacted();
         assert_eq!(
             config
                 .wasabi
@@ -916,4 +1005,87 @@ mod tests {
             "REDACTED"
         );
     }
+
+    #[test]
+    fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
+        let cases = [
+            (
+                "s3",
+                TierConfig {
+                    tier_type: TierType::S3,
+                    s3: Some(TierS3 {
+                        secret_key: "s3-secret-bytes".to_string(),
+                        aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["s3-secret-bytes", "/var/run/s3-private-token"],
+            ),
+            (
+                "gcs",
+                TierConfig {
+                    tier_type: TierType::GCS,
+                    gcs: Some(TierGCS {
+                        creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["gcs-private-key-bytes"],
+            ),
+            (
+                "azure",
+                TierConfig {
+                    tier_type: TierType::Azure,
+                    azure: Some(TierAzure {
+                        secret_key: "azure-account-secret-bytes".to_string(),
+                        sp_auth: ServicePrincipalAuth {
+                            client_secret: "azure-client-secret-bytes".to_string(),
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
+            ),
+        ];
+
+        for (provider, config, secrets) in cases {
+            let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
+            let debug = format!("{config:?}");
+            assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
+            assert!(
+                debug.contains(TIER_CREDENTIAL_REDACTED),
+                "{provider} Debug output should be visibly redacted"
+            );
+            for secret in secrets {
+                assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
+                assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
+            }
+        }
+    }
+
+    #[test]
+    fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
+        let config = TierConfig {
+            tier_type: TierType::Azure,
+            azure: Some(TierAzure {
+                secret_key: "azure-account-secret-bytes".to_string(),
+                sp_auth: ServicePrincipalAuth::default(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
+        let debug = format!("{config:?}");
+
+        assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
+        assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
+        assert!(debug.contains("client_secret: \"\""));
+        assert!(!debug.contains("client_secret: \"REDACTED\""));
+        assert!(!debug.contains("azure-account-secret-bytes"));
+    }
 }

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -460,6 +460,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_none_match: Some("*".to_string()),
                 ..Default::default()
@@ -556,6 +557,7 @@ where
         data,
         &ObjectOptions {
             max_parity: true,
+            write_completion: crate::object_api::WriteCompletion::TailDrained,
             http_preconditions: Some(HTTPPreconditions {
                 if_match: Some(current_etag.to_string()),
                 ..Default::default()

crates/ecstore/src/services/tier/warm_backend.rs

@@ -19,13 +19,14 @@
 #![allow(clippy::all)]
 
 use crate::error::is_err_bucket_not_found;
+#[cfg(feature = "gcs")]
+use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
 use crate::services::tier::{
-    tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
+    tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
     tier_config::{TierConfig, TierType},
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
     warm_backend_azure::WarmBackendAzure,
-    warm_backend_gcs::WarmBackendGCS,
     warm_backend_huaweicloud::WarmBackendHuaweicloud,
     warm_backend_minio::WarmBackendMinIO,
     warm_backend_r2::WarmBackendR2,
@@ -37,9 +38,10 @@ use crate::services::tier::{
 use bytes::Bytes;
 use http::StatusCode;
 use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
-use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
+use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
 use rustfs_s3_client::{
     admin_handler_utils::AdminError,
+    api_error_response::to_error_response,
     api_put_object::{AdvancedPutOptions, PutObjectOptions},
     transition_api::{ReadCloser, ReaderImpl},
 };
@@ -48,25 +50,31 @@ use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::http::headers::{
     CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
 };
-use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
 use s3s::header::{
     X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
     X_AMZ_STORAGE_CLASS,
 };
+use s3s::{
+    S3ErrorCode,
+    dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
+};
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 use time::OffsetDateTime;
 use time::format_description::well_known::{Rfc2822, Rfc3339};
+use tokio::io::AsyncReadExt;
 use tracing::{info, warn};
 
 pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
 
-const PROBE_OBJECT: &str = "probeobject";
-
 /// Largest object the S3-compatible warm backends accept for a multipart put.
 pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
 /// Part-count ceiling S3-compatible services impose on a multipart upload.
 pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
+pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
+const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
+const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);
 
 #[derive(Default)]
 pub struct WarmBackendGetOpts {
@@ -138,6 +146,42 @@ pub trait WarmBackend {
     async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
         Ok(TransitionCandidateProbe::Unsupported)
     }
+    async fn probe_transition_version(
+        &self,
+        object: &str,
+        remote_version_id: &str,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        if remote_version_id.is_empty() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "an exact tier probe requires a remote version ID",
+            ));
+        }
+        self.validate_remote_version_id(remote_version_id)?;
+        match self
+            .get(
+                object,
+                remote_version_id,
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 1,
+                },
+            )
+            .await
+        {
+            Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
+            Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
+                Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
+            }
+            Err(err)
+                if err.kind() == std::io::ErrorKind::NotFound
+                    || matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
+            {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+            Err(err) => Err(err),
+        }
+    }
     async fn in_use(&self) -> Result<bool, std::io::Error>;
 }
 
@@ -260,6 +304,44 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
     pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
 }
 
+/// Return the authority format accepted by `TransitionClient::new` while
+/// retaining an explicitly configured port. `url::Url::host_str()` omits the
+/// brackets needed when an IPv6 literal is combined with a port.
+pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
+    let host = url
+        .host_str()
+        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+    let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
+    if host.starts_with('[') && host.ends_with(']') {
+        Ok(format!("{host}:{port}"))
+    } else if host.contains(':') {
+        Ok(format!("[{host}]:{port}"))
+    } else {
+        Ok(format!("{host}:{port}"))
+    }
+}
+
+fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
+}
+
+pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
+    TransitionClientTimeouts::new(
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
+        ),
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
+        ),
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+        ),
+    )
+}
+
 /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
 ///
 /// Credential, bucket, and endpoint validation run in this order because the
@@ -290,6 +372,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
         signer_type: SignatureType::SignatureV4,
         ..Default::default()
     }));
+    let timeouts = transition_client_timeouts_from_env();
     let opts = Options {
         creds,
         secure: u.scheme() == "https",
@@ -298,17 +381,11 @@ pub(crate) async fn new_s3_compatible_warm_backend(
         bucket_lookup: params.bucket_lookup,
         ..Default::default()
     };
-    let scheme = u.scheme();
-    let default_port = if scheme == "https" { 443 } else { 80 };
-    let host = u
-        .host_str()
-        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-    // Runs after the host-presence check above (not immediately after Url::parse) so a
-    // host-less endpoint still reports this constructor's own "missing host" text instead of
-    // validate_endpoint's differently-worded rejection for the same input.
+    let endpoint = endpoint_authority(&u)?;
+    // Run the SSRF guard after the host-presence check so a host-less endpoint
+    // keeps this constructor's stable error text.
     (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
-    let client =
-        TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
+    let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
 
     let client = Arc::new(client);
     let core = TransitionCore(Arc::clone(&client));
@@ -423,6 +500,17 @@ impl WarmBackend for MeteredWarmBackend {
         Self::record(TierRequestOperation::Probe, result)
     }
 
+    async fn probe_transition_version(
+        &self,
+        object: &str,
+        remote_version_id: &str,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        Self::record(
+            TierRequestOperation::Probe,
+            self.inner.probe_transition_version(object, remote_version_id).await,
+        )
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
     }
@@ -451,25 +539,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
     }
 }
 
-pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
-    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
-    w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
-    let remote_version_id = w
-        .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
-        .await
-        .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+async fn remove_discovered_probe_candidate(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    candidate: TransitionCandidateProbe,
+) -> Result<bool, std::io::Error> {
+    match candidate {
+        TransitionCandidateProbe::Missing => Ok(false),
+        TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
+            w.remove_exact(probe_object, &remote_version_id).await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::UnversionedPresent => {
+            w.remove(probe_object, "").await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::Ambiguous => {
+            Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
+        }
+        TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "remote tier cannot discover the outcome of a probe PUT",
+        )),
+    }
+}
 
-    if w.validate_remote_version_id(&remote_version_id).is_err() {
-        w.remove_exact(PROBE_OBJECT, &remote_version_id)
+async fn compensate_uncertain_probe_put(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    settle_deadline: tokio::time::Instant,
+) -> Result<(), std::io::Error> {
+    let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
+    let mut removed_any = false;
+    while tokio::time::Instant::now() < settle_deadline {
+        let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
+            Ok(candidate) => candidate?,
+            Err(_) => break,
+        };
+        if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
+            break;
+        }
+        removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
             .await
-            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
-        return Err(ERR_TIER_INVALID_CONFIG.clone());
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
+
+        let now = tokio::time::Instant::now();
+        if now >= settle_deadline {
+            break;
+        }
+        tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
     }
 
-    let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
-    let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
-    //xhttp.DrainBody(r);
-    if read_result.is_err() || remove_result.is_err() {
+    let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
+        })??;
+    if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
+        })??
+    {
+        return Ok(());
+    }
+
+    let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
+    match final_candidate {
+        TransitionCandidateProbe::Missing => Ok(()),
+        _ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
+    }
+}
+
+fn probe_cleanup_incomplete_error() -> AdminError {
+    let mut err = ERR_TIER_PERM_ERR.clone();
+    err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
+    err
+}
+
+async fn check_warm_backend_with_deadlines(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+    cleanup_deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
+    let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
+    let timeout_error = || {
+        let mut err = ERR_TIER_BACKEND_IN_USE.clone();
+        err.message = "Timed out validating the remote tier mutation".to_string();
+        err
+    };
+    tokio::time::timeout_at(deadline, w.validate())
+        .await
+        .map_err(|_| timeout_error())?
+        .map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+    let put_result =
+        tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
+    let remote_version_id = match put_result {
+        Ok(Ok(remote_version_id)) => remote_version_id,
+        Ok(Err(_)) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_PERM_ERR.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+        Err(_) => {
+            let err = timeout_error();
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => err,
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+
+    // S3-family backends do not replay a failed request before returning `Ok`,
+    // while GCS discovers every matching generation. The authoritative probe
+    // below therefore closes the acknowledged-PUT path; only an error or
+    // timeout needs the longer visibility reconciliation above.
+    let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
+        Ok(Ok(candidate)) => candidate,
+        Ok(Err(_)) | Err(_) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+    let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
+    let response_matches_candidate = match &authoritative_candidate {
+        TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
+        TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
+        TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
+    };
+    if !response_version_is_valid || !response_matches_candidate {
+        return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+            Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+            Err(_) => probe_cleanup_incomplete_error(),
+        });
+    }
+
+    let read_result = tokio::time::timeout_at(deadline, async {
+        let mut reader = w
+            .get(
+                &probe_object,
+                &remote_version_id,
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 7,
+                },
+            )
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        let mut body = Vec::new();
+        reader
+            .take(7)
+            .read_to_end(&mut body)
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        if body != b"RustFS" {
+            return Err(ERR_TIER_PERM_ERR.clone());
+        }
+        Ok(())
+    })
+    .await
+    .map_err(|_| timeout_error())
+    .and_then(|result| result);
+    let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
+        if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
+            return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
+        }
+        match w.probe_transition_candidate(&probe_object).await? {
+            TransitionCandidateProbe::Missing => Ok(()),
+            _ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
+        }
+    })
+    .await;
+    if !matches!(cleanup_result, Ok(Ok(()))) {
+        return Err(probe_cleanup_incomplete_error());
+    }
+    if let Err(err) = read_result {
         //if is_err_bucket_not_found(&err) {
         //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
         //}
@@ -477,12 +727,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
             return Err(ERR_TIER_MISSING_CREDENTIALS);
         }*/
         //else {
-        return Err(ERR_TIER_PERM_ERR.clone());
+        return Err(err);
         //}
     }
     Ok(())
 }
 
+/// Validate a backend using a caller-owned deadline while retaining a bounded
+/// reconciliation window for an uncertain probe PUT. The validation future is
+/// kept alive through cleanup so an outer timeout cannot abandon the remote
+/// probe object.
+pub(crate) async fn check_warm_backend_until(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
+}
+
+pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
+    let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
+}
+
 pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
     let mut d: Option<WarmBackendImpl> = None;
     match tier.tier_type {
@@ -647,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
                 });
             }
         }
+        #[cfg(not(feature = "gcs"))]
+        TierType::GCS => {
+            return Err(AdminError {
+                code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
+                message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
+                status_code: StatusCode::NOT_IMPLEMENTED,
+            });
+        }
+        #[cfg(feature = "gcs")]
         TierType::GCS => {
             if let Some(gcs_config) = tier.gcs.as_ref() {
                 let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
@@ -701,7 +976,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
 
     if probe {
-        d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+        check_warm_backend(Some(&d)).await?;
     }
     Ok(d)
 }
@@ -754,6 +1029,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
     use crate::services::tier::tier_config::TierWasabi;
     use std::sync::{
         Arc,
@@ -762,6 +1038,27 @@ mod tests {
 
     const PROBE_VERSION: &str = "remote-v2";
 
+    #[cfg(not(feature = "gcs"))]
+    #[tokio::test]
+    async fn gcs_backend_not_compiled_preserves_config() {
+        let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
+        let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
+        assert_eq!(tier.tier_type, TierType::GCS);
+        let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
+        let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
+        assert_eq!(restored.tier_type, TierType::GCS);
+        let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
+        assert_eq!(restored_gcs.bucket, "archive");
+        assert_eq!(restored_gcs.creds, "secret");
+        assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
+        let error = match new_warm_backend(&tier, false).await {
+            Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
+            Err(error) => error,
+        };
+        assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
+        assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
+    }
+
     struct CountingBackend {
         put_result: fn() -> Result<String, std::io::Error>,
         removes: Arc<AtomicUsize>,
@@ -920,13 +1217,38 @@ mod tests {
 
     struct RejectingProbeVersionBackend {
         gets: Arc<AtomicUsize>,
+        present: Arc<std::sync::atomic::AtomicBool>,
         removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        returned_version: String,
     }
 
     struct RecordingProbeBackend {
         get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        present: Arc<std::sync::atomic::AtomicBool>,
         removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        remove_clears_candidate: bool,
         fail_get: bool,
+        body: ProbeBody,
+    }
+
+    struct HangingProbePutBackend {
+        put_started: Arc<tokio::sync::Notify>,
+        present: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    struct LateVisibleProbeBackend {
+        visible_at: tokio::time::Instant,
+        removed: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    #[derive(Clone, Copy)]
+    enum ProbeBody {
+        Exact,
+        Mismatch,
     }
 
     #[async_trait::async_trait]
@@ -976,7 +1298,7 @@ mod tests {
         }
 
         async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
-            Ok(uuid::Uuid::nil().to_string())
+            Ok(self.returned_version.clone())
         }
 
         async fn put_with_meta(
@@ -999,10 +1321,19 @@ mod tests {
         }
 
         async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
             self.removed_versions.lock().await.push(rv.to_string());
             Ok(())
         }
 
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
         async fn in_use(&self) -> Result<bool, std::io::Error> {
             Ok(false)
         }
@@ -1029,15 +1360,121 @@ mod tests {
             if self.fail_get {
                 Err(std::io::Error::other("probe GET failed"))
             } else {
-                Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
+                match self.body {
+                    ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
+                    ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
+                }
             }
         }
 
         async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            if self.remove_clears_candidate {
+                self.present.store(false, Ordering::SeqCst);
+            }
             self.removed_versions.lock().await.push(rv.to_string());
             Ok(())
         }
 
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for HangingProbePutBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            self.put_started.notify_one();
+            std::future::pending().await
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for LateVisibleProbeBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            Err(std::io::Error::new(
+                std::io::ErrorKind::ConnectionReset,
+                "probe PUT response was lost before the object became visible",
+            ))
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.removed.store(true, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
         async fn in_use(&self) -> Result<bool, std::io::Error> {
             Ok(false)
         }
@@ -1098,13 +1535,15 @@ mod tests {
         assert_eq!(probe, TransitionCandidateProbe::Unsupported);
     }
 
-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
     async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
         let gets = Arc::new(AtomicUsize::new(0));
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
             gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            returned_version: uuid::Uuid::nil().to_string(),
         });
 
         let err = check_warm_backend(Some(&backend))
@@ -1113,7 +1552,27 @@ mod tests {
 
         assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
         assert_eq!(gets.load(Ordering::SeqCst), 0);
-        assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
+        let gets = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
+            gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            returned_version: String::new(),
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("an empty PUT version must not read or generically delete a versioned object");
+
+        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
+        assert_eq!(gets.load(Ordering::SeqCst), 0);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
     }
 
     #[tokio::test]
@@ -1122,8 +1581,11 @@ mod tests {
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
             get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
             fail_get: false,
+            body: ProbeBody::Exact,
         });
 
         check_warm_backend(Some(&backend))
@@ -1140,8 +1602,11 @@ mod tests {
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
             get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
             fail_get: true,
+            body: ProbeBody::Exact,
         });
 
         let err = check_warm_backend(Some(&backend))
@@ -1153,6 +1618,169 @@ mod tests {
         assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
     }
 
+    #[tokio::test]
+    async fn check_warm_backend_removes_probe_after_body_mismatch() {
+        let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions,
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
+            fail_get: false,
+            body: ProbeBody::Mismatch,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("a mismatched body should fail after cleanup");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: false,
+            fail_get: false,
+            body: ProbeBody::Exact,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("cleanup must not succeed while the deleted candidate remains visible");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_lost_put_response() {
+        let backend = MockWarmBackend::new();
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a lost probe PUT response must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(backend.object_count().await, 0);
+        assert_eq!(backend.exact_remove_count(), 1);
+        let operations = backend.op_log().await;
+        let put = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Put { object } => Some(object),
+            _ => None,
+        });
+        let probe = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Probe { object } => Some(object),
+            _ => None,
+        });
+        let remove = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Remove { object } => Some(object),
+            _ => None,
+        });
+        let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
+            panic!("lost-response compensation should PUT, probe, and remove");
+        };
+        assert_eq!(put, probe);
+        assert_eq!(probe, remove);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
+            visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
+            removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a late-visible probe PUT must still report the lost response");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 5,
+            "reconciliation must not stop at the first Missing result"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
+        for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
+            let backend = MockWarmBackend::new();
+            backend.set_transition_candidate_probe_override(Some(candidate)).await;
+            backend.lose_next_put_response();
+            let driver: WarmBackendImpl = Box::new(backend.clone());
+
+            let err = check_warm_backend(Some(&driver))
+                .await
+                .expect_err("an uncertain candidate must fail without a guessed delete");
+
+            assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+            assert!(err.message.contains("cleanup is incomplete"));
+            assert_eq!(backend.remove_count().await, 0);
+            assert_eq!(backend.object_count().await, 1);
+        }
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_an_exact_cleanup_failure() {
+        let backend = MockWarmBackend::new();
+        backend.set_remove_failure(true);
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("an exact cleanup failure must replace the ambiguous PUT error");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.object_count().await, 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_timed_out_put() {
+        let put_started = Arc::new(tokio::sync::Notify::new());
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
+            put_started: put_started.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+        let check = check_warm_backend(Some(&driver));
+        tokio::pin!(check);
+        tokio::select! {
+            _ = put_started.notified() => {}
+            result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
+        }
+
+        tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
+        let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 1,
+            "timed-out PUT reconciliation must keep checking through the visibility window"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
     #[tokio::test]
     async fn new_wasabi_backend_honors_probe_flag() {
         let tier = TierConfig {
@@ -1298,6 +1926,15 @@ mod tests {
         assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
     }
 
+    #[test]
+    fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
+        let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
+        assert_eq!(
+            endpoint_authority(&url).expect("the endpoint should have an authority"),
+            "[2001:db8::1]:9443"
+        );
+    }
+
     #[tokio::test]
     async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
         let mut params = s3_compatible_params("http://tier.example.com:9000");

crates/ecstore/src/services/tier/warm_backend_aliyun.rs

@@ -15,15 +15,13 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 
 use crate::services::tier::{
     tier_config::TierAliyun,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +87,10 @@ impl WarmBackend for WarmBackendAliyun {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_azure.rs

@@ -15,15 +15,13 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 
 use crate::services::tier::{
     tier_config::TierAzure,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +87,16 @@ impl WarmBackend for WarmBackendAzure {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        // Azure currently uses the shared S3/SigV4 transport, but its normal
+        // object path cannot persist exact remote versions across mixed
+        // RustFS releases. The mutation probe may still detect and precisely
+        // remove a versioned test object before rejecting that configuration.
+        self.0
+            .probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
+            .await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }
@@ -98,6 +106,23 @@ impl WarmBackend for WarmBackendAzure {
 mod tests {
     use super::*;
     use crate::services::tier::tier_config::TierAzure;
+    use rustfs_s3_client::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{Options, TransitionClient, TransitionCore},
+    };
+    use std::sync::Arc;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+            let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
+            assert_ne!(read, 0, "connection closed before request headers were received");
+            request.extend_from_slice(&buffer[..read]);
+        }
+        String::from_utf8_lossy(&request).into_owned()
+    }
 
     /// The SSRF guard itself is exercised once, generically, in
     /// `warm_backend::tests` (see backlog#2040/backlog#2041 and
@@ -119,4 +144,85 @@ mod tests {
             Err(err) => assert!(err.to_string().contains("not allowed")),
         }
     }
+
+    #[tokio::test]
+    async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+            let get_request = read_request_head(&mut get_stream).await;
+            get_stream
+                .write_all(
+                    b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
+                )
+                .await
+                .expect("fixture should write candidate GET response");
+
+            let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
+            let delete_request = read_request_head(&mut delete_stream).await;
+            delete_stream
+                .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .await
+                .expect("fixture should write exact DELETE response");
+            (get_request, delete_request)
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "azure",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendAzure(WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        });
+        assert!(
+            !backend.0.client.provider_version_capabilities().exact_get_delete,
+            "probe-only version discovery must not change Azure's persisted data-path contract"
+        );
+
+        let candidate = backend
+            .probe_transition_candidate("probe")
+            .await
+            .expect("Azure candidate should be discovered");
+        assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
+        backend
+            .remove_exact("probe", "azure-version")
+            .await
+            .expect("Azure candidate should be deleted by exact version");
+
+        let (get_request, delete_request) = fixture.await.expect("fixture should join");
+        assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
+        assert!(
+            delete_request
+                .lines()
+                .next()
+                .is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
+        );
+    }
 }

crates/ecstore/src/services/tier/warm_backend_gcs.rs

@@ -15,16 +15,14 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
 use std::io::{Error, ErrorKind};
 use std::sync::Arc;
 
 use bytes::Bytes;
-use google_cloud_auth::credentials::Credentials;
-use google_cloud_auth::credentials::user_account::Builder;
+use google_cloud_auth::credentials::service_account::Builder;
 use google_cloud_storage as gcs;
 use google_cloud_storage::client::Storage;
 use google_cloud_storage::client::StorageControl;
@@ -32,7 +30,7 @@ use std::convert::TryFrom;
 
 use crate::services::tier::{
     tier_config::TierGCS,
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
 };
 use rustfs_s3_client::{
     admin_handler_utils::AdminError,
@@ -43,6 +41,7 @@ use rustfs_utils::egress::validate_outbound_url;
 use tracing::warn;
 
 const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MAX_GCS_CANDIDATE_PAGES: usize = 64;
 
 fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
     if remote_version.is_empty() {
@@ -57,6 +56,85 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
     Ok(Some(generation))
 }
 
+fn append_gcs_chunk<E: std::fmt::Display>(
+    contents: &mut Vec<u8>,
+    chunk: Result<Bytes, E>,
+    max_response_bytes: Option<usize>,
+) -> std::io::Result<()> {
+    let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
+    if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidData,
+            "GCS object response exceeded the configured byte limit",
+        ));
+    }
+    contents.extend_from_slice(&chunk);
+    Ok(())
+}
+
+fn gcs_bucket_resource_name(bucket: &str) -> String {
+    format!("projects/_/buckets/{bucket}")
+}
+
+struct GcsCandidateObject {
+    name: String,
+    generation: i64,
+}
+
+struct GcsCandidatePage {
+    objects: Vec<GcsCandidateObject>,
+    next_page_token: String,
+}
+
+async fn probe_exact_gcs_candidate<F, Fut>(
+    remote_object: &str,
+    mut fetch_page: F,
+) -> Result<TransitionCandidateProbe, std::io::Error>
+where
+    F: FnMut(String) -> Fut,
+    Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
+{
+    let mut page_token = String::new();
+    let mut seen_page_tokens = HashSet::new();
+    let mut generation = None;
+    let mut pages_seen = 0_usize;
+
+    loop {
+        pages_seen += 1;
+        if pages_seen > MAX_GCS_CANDIDATE_PAGES {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "GCS candidate listing exceeded the page limit",
+            ));
+        }
+        let response = fetch_page(page_token.clone()).await?;
+        for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
+            if candidate.generation <= 0 {
+                return Err(std::io::Error::new(
+                    ErrorKind::InvalidData,
+                    "GCS candidate listing returned a non-positive generation",
+                ));
+            }
+            if generation.replace(candidate.generation).is_some() {
+                return Ok(TransitionCandidateProbe::Ambiguous);
+            }
+        }
+
+        if response.next_page_token.is_empty() {
+            break;
+        }
+        if !seen_page_tokens.insert(response.next_page_token.clone()) {
+            return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
+        }
+        page_token = response.next_page_token;
+    }
+
+    Ok(match generation {
+        Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
+        None => TransitionCandidateProbe::Missing,
+    })
+}
+
 pub struct WarmBackendGCS {
     pub client: Arc<Storage>,
     pub control: Arc<StorageControl>,
@@ -66,11 +144,11 @@ pub struct WarmBackendGCS {
 
 impl WarmBackendGCS {
     pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
-        if conf.creds == "" {
+        if conf.creds.is_empty() {
             return Err(std::io::Error::other("both access and secret keys are required"));
         }
 
-        if conf.bucket == "" {
+        if conf.bucket.is_empty() {
             return Err(std::io::Error::other("no bucket name was provided"));
         }
 
@@ -80,8 +158,8 @@ impl WarmBackendGCS {
                 .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
         }
 
-        let authorized_user = serde_json::from_str(&conf.creds)?;
-        let credentials = Builder::new(authorized_user)
+        let service_account = serde_json::from_str(&conf.creds)?;
+        let credentials = Builder::new(service_account)
             //.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
             //.with_backoff_policy(backoff)
             .build()
@@ -98,7 +176,11 @@ impl WarmBackendGCS {
         let client = Arc::new(client);
         // Control-plane client: the data-plane `Storage` client cannot delete or list objects;
         // delete_object/list_objects live on StorageControl.
-        let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
+        let mut control_builder = StorageControl::builder().with_credentials(credentials);
+        if !conf.endpoint.is_empty() {
+            control_builder = control_builder.with_endpoint(conf.endpoint.clone());
+        }
+        let Ok(control) = control_builder.build().await else {
             return Err(std::io::Error::other("StorageControl::builder error"));
         };
         let control = Arc::new(control);
@@ -111,11 +193,11 @@ impl WarmBackendGCS {
     }
 
     pub fn get_dest(&self, object: &str) -> String {
-        let mut dest_obj = object.to_string();
-        if self.prefix != "" {
-            dest_obj = format!("{}/{}", &self.prefix, object);
+        if self.prefix.is_empty() {
+            object.to_string()
+        } else {
+            format!("{}/{}", self.prefix, object)
         }
-        return dest_obj;
     }
 }
 
@@ -136,9 +218,10 @@ impl WarmBackend for WarmBackendGCS {
             ReaderImpl::Body(content_body) => content_body.to_vec(),
             ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
         };
+        let bucket = gcs_bucket_resource_name(&self.bucket);
         let Ok(res) = Box::pin(
             self.client
-                .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
+                .write_object(&bucket, self.get_dest(object), Bytes::from(d))
                 .send_buffered(),
         )
         .await
@@ -154,7 +237,9 @@ impl WarmBackend for WarmBackendGCS {
     }
 
     async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
-        let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
+        let bucket = gcs_bucket_resource_name(&self.bucket);
+        let mut req = self.client.read_object(&bucket, self.get_dest(object));
+        let mut max_response_bytes = None;
         if let Some(generation) = parse_generation(rv)? {
             req = req.set_generation(generation);
         }
@@ -170,6 +255,11 @@ impl WarmBackend for WarmBackendGCS {
                 .length
                 .try_into()
                 .map_err(|_| std::io::Error::other("invalid range: negative length"))?;
+            max_response_bytes = Some(
+                opts.length
+                    .try_into()
+                    .map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
+            );
             req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
         }
 
@@ -177,8 +267,8 @@ impl WarmBackend for WarmBackendGCS {
             return Err(std::io::Error::other("read_object error"));
         };
         let mut contents = Vec::new();
-        while let Ok(Some(chunk)) = reader.next().await.transpose() {
-            contents.extend_from_slice(&chunk);
+        while let Some(chunk) = reader.next().await {
+            append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
         }
         Ok(ReadCloser::new(std::io::Cursor::new(contents)))
     }
@@ -190,7 +280,7 @@ impl WarmBackend for WarmBackendGCS {
         let mut req = self
             .control
             .delete_object()
-            .set_bucket(format!("projects/_/buckets/{}", self.bucket))
+            .set_bucket(gcs_bucket_resource_name(&self.bucket))
             .set_object(self.get_dest(object));
         if let Some(generation) = parse_generation(rv)? {
             req = req.set_generation(generation);
@@ -199,13 +289,47 @@ impl WarmBackend for WarmBackendGCS {
         Ok(())
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let parent = gcs_bucket_resource_name(&self.bucket);
+        probe_exact_gcs_candidate(&remote_object, |page_token| {
+            let control = self.control.clone();
+            let parent = parent.clone();
+            let prefix = remote_object.clone();
+            async move {
+                let response = control
+                    .list_objects()
+                    .set_parent(parent)
+                    .set_prefix(prefix)
+                    .set_versions(true)
+                    .set_page_size(2)
+                    .set_page_token(page_token)
+                    .send()
+                    .await
+                    .map_err(|err| std::io::Error::other(err.to_string()))?;
+                Ok(GcsCandidatePage {
+                    objects: response
+                        .objects
+                        .into_iter()
+                        .map(|candidate| GcsCandidateObject {
+                            name: candidate.name,
+                            generation: candidate.generation,
+                        })
+                        .collect(),
+                    next_page_token: response.next_page_token,
+                })
+            }
+        })
+        .await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         // Scope the listing to this tier's prefix (matching the other warm backends) and only
         // need to know whether a single object exists.
         let resp = self
             .control
             .list_objects()
-            .set_parent(format!("projects/_/buckets/{}", self.bucket))
+            .set_parent(gcs_bucket_resource_name(&self.bucket))
             .set_prefix(self.prefix.clone())
             .set_page_size(1)
             .send()
@@ -218,10 +342,126 @@ impl WarmBackend for WarmBackendGCS {
 
 #[cfg(test)]
 mod tests {
+    use super::GcsCandidateObject;
+    use super::GcsCandidatePage;
+    use super::MAX_GCS_CANDIDATE_PAGES;
     use super::WarmBackendGCS;
+    use super::append_gcs_chunk;
+    use super::gcs_bucket_resource_name;
     use super::parse_generation;
+    use super::probe_exact_gcs_candidate;
     use crate::services::tier::tier_config::TierGCS;
+    use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
+    use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
+    use google_cloud_storage::client::{Storage, StorageControl};
     use std::io::ErrorKind;
+    use std::sync::Arc;
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
+        let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
+        let responses = [
+            format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
+                upload_body.len()
+            ),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
+                .to_string(),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
+                .to_string(),
+        ];
+        let mut requests = Vec::new();
+
+        for response in responses {
+            let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
+            let mut request = Vec::new();
+            loop {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read request headers");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let header_end = request
+                .windows(4)
+                .position(|window| window == b"\r\n\r\n")
+                .map(|position| position + 4)
+                .expect("the GCS fixture should receive complete request headers");
+            let headers = String::from_utf8_lossy(&request[..header_end]);
+            if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
+                stream
+                    .write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
+                    .await
+                    .expect("the GCS fixture should acknowledge 100-continue");
+            }
+            let content_length = headers
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
+                })
+                .unwrap_or_default();
+            while request.len() < header_end.saturating_add(content_length) {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read the request body");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+            }
+            requests.push(String::from_utf8_lossy(&request).into_owned());
+            stream
+                .write_all(response.as_bytes())
+                .await
+                .expect("the GCS fixture should write its response");
+        }
+
+        requests
+    }
+
+    fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
+        GcsCandidatePage {
+            objects: objects
+                .iter()
+                .map(|(name, generation)| GcsCandidateObject {
+                    name: (*name).to_string(),
+                    generation: *generation,
+                })
+                .collect(),
+            next_page_token: next_page_token.to_string(),
+        }
+    }
+
+    async fn probe_candidate_pages(
+        remote_object: &str,
+        pages: Vec<GcsCandidatePage>,
+    ) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
+        let mut pages = pages.into_iter();
+        let mut requested_tokens = Vec::new();
+        let result = probe_exact_gcs_candidate(remote_object, |page_token| {
+            requested_tokens.push(page_token);
+            std::future::ready(
+                pages
+                    .next()
+                    .ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
+            )
+        })
+        .await;
+        (result, requested_tokens)
+    }
 
     #[test]
     fn generation_parser_preserves_exact_numeric_versions() {
@@ -241,6 +481,243 @@ mod tests {
         }
     }
 
+    #[test]
+    fn body_collection_propagates_an_error_after_a_complete_prefix() {
+        let mut contents = Vec::new();
+        append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
+            .expect("the prefix chunk should be collected");
+        let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
+            .expect_err("a trailing stream error must not be mistaken for EOF");
+
+        assert_eq!(contents, b"RustFS");
+        assert!(err.to_string().contains("trailing stream failure"));
+    }
+
+    #[test]
+    fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
+        let mut contents = Vec::new();
+        let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
+            .expect_err("the GCS collection layer must reject an oversized probe response");
+
+        assert!(contents.is_empty());
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
+        let (first, first_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
+        assert_eq!(
+            first.expect("an exact first-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+        assert_eq!(first_tokens, [""]);
+
+        let (later, later_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object-shadow", 8)], "next"),
+                candidate_page(&[("prefix/object", 9)], ""),
+            ],
+        )
+        .await;
+        assert_eq!(
+            later.expect("an exact later-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("9".to_string())
+        );
+        assert_eq!(later_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_ignores_non_exact_prefix_matches() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(
+                &[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
+                "",
+            )],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("prefix-only matches should not hide the exact object"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object", 7)], "next"),
+                candidate_page(&[("prefix/object", 8)], ""),
+            ],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("multiple exact generations should produce a conservative result"),
+            TransitionCandidateProbe::Ambiguous
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_missing_without_an_exact_name() {
+        let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
+
+        assert_eq!(
+            probe.expect("a complete listing without an exact name should be definitive"),
+            TransitionCandidateProbe::Missing
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_non_positive_generations() {
+        for generation in [0, -1] {
+            let (probe, _) =
+                probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
+            let err = probe.expect_err("a non-positive GCS generation must fail closed");
+            assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
+        }
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
+        let (probe, requested_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
+
+        let err = probe.expect_err("a repeated GCS page token must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
+        let (probe, requested_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
+        )
+        .await;
+
+        let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "a", "b"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
+        let pages = (0..MAX_GCS_CANDIDATE_PAGES)
+            .map(|index| candidate_page(&[], &format!("token-{index}")))
+            .collect();
+        let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
+
+        let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
+    }
+
+    #[tokio::test]
+    async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("the GCS fixture should bind a loopback port");
+        let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
+        let fixture = tokio::spawn(serve_data_plane_fixture(listener));
+        let credentials = Anonymous::new().build();
+        let client = Storage::builder()
+            .with_endpoint(endpoint.clone())
+            .with_credentials(credentials.clone())
+            .build()
+            .await
+            .expect("the GCS data client should build");
+        let control = StorageControl::builder()
+            .with_endpoint(endpoint)
+            .with_credentials(credentials)
+            .build()
+            .await
+            .expect("the GCS control client should build");
+        let backend = WarmBackendGCS {
+            client: Arc::new(client),
+            control: Arc::new(control),
+            bucket: "tier-bucket".to_string(),
+            prefix: String::new(),
+        };
+
+        let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
+            let version = backend
+                .put(
+                    "probe",
+                    rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
+                    6,
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS upload transport");
+            let mut reader = backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS read transport");
+            let mut body = Vec::new();
+            reader
+                .read_to_end(&mut body)
+                .await
+                .expect("the fixture body should be readable");
+            let oversized_error = match backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+            {
+                Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
+                Err(err) => err,
+            };
+            let requests = fixture.await.expect("the GCS fixture task should finish");
+            (version, body, oversized_error.kind(), requests)
+        })
+        .await
+        .expect("the GCS data-plane requests should not be rejected before transport");
+
+        assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
+        assert_eq!(version, "123");
+        assert_eq!(body, b"RustFS!");
+        assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
+        assert!(
+            requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
+            "unexpected upload request line: {}",
+            requests[0].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected read request line: {}",
+            requests[1].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the GCS probe read must preserve its seven-byte range"
+        );
+        assert!(
+            requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected oversized read request line: {}",
+            requests[2].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the oversized response must be fetched under the same seven-byte request boundary"
+        );
+    }
+
     #[tokio::test]
     async fn new_rejects_loopback_endpoint_before_credential_setup() {
         let conf = TierGCS {

crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs

@@ -15,15 +15,13 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 
 use crate::services::tier::{
     tier_config::TierHuaweicloud,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +87,10 @@ impl WarmBackend for WarmBackendHuaweicloud {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -15,8 +15,6 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -26,11 +26,12 @@ use crate::services::tier::{
     tier_config::TierS3,
     warm_backend::{
         TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
-        build_transition_put_options,
+        build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
     },
 };
 use http::HeaderMap;
 use rustfs_s3_client::{
+    api_error_response::to_error_response,
     api_get_options::GetObjectOptions,
     api_list::ListObjectsOptions,
     api_put_object::PutObjectOptions,
@@ -43,7 +44,7 @@ use rustfs_s3_client::{
 };
 use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::path::SLASH_SEPARATOR;
-use s3s::dto::BucketVersioningStatus;
+use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
 
 pub struct WarmBackendS3 {
     pub client: Arc<TransitionClient>,
@@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
     })
 }
 
+fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
+    if opts.start_offset < 0 || opts.length <= 0 {
+        return Ok(None);
+    }
+    usize::try_from(opts.length)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
+    let end_offset = opts
+        .start_offset
+        .checked_add(opts.length - 1)
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
+    Ok(Some((opts.start_offset, end_offset)))
+}
+
 impl WarmBackendS3 {
     pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
         Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -125,6 +139,7 @@ impl WarmBackendS3 {
         } else {
             return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
         }
+        let timeouts = transition_client_timeouts_from_env();
         let opts = Options {
             creds,
             secure: u.scheme() == "https",
@@ -132,10 +147,8 @@ impl WarmBackendS3 {
             bucket_lookup,
             ..Default::default()
         };
-        let host = u
-            .host()
-            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-        let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
+        let endpoint = endpoint_authority(&u)?;
+        let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));
@@ -177,10 +190,8 @@ impl WarmBackendS3 {
         if !rv.is_empty() {
             gopts.version_id = rv.to_string();
         }
-        if opts.start_offset >= 0 && opts.length > 0 {
-            gopts
-                .set_range(opts.start_offset, opts.start_offset + opts.length - 1)
-                .map_err(std::io::Error::other)?;
+        if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
+            gopts.set_range(start_offset, end_offset)?;
         }
         let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
         Ok((headers, reader))
@@ -191,36 +202,64 @@ impl WarmBackendS3 {
         remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
     }
 
-    async fn probe_transition_candidate_versions(
+    async fn probe_current_transition_candidate_with_header(
         &self,
         object: &str,
-        bucket_versioning: RemoteBucketVersioning,
+        raw_version_header: Option<&'static str>,
     ) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let remote_object = self.get_dest(object);
-        let mut opts = ListObjectsOptions::default();
-        opts.set("prefix", &remote_object);
-        opts.set("max-keys", "1000");
-
-        let mut key_marker = String::new();
-        let mut version_id_marker = String::new();
-        let mut candidates = TransitionCandidateVersions::default();
-        loop {
-            let versions = self
-                .client
-                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
-                .await?;
-            candidates.extend(&remote_object, &versions);
-            if candidates.is_ambiguous() {
-                return Ok(TransitionCandidateProbe::Ambiguous);
+        match self
+            .get_with_headers(
+                object,
+                "",
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 1,
+                },
+            )
+            .await
+        {
+            Ok((headers, _)) => {
+                let version_id = match raw_version_header {
+                    Some(header_name) => match headers.get(header_name) {
+                        Some(value) => {
+                            let version_id = value.to_str().map_err(|_| {
+                                std::io::Error::new(
+                                    std::io::ErrorKind::InvalidData,
+                                    "remote object version id is not valid ASCII",
+                                )
+                            })?;
+                            validate_remote_version_id(version_id)?;
+                            Some(version_id)
+                        }
+                        None => None,
+                    },
+                    None => self.client.raw_version_id(&headers)?,
+                };
+                Ok(match version_id {
+                    Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
+                    None => TransitionCandidateProbe::UnversionedPresent,
+                })
             }
-            if !versions.is_truncated {
-                return classify_transition_candidates(candidates, bucket_versioning);
+            Err(err) => {
+                let response = to_error_response(&err);
+                if response.code == S3ErrorCode::NoSuchKey {
+                    Ok(TransitionCandidateProbe::Missing)
+                } else {
+                    Err(err)
+                }
             }
-
-            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
         }
     }
 
+    pub(crate) async fn probe_transition_candidate_with_raw_version_header(
+        &self,
+        object: &str,
+        raw_version_header: &'static str,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
+            .await
+    }
+
     async fn probe_transition_candidate_identity(
         &self,
         object: &str,
@@ -343,6 +382,7 @@ struct TransitionCandidateVersions {
 }
 
 impl TransitionCandidateVersions {
+    #[cfg(test)]
     fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
         for version in versions.versions.iter().filter(|version| version.key == remote_object) {
             if self.version_id.is_some() {
@@ -353,10 +393,6 @@ impl TransitionCandidateVersions {
         }
     }
 
-    fn is_ambiguous(&self) -> bool {
-        self.ambiguous
-    }
-
     fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
         if self.ambiguous {
             return TransitionCandidateProbe::Ambiguous;
@@ -380,6 +416,8 @@ impl TransitionCandidateVersions {
 mod tests {
     use super::*;
     use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
     #[tokio::test]
     async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -398,6 +436,245 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn new_preserves_an_explicit_endpoint_port() {
+        let conf = TierS3 {
+            endpoint: "https://tier.example.com:9443".to_string(),
+            bucket: "tier-bucket".to_string(),
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        };
+
+        let backend = WarmBackendS3::new(&conf, "tier")
+            .await
+            .expect("a well-formed S3 endpoint should initialize without network I/O");
+        assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
+        assert_eq!(backend.client.endpoint_url.port(), Some(9443));
+    }
+
+    #[tokio::test]
+    async fn overflowing_get_range_is_rejected_before_network_io() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        };
+
+        let err = backend
+            .get_with_headers(
+                "probe",
+                "",
+                WarmBackendGetOpts {
+                    start_offset: i64::MAX,
+                    length: 2,
+                },
+            )
+            .await
+            .expect_err("an overflowing range must fail before issuing a GET");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), listener.accept())
+                .await
+                .is_err()
+        );
+    }
+
+    async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let responses = [
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+            ];
+            let mut requests = Vec::new();
+            for response in responses {
+                let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+                let mut request = Vec::new();
+                let mut buffer = [0; 1024];
+                loop {
+                    let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                    assert_ne!(read, 0, "connection closed before request headers were received");
+                    request.extend_from_slice(&buffer[..read]);
+                    if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                        break;
+                    }
+                }
+                requests.push(String::from_utf8_lossy(&request).into_owned());
+                stream
+                    .write_all(response.as_bytes())
+                    .await
+                    .expect("fixture should write candidate response");
+            }
+            requests
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        Some((
+            WarmBackendS3 {
+                core: TransitionCore(Arc::clone(&client)),
+                client,
+                bucket: "bucket".to_string(),
+                prefix: String::new(),
+                storage_class: String::new(),
+            },
+            fixture,
+        ))
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
+        let Some((backend, fixture)) = candidate_probe_fixture().await else {
+            return;
+        };
+
+        assert_eq!(
+            backend
+                .probe_transition_candidate("versioned-probe")
+                .await
+                .expect("versioned candidate should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("unversioned-probe")
+                .await
+                .expect("unversioned candidate should be discovered"),
+            TransitionCandidateProbe::UnversionedPresent
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("missing-probe")
+                .await
+                .expect("a missing key should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("provider-missing-probe")
+                .await
+                .expect("a provider-specific missing code should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        let err = backend
+            .probe_transition_candidate("forbidden-probe")
+            .await
+            .expect_err("an authorization failure must not be mistaken for a missing key");
+        assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
+        assert_eq!(
+            backend
+                .probe_transition_candidate("delete-marker-hidden")
+                .await
+                .expect("a current delete marker should hide the data version"),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            backend
+                .probe_transition_version("delete-marker-hidden", "historical-version")
+                .await
+                .expect("the stored historical version should be probed exactly"),
+            TransitionCandidateProbe::VersionedPresent("historical-version".to_string())
+        );
+        assert_eq!(
+            backend
+                .probe_transition_version("delete-marker-hidden", "missing-version")
+                .await
+                .expect("a missing exact version should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            backend
+                .probe_transition_version("missing-object", "historical-version")
+                .await
+                .expect("a missing key for an exact version probe should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+
+        let requests = fixture.await.expect("candidate fixture should join");
+        for request in &requests[..6] {
+            let request = request.to_ascii_lowercase();
+            assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
+            assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
+            assert!(!request.contains("?versioning"));
+            assert!(!request.contains("?versions"));
+        }
+        for request in &requests[6..] {
+            let request = request.to_ascii_lowercase();
+            assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET");
+            assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
+        }
+        assert!(!requests[5].to_ascii_lowercase().contains("versionid="));
+        assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version"));
+        assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version"));
+        assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
+    }
+
     fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
         ListVersionsResult {
             versions: versions
@@ -631,8 +908,7 @@ impl WarmBackend for WarmBackendS3 {
     }
 
     async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let bucket_versioning = self.remote_bucket_versioning().await?;
-        self.probe_transition_candidate_versions(object, bucket_versioning).await
+        self.probe_current_transition_candidate_with_header(object, None).await
     }
 
     async fn in_use(&self) -> Result<bool, std::io::Error> {

crates/ecstore/src/services/tier/warm_backend_tencent.rs

@@ -15,15 +15,13 @@
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use std::collections::HashMap;
 
 use crate::services::tier::{
     tier_config::TierTencent,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +87,10 @@ impl WarmBackend for WarmBackendTencent {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_wasabi.rs

@@ -23,7 +23,7 @@ use uuid::Uuid;
 
 use crate::services::tier::{
     tier_config::{TierS3, TierWasabi},
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
     warm_backend_s3::WarmBackendS3,
 };
 use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
@@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi {
         self.s3.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
+        self.s3.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> io::Result<bool> {
         self.check_remote_bucket_unversioned().await?;
         let in_use = self.s3.in_use().await?;

crates/ecstore/src/set_disk/core/metadata_quorum.rs

@@ -0,0 +1,385 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
+//!
+//! Disk scheduling, coalescing, cancellation, and late shard materialization
+//! remain with their existing owners; this module only classifies observations.
+
+use crate::diagnostics::get::{
+    GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
+    GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
+    GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
+    GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
+    GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
+};
+use crate::disk::error::DiskError;
+use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
+use crate::set_disk::file_info_is_valid_for_metadata;
+use rustfs_filemeta::FileInfo;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(in crate::set_disk) struct MetadataEarlyStopDecision {
+    pub(in crate::set_disk) reason: &'static str,
+}
+
+#[derive(Clone, Debug)]
+pub(in crate::set_disk) struct MetadataQuorumAccumulator {
+    pub(in crate::set_disk) total_disks: usize,
+    pub(in crate::set_disk) default_parity_count: usize,
+    pub(in crate::set_disk) allow_early_stop: bool,
+    pub(in crate::set_disk) valid_responses: usize,
+    pub(in crate::set_disk) not_found_responses: usize,
+    pub(in crate::set_disk) version_not_found_responses: usize,
+    pub(in crate::set_disk) ignored_errors: usize,
+    pub(in crate::set_disk) hard_errors: usize,
+    pub(in crate::set_disk) candidate: Option<FileInfo>,
+    pub(in crate::set_disk) candidate_votes: usize,
+    // Bitset of shard indexes whose metadata matches the candidate. Erasure
+    // layouts are capped at 16 shards, so this stays allocation-free on the
+    // GET metadata hot path.
+    candidate_shard_mask: u16,
+    pub(in crate::set_disk) conflicting_metadata: bool,
+    pub(in crate::set_disk) delete_marker_seen: bool,
+    pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
+    pub(in crate::set_disk) delete_marker_votes: usize,
+    pub(in crate::set_disk) requested_version_id: String,
+    pub(in crate::set_disk) matching_version_votes: usize,
+}
+
+impl MetadataQuorumAccumulator {
+    pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
+        Self {
+            total_disks,
+            default_parity_count,
+            allow_early_stop,
+            valid_responses: 0,
+            not_found_responses: 0,
+            version_not_found_responses: 0,
+            ignored_errors: 0,
+            hard_errors: 0,
+            candidate: None,
+            candidate_votes: 0,
+            candidate_shard_mask: 0,
+            conflicting_metadata: false,
+            delete_marker_seen: false,
+            delete_marker_candidates: Vec::new(),
+            delete_marker_votes: 0,
+            requested_version_id: String::new(),
+            matching_version_votes: 0,
+        }
+    }
+
+    pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
+        self.requested_version_id = version_id.to_string();
+        self
+    }
+
+    pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
+        self.observe_file_info_with_index(None, file_info);
+    }
+
+    pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
+        self.observe_file_info_with_index(Some(disk_index), file_info);
+    }
+
+    fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
+        if !file_info_is_valid_for_metadata(file_info) {
+            self.hard_errors = self.hard_errors.saturating_add(1);
+            return;
+        }
+
+        self.valid_responses = self.valid_responses.saturating_add(1);
+
+        // Track version match for versioned requests
+        if !self.requested_version_id.is_empty()
+            && let Some(ref vid) = file_info.version_id
+            && vid.to_string() == self.requested_version_id
+        {
+            self.matching_version_votes = self.matching_version_votes.saturating_add(1);
+        }
+
+        if file_info.is_canonical_delete_marker() {
+            self.delete_marker_seen = true;
+            if let Some((_, votes)) = self
+                .delete_marker_candidates
+                .iter_mut()
+                .find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
+            {
+                *votes = votes.saturating_add(1);
+            } else {
+                self.delete_marker_candidates.push((file_info.clone(), 1));
+            }
+            self.delete_marker_votes = self
+                .delete_marker_candidates
+                .iter()
+                .map(|(_, votes)| *votes)
+                .max()
+                .unwrap_or_default();
+            self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
+            return;
+        }
+
+        match &self.candidate {
+            Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
+                self.candidate_votes = self.candidate_votes.saturating_add(1);
+                if let Some(disk_index) = disk_index
+                    && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
+                {
+                    self.candidate_shard_mask |= bit;
+                }
+            }
+            Some(_) => {
+                self.conflicting_metadata = true;
+            }
+            None => {
+                self.candidate = Some(file_info.clone());
+                self.candidate_votes = 1;
+                if let Some(disk_index) = disk_index
+                    && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
+                {
+                    self.candidate_shard_mask |= bit;
+                }
+            }
+        }
+    }
+
+    fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
+        let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
+        if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
+            return None;
+        }
+        Some(1u16 << (erasure_index - 1))
+    }
+
+    pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
+        self.candidate_read_reserve_target()
+            .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
+    }
+
+    pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
+        let candidate = self.candidate.as_ref()?;
+        Some(
+            candidate
+                .erasure
+                .data_blocks
+                .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
+        )
+    }
+
+    pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
+        match err {
+            DiskError::FileNotFound | DiskError::VolumeNotFound => {
+                self.not_found_responses = self.not_found_responses.saturating_add(1);
+            }
+            DiskError::FileVersionNotFound => {
+                self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
+            }
+            _ if is_metadata_fanout_ignored_error(err) => {
+                self.ignored_errors = self.ignored_errors.saturating_add(1);
+            }
+            _ => {
+                self.hard_errors = self.hard_errors.saturating_add(1);
+            }
+        }
+    }
+
+    pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
+        if !self.allow_early_stop {
+            return None;
+        }
+        if self.delete_marker_votes >= self.default_write_quorum() {
+            return Some(MetadataEarlyStopDecision {
+                reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
+            });
+        }
+        if self.conflicting_metadata
+            || self.delete_marker_seen
+            || self.not_found_responses > 0
+            || self.version_not_found_responses > 0
+            || self.hard_errors > 0
+        {
+            return None;
+        }
+        if self
+            .candidate
+            .as_ref()
+            .and_then(|candidate| self.candidate_latest_quorum(candidate))
+            .is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
+        {
+            return Some(MetadataEarlyStopDecision {
+                reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
+            });
+        }
+        None
+    }
+
+    /// Check if a versioned request can early-stop because the requested
+    /// version_id has reached quorum across disks.
+    pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
+        if !self.allow_early_stop {
+            return None;
+        }
+        if self.requested_version_id.is_empty() {
+            return None;
+        }
+        if self.conflicting_metadata
+            || self.delete_marker_seen
+            || self.not_found_responses > 0
+            || self.version_not_found_responses > 0
+            || self.hard_errors > 0
+        {
+            return None;
+        }
+        if self.matching_version_votes >= self.read_quorum_for_version() {
+            return Some(MetadataEarlyStopDecision {
+                reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
+            });
+        }
+        None
+    }
+
+    pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
+        if !self.allow_early_stop {
+            return false;
+        }
+        if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
+            return true;
+        }
+        if self.conflicting_metadata
+            || self.delete_marker_seen
+            || self.not_found_responses > 0
+            || self.version_not_found_responses > 0
+            || self.hard_errors > 0
+        {
+            return false;
+        }
+        if !self.requested_version_id.is_empty()
+            && self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
+        {
+            return true;
+        }
+        match &self.candidate {
+            Some(candidate) => self
+                .candidate_latest_quorum(candidate)
+                .is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
+            None => pending >= self.default_write_quorum(),
+        }
+    }
+
+    /// Compute the read quorum threshold for version-aware early-stop.
+    /// Uses `total_disks / 2` (like `missing_response_quorum`) when
+    /// `default_parity_count` is set, otherwise requires all disks.
+    pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
+        self.missing_response_quorum()
+    }
+
+    pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
+        if !self.allow_early_stop {
+            return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
+        }
+        if self.conflicting_metadata {
+            return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
+        }
+        if self.delete_marker_seen {
+            return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
+        }
+        let missing_response_quorum = self.missing_response_quorum();
+        if self.version_not_found_responses >= missing_response_quorum {
+            return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
+        }
+        if self.not_found_responses >= missing_response_quorum {
+            return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
+        }
+        if self.hard_errors > 0 {
+            return GET_METADATA_EARLY_STOP_REASON_ERROR;
+        }
+        if self.ignored_errors > 0 {
+            return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
+        }
+        GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
+    }
+
+    pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
+        if self.default_parity_count == 0 {
+            return Some(self.total_disks);
+        }
+        if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
+            return None;
+        }
+        let data_blocks = candidate.erasure.data_blocks;
+        Some(if data_blocks == candidate.erasure.parity_blocks {
+            data_blocks.saturating_add(1)
+        } else {
+            data_blocks
+        })
+    }
+
+    pub(crate) fn default_write_quorum(&self) -> usize {
+        if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
+            return self.total_disks;
+        }
+        let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
+        if data_blocks == self.default_parity_count {
+            data_blocks.saturating_add(1)
+        } else {
+            data_blocks
+        }
+    }
+
+    pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
+        if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
+            self.total_disks
+        } else {
+            self.total_disks / 2
+        }
+    }
+}
+
+pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
+    left.volume == right.volume
+        && left.name == right.name
+        && left.version_id == right.version_id
+        && left.is_latest == right.is_latest
+        && left.deleted == right.deleted
+        && left.mark_deleted == right.mark_deleted
+        && left.transition_status == right.transition_status
+        && left.transitioned_objname == right.transitioned_objname
+        && left.transition_tier == right.transition_tier
+        && left.transition_version_id == right.transition_version_id
+        && left.transition_version == right.transition_version
+        && left.transition_version_state == right.transition_version_state
+        && left.expire_restored == right.expire_restored
+        && left.size == right.size
+        && left.mod_time == right.mod_time
+        && left.mode == right.mode
+        && left.written_by_version == right.written_by_version
+        && left.metadata == right.metadata
+        && left.replication_state_internal == right.replication_state_internal
+        && left.parts == right.parts
+        && left.checksum == right.checksum
+        && left.versioned == right.versioned
+        && left.num_versions == right.num_versions
+        && left.successor_mod_time == right.successor_mod_time
+        && left.data_dir == right.data_dir
+        && left.erasure.algorithm == right.erasure.algorithm
+        && left.erasure.data_blocks == right.erasure.data_blocks
+        && left.erasure.parity_blocks == right.erasure.parity_blocks
+        && left.erasure.block_size == right.erasure.block_size
+        && left.erasure.distribution == right.erasure.distribution
+}
+
+pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
+    OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
+}

crates/ecstore/src/set_disk/core/mod.rs

@@ -18,3 +18,4 @@
 //! duplicating read/write/erasure logic.
 
 pub(crate) mod io_primitives;
+mod metadata_quorum;

crates/ecstore/src/set_disk/mod.rs

@@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
 pub(crate) use ops::object::DeleteObjectCommitBarrier;
 #[cfg(any(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]

crates/ecstore/src/set_disk/ops/heal.rs

@@ -2490,9 +2490,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
             return Ok((result, err.map(|e| e.into())));
         }
 
-        let disks = self.disks.read().await;
-
-        let disks = disks.clone();
+        // The inner heal and missing-object report read the registry again;
+        // release this snapshot guard before a topology writer can queue between reads.
+        let disks = self.get_disks_internal().await;
         let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
             .await
             .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -3419,6 +3419,366 @@ mod heal_result_report_tests {
         assert_eq!(unformatted, DiskError::UnformattedDisk);
     }
 
+    #[derive(Clone, Copy)]
+    enum InventoryWriterHealCase {
+        Existing,
+        Missing,
+        MissingVersion,
+    }
+
+    async fn assert_heal_object_inventory_writer(case: InventoryWriterHealCase) {
+        use crate::set_disk::core::io_primitives::disk_call_counters;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "heal-inventory-writer-bucket";
+        let object = match case {
+            InventoryWriterHealCase::Existing => "heal-inventory-writer-existing",
+            InventoryWriterHealCase::Missing => "heal-inventory-writer-missing",
+            InventoryWriterHealCase::MissingVersion => "heal-inventory-writer-missing-version",
+        };
+        set.make_bucket(
+            bucket,
+            &MakeBucketOptions {
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("heal fixture bucket should be created");
+        let body = vec![0x67; 64 * 1024];
+        let stored_version = Uuid::new_v4();
+        let stored_version_string = stored_version.to_string();
+        let published = if matches!(case, InventoryWriterHealCase::Missing) {
+            None
+        } else {
+            let mut reader = PutObjReader::from_vec(body.clone());
+            let info = set
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        no_lock: true,
+                        versioned: true,
+                        version_id: Some(stored_version_string.clone()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("full-fanout PUT should seed the heal fixture");
+            for disk in &disks {
+                let metadata = disk
+                    .read_version("", bucket, object, &stored_version_string, &ReadOptions::default())
+                    .await
+                    .expect("the seeded version must be present on every disk");
+                assert_eq!(metadata.version_id, Some(stored_version));
+                assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
+            }
+            Some(info)
+        };
+        let requested_version = match case {
+            InventoryWriterHealCase::Existing => stored_version_string.clone(),
+            InventoryWriterHealCase::Missing => String::new(),
+            InventoryWriterHealCase::MissingVersion => Uuid::new_v4().to_string(),
+        };
+        let opts = HealOpts {
+            no_lock: true,
+            ..Default::default()
+        };
+        let calls = disk_call_counters::observe(object);
+        let read_gate = set.disks.read().await;
+        // UFCS selects the trait's outer precheck, not the same-named inherent heal.
+        let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
+            set.as_ref(),
+            bucket,
+            object,
+            &requested_version,
+            &opts,
+        );
+        tokio::pin!(heal);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(heal.as_mut())),
+            std::task::Poll::Pending
+        ));
+        // These tests use the current-thread runtime: full-wait metadata tasks
+        // have been spawned, but cannot run during the single unconstrained poll.
+        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
+        let writer = set.disks.write();
+        tokio::pin!(writer);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(writer.as_mut())),
+            std::task::Poll::Pending
+        ));
+        assert!(set.disks.try_read().is_err(), "the writer must already block new inventory readers");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the suspended trait heal must have started the real metadata fanout");
+        for disk_index in 0..4 {
+            assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
+        }
+        drop(read_gate);
+
+        let (_, outcome) =
+            tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(async { drop(writer.await) }, heal) })
+                .await
+                .expect("trait heal must not deadlock its nested inventory read with the queued writer");
+        let (result, error) = outcome.expect("heal should report the object's outcome");
+        match case {
+            InventoryWriterHealCase::Existing => assert!(error.is_none(), "existing object heal failed: {error:?}"),
+            InventoryWriterHealCase::Missing => assert!(matches!(error, Some(Error::FileNotFound))),
+            InventoryWriterHealCase::MissingVersion => assert!(matches!(error, Some(Error::FileVersionNotFound))),
+        }
+        assert_eq!(result.bucket, bucket);
+        assert_eq!(result.object, object);
+        assert_eq!(result.version_id, requested_version);
+        assert_eq!(result.disk_count, 4);
+        assert_eq!(result.before.drives.len(), 4);
+        assert_eq!(result.after.drives.len(), 4);
+        for disk_index in 0..4 {
+            let endpoint = set.set_endpoints[disk_index].to_string();
+            assert_eq!(result.before.drives[disk_index].endpoint, endpoint);
+            assert_eq!(result.after.drives[disk_index].endpoint, endpoint);
+        }
+        if let Some(published) = published {
+            tokio::time::timeout(Duration::from_secs(10), async {
+                let mut reader = set
+                    .get_object_reader(
+                        bucket,
+                        object,
+                        None,
+                        Default::default(),
+                        &ObjectOptions {
+                            versioned: true,
+                            version_id: Some(stored_version_string),
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("the stored version must remain readable after heal");
+                assert_eq!(reader.object_info.etag, published.etag);
+                assert_eq!(reader.object_info.version_id, Some(stored_version));
+                let mut observed_body = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut observed_body)
+                    .await
+                    .expect("stored body should stream");
+                assert_eq!(observed_body, body);
+            })
+            .await
+            .expect("GET must finish after the inventory writer and heal");
+        }
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_existing() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::Existing).await;
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_missing() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::Missing).await;
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_missing_version() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::MissingVersion).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn heal_object_with_queued_disk_renewal() {
+        use crate::layout::endpoints::SetupType;
+        use crate::runtime::instance::InstanceContext;
+        use crate::set_disk::core::io_primitives::disk_call_counters;
+        use std::collections::HashMap;
+        use std::future::Future;
+        use std::task::Poll;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        // renew_disk still registers local disks on the ambient context. Match
+        // the default serial group used by its other setup/registry fixtures,
+        // and restore only this temporary endpoint, including on a failed join.
+        struct RenewDiskTestState {
+            ctx: Arc<InstanceContext>,
+            was_dist_erasure: bool,
+            map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
+            endpoint: String,
+            previous_disk: Option<Option<DiskStore>>,
+        }
+
+        impl Drop for RenewDiskTestState {
+            fn drop(&mut self) {
+                let ctx = self.ctx.clone();
+                let was_dist_erasure = self.was_dist_erasure;
+                let map = self.map.clone();
+                let endpoint = self.endpoint.clone();
+                let previous_disk = self.previous_disk.take();
+                let handle = tokio::runtime::Handle::current();
+                std::thread::spawn(move || {
+                    handle.block_on(async move {
+                        let mut map = map.write().await;
+                        match previous_disk {
+                            Some(disk) => {
+                                map.insert(endpoint, disk);
+                            }
+                            None => {
+                                map.remove(&endpoint);
+                            }
+                        }
+                        drop(map);
+                        if was_dist_erasure {
+                            ctx.update_erasure_type(SetupType::DistErasure).await;
+                        }
+                    });
+                })
+                .join()
+                .expect("renew fixture state restoration should finish");
+            }
+        }
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let endpoint = set.set_endpoints[0].clone();
+        let ctx = crate::runtime::global::current_ctx();
+        let map = ctx.local_disk_map();
+        let restore = RenewDiskTestState {
+            ctx: ctx.clone(),
+            was_dist_erasure: ctx.is_dist_erasure().await,
+            map: map.clone(),
+            endpoint: endpoint.to_string(),
+            previous_disk: map.read().await.get(&endpoint.to_string()).cloned(),
+        };
+        // Only distributed erasure needs an override to avoid the ambient slot array.
+        if restore.was_dist_erasure {
+            ctx.update_erasure_type(SetupType::Erasure).await;
+        }
+
+        let bucket = "heal-disk-renewal-bucket";
+        let object = "heal-disk-renewal-object";
+        set.make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("renew fixture bucket should be created");
+        let body = vec![0x73; 64 * 1024];
+        let mut reader = PutObjReader::from_vec(body.clone());
+        let published = set
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("full-fanout PUT should seed the renewal fixture");
+        for disk in &disks {
+            let metadata = disk
+                .read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("the seeded object must be present on every disk");
+            assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
+        }
+
+        let opts = HealOpts {
+            no_lock: true,
+            ..Default::default()
+        };
+        let calls = disk_call_counters::observe(object);
+        let read_gate = set.disks.read().await;
+        let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
+            set.as_ref(),
+            bucket,
+            object,
+            "",
+            &opts,
+        );
+        tokio::pin!(heal);
+        assert!(matches!(futures::poll!(tokio::task::unconstrained(heal.as_mut())), Poll::Pending));
+        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
+
+        let renew = set.renew_disk(&endpoint);
+        tokio::pin!(renew);
+        tokio::time::timeout(
+            Duration::from_secs(5),
+            futures::future::poll_fn(|cx| {
+                assert!(
+                    std::pin::pin!(tokio::task::unconstrained(renew.as_mut()))
+                        .poll(cx)
+                        .is_pending(),
+                    "renewal must reach its inventory write before returning"
+                );
+                if set.disks.try_read().is_err() {
+                    Poll::Ready(())
+                } else {
+                    Poll::Pending
+                }
+            }),
+        )
+        .await
+        .expect("real renewal must queue its topology writer behind the read gate");
+        let registered = map
+            .read()
+            .await
+            .get(&endpoint.to_string())
+            .cloned()
+            .flatten()
+            .expect("renewal must register the connected disk before its inventory write");
+        assert!(!Arc::ptr_eq(&registered, &disks[0]), "renewal must construct a new disk handle");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the suspended trait heal must have started the real metadata fanout");
+        for disk_index in 0..4 {
+            assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
+        }
+        drop(read_gate);
+
+        let (_, outcome) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(renew, heal) })
+            .await
+            .expect("trait heal and real disk renewal must finish without a nested inventory read deadlock");
+        let (report, error) = outcome.expect("heal should report the existing object");
+        assert!(error.is_none(), "existing object heal failed after renewal: {error:?}");
+        assert_eq!(report.bucket, bucket);
+        assert_eq!(report.object, object);
+        assert_eq!(report.disk_count, 4);
+        let renewed = set.get_disks_internal().await[0]
+            .clone()
+            .expect("the renewed slot must remain online");
+        assert!(Arc::ptr_eq(&renewed, &registered), "the set must publish the newly connected handle");
+        assert_eq!(renewed.endpoint(), endpoint);
+        let format = load_format_erasure(&renewed, false)
+            .await
+            .expect("renewed disk format should remain readable");
+        assert_eq!(format.erasure.this, set.format.erasure.sets[0][0]);
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let mut reader = set
+                .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("the object must remain readable after renewal and heal");
+            assert_eq!(reader.object_info.etag, published.etag);
+            let mut observed_body = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut observed_body)
+                .await
+                .expect("stored body should stream");
+            assert_eq!(observed_body, body);
+        })
+        .await
+        .expect("GET must finish after renewal and heal");
+    }
+
     // Regression for #955: an offline disk must contribute exactly one drive
     // record. Before the fix the offline branch fell through and pushed a second
     // (Corrupt) record for the same disk, so `before/after.drives` grew to

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2452,10 +2452,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let write_quorum = fi.write_quorum(self.default_write_quorum());
         let read_quorum = fi.read_quorum(self.default_read_quorum());
 
-        let disks = self.disks.read().await;
-
-        let disks = disks.clone();
-        // let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
+        // Release the registry guard before recovery and cleanup read it again:
+        // a queued topology writer would otherwise deadlock those nested reads.
+        let disks = self.get_disks_internal().await;
 
         let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
         self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -4051,6 +4050,7 @@ mod tests {
             let _ = drain_global_dirty_scopes();
 
             let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
             let complete_store = Arc::clone(&set_disks);
             let mut complete = tokio::spawn(async move {
                 let mut opts = ObjectOptions::default();
@@ -4062,16 +4062,6 @@ mod tests {
             tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                 .await
                 .expect("multipart completion should pause one tail disk during rename");
-            assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
-                "multipart completion must not publish success while a tail rename is still paused"
-            );
-
-            let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
-            assert!(
-                initial.is_empty(),
-                "capacity must not be marked as committed before the full multipart rename finishes"
-            );
 
             let abort_store = Arc::clone(&set_disks);
             let abort = tokio::spawn(async move {
@@ -4080,21 +4070,46 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(2).await;
-            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
 
-            let retained_staging = futures::future::join_all(
-                disk_stores
-                    .iter()
-                    .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
-            )
+            // A paused rename does not establish that the other disks reached quorum.
+            let retained_staging = tokio::time::timeout(Duration::from_secs(30), async {
+                loop {
+                    let mut retained = 0;
+                    for result in futures::future::join_all(
+                        disk_stores
+                            .iter()
+                            .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
+                    )
+                    .await
+                    {
+                        match result {
+                            Ok(_) => retained += 1,
+                            Err(DiskError::FileNotFound) => {}
+                            Err(error) => panic!("staged rename source lookup failed: {error}"),
+                        }
+                    }
+                    if retained <= 1 && rename_tasks.running() == 1 {
+                        break retained;
+                    }
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
             .await
-            .into_iter()
-            .filter(|result| result.is_ok())
-            .count();
+            .expect("unpaused multipart renames should finish before the tail is released");
             assert_eq!(
                 retained_staging, 1,
                 "only the paused tail disk should still retain the multipart rename source"
             );
+            assert!(
+                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
+                "multipart completion must not publish success while a tail rename is still paused"
+            );
+            let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
+            assert!(
+                initial.is_empty(),
+                "capacity must not be marked as committed before the full multipart rename finishes"
+            );
+            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
 
             signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object));
             let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1;
@@ -6743,6 +6758,87 @@ mod tests {
         .await;
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-topology-lock-bucket";
+        let object = "object";
+        let body = vec![0x65; 4096];
+        make_bucket_on_all(&disk_stores, bucket).await;
+        let (upload_id, parts) =
+            stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
+        let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
+        for dir in &temp_dirs {
+            assert!(
+                dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
+                "the test must create real upload staging on every disk"
+            );
+        }
+        let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
+        let complete_store = set_disks.clone();
+        let complete_upload_id = upload_id.clone();
+        let complete = tokio::spawn(async move {
+            complete_store
+                .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                .await
+        });
+        barrier.wait_until_paused().await;
+
+        // Hold a separate read gate so the real writer queues even when completion
+        // correctly releases its snapshot guard. Polling Pending proves admission
+        // to Tokio's write-preferring queue before the cleanup attempts another read.
+        let read_gate = set_disks.disks.read().await;
+        let writer = set_disks.disks.write();
+        tokio::pin!(writer);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(writer.as_mut())),
+            std::task::Poll::Pending
+        ));
+        assert!(
+            set_disks.disks.try_read().is_err(),
+            "the pending writer must already block new readers before the cleanup resumes"
+        );
+        drop(read_gate);
+        barrier.release();
+
+        let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
+            .await
+            .expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
+        // A reconnect can publish the same handles; this test isolates admission
+        // order without changing the disks that contain the committed object.
+        drop(writer_guard);
+        tokio::time::timeout(Duration::from_secs(10), complete)
+            .await
+            .expect("multipart cleanup must finish after the topology writer releases")
+            .expect("completion task should not panic")
+            .expect("completion should preserve the successful object commit");
+
+        let mut reader = tokio::time::timeout(
+            Duration::from_secs(10),
+            set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
+        )
+        .await
+        .expect("GET should finish after completion")
+        .expect("the completed object should remain readable");
+        let mut observed_body = Vec::new();
+        tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
+            .await
+            .expect("the completed object body should finish streaming")
+            .expect("the completed object body should be readable");
+        assert_eq!(observed_body, body);
+        assert!(matches!(
+            set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
+            Err(StorageError::InvalidUploadID(..))
+        ));
+        for dir in &temp_dirs {
+            assert!(
+                !dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
+                "successful completion must remove its upload staging from every disk"
+            );
+        }
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
 use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
 use crate::object_api::{
     NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
-    SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
+    SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
 };
 use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
 use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
-use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
+use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
 #[cfg(test)]
 use crate::storage_api_contracts::namespace::NamespaceLocking;
 #[cfg(test)]
@@ -3548,6 +3548,7 @@ impl SetDisks {
                 (None, None, None)
             };
         let mut tmp_cleanup_owned = false;
+        let rollback_receipt = RenameRollbackReceipt::default();
         let operation = async {
             let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
 
@@ -4256,6 +4257,7 @@ impl SetDisks {
             let commit_bucket = bucket.to_owned();
             let commit_object = object.to_owned();
             let commit_tmp_dir = tmp_dir.clone();
+            let commit_rollback_receipt = rollback_receipt.clone();
             let commit_object_lock_guard = object_lock_guard.take();
             let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
             let commit_publication_guard = publication_commit_guard.take();
@@ -4266,13 +4268,17 @@ impl SetDisks {
             // complete rename fan-out drains. Keep this path synchronous so
             // its terminal state is known before the coordinator releases
             // remote leases.
-            let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
-                && (commit_object_lock_guard.is_some()
-                    || commit_decommission_object_lock_guard.is_some()
-                    || commit_publication_guard.is_some())
+            let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
+                || commit_decommission_object_lock_guard.is_some()
+                || commit_publication_guard.is_some();
+            let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
+                && !(opts.data_movement && opts.has_decommission_capacity_reservation())
+                && commit_owns_namespace_guard
                 && commit_scanner_publication_scope.is_none();
+            // Full-tail callers also transfer owned guards to the coordinator:
+            // cancelling their ACK waiter must not cancel an in-flight rename.
             let detach_commit_owner = commit_scanner_publication_scope.is_some()
-                || commit_allows_early_ack
+                || commit_owns_namespace_guard
                 || commit_bucket_lifecycle_guard.is_some()
                 || quota_mutation_fence;
             let commit_write_path_label = write_path.metric_label();
@@ -4452,7 +4458,11 @@ impl SetDisks {
                         write_quorum,
                         commit_scanner_publication_lease_tokens.as_ref(),
                     )
-                    .with_publication_scope(commit_scanner_publication_scope.clone()),
+                    .with_publication_scope(commit_scanner_publication_scope.clone())
+                    .with_rollback_receipt(commit_rollback_receipt.clone())
+                    .with_namespace_commit_guard(
+                        (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
+                    ),
                 )
                 .await;
                 if let Some(scope) = commit_scanner_publication_scope.as_ref() {
@@ -4585,6 +4595,11 @@ impl SetDisks {
                 let rename_commit = match rename_result {
                     Ok(commit) => commit,
                     Err(err) => {
+                        if commit_rollback_receipt.is_incomplete() {
+                            // Incomplete undo retains the staging source and
+                            // rollback backup for recovery; cleanup is unsafe.
+                            return Err(err.into());
+                        }
                         if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
                             warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
                         } else if issue3031_diag_enabled() {
@@ -4617,9 +4632,8 @@ impl SetDisks {
                     request.object_version_id = committed_version_id
                         .or_else(|| commit_version_suspended.then(Uuid::nil))
                         .map(|version_id| version_id.to_string());
-                    tokio::spawn(async move {
-                        let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
-                    });
+                    let heal_set = commit_set.clone();
+                    tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
                 }
 
                 let rename_stage_elapsed = rename_stage_start.elapsed();
@@ -4885,7 +4899,7 @@ impl SetDisks {
                     );
                 }
             });
-        } else {
+        } else if !rollback_receipt.is_incomplete() {
             // Failure path (quorum loss / rollback): keep the cleanup inline so
             // a failed PUT never returns while its tmp shards are still on disk
             // (state-residue hardening tracked by backlog#864 / backlog#898).
@@ -17494,27 +17508,69 @@ mod put_object_tmp_cleanup_tests {
     }
 
     #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
     async fn put_object_failure_cleans_tmp_workspace_inline() {
-        let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
+                let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
+                let bucket = "tmp-clean-missing-bucket";
+                let object = "orphan-object";
+                let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
+                let writer = Arc::clone(&set_disks);
+                let put = tokio::spawn(async move {
+                    let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
+                    writer
+                        .put_object(
+                            bucket,
+                            object,
+                            &mut reader,
+                            &ObjectOptions {
+                                write_completion,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                });
+                tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                    .await
+                    .expect("missing-bucket PUT must stage before rename");
+                let staged = non_trash_tmp_entries(&temp_dirs).await;
+                assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
+                for workspace in staged {
+                    let mut entries = tokio::fs::read_dir(&workspace)
+                        .await
+                        .expect("staged workspace should be readable");
+                    let mut shards = 0;
+                    while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
+                        if entry.file_type().await.expect("staged entry type").is_dir() {
+                            let part = tokio::fs::metadata(entry.path().join("part.1"))
+                                .await
+                                .expect("staging must contain an actual erasure shard");
+                            assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
+                            shards += 1;
+                        }
+                    }
+                    assert_eq!(shards, 1);
+                }
+                assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
+                barrier.release();
+                let err = tokio::time::timeout(Duration::from_secs(30), put)
+                    .await
+                    .expect("missing-bucket PUT must finish")
+                    .expect("PUT task should join")
+                    .expect_err("put_object into a missing bucket volume must fail");
+                assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
 
-        // The bucket volume is never created, so the shards are written into
-        // the tmp workspace and the commit fails at rename_data with a quorum
-        // error — exercising the failure-path cleanup.
-        let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
-        let err = set_disks
-            .put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
-            .await
-            .expect_err("put_object into a missing bucket volume must fail");
-
-        // No polling: the failure path must clean the tmp workspace inline,
-        // before put_object returns (backlog#864 / backlog#898 hardening).
-        let leftovers = non_trash_tmp_entries(&temp_dirs).await;
-        assert!(
-            leftovers.is_empty(),
-            "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
-        );
-
-        drop(temp_dirs);
+                // No polling: known pre-publication rejection must clean staging
+                // inline, before PUT returns (backlog#864 / backlog#898).
+                let leftovers = non_trash_tmp_entries(&temp_dirs).await;
+                assert!(
+                    leftovers.is_empty(),
+                    "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
+                );
+            }
+        })
+        .await;
     }
 
     #[tokio::test]
@@ -18157,6 +18213,354 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
+    async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
+        for disk in disks {
+            disk.make_volume(bucket)
+                .await
+                .expect("completion test bucket should be created");
+        }
+    }
+
+    /// Observe the actual metadata quorum while the remaining rename is parked.
+    /// A completed task count alone can race tasks that have not started yet.
+    async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let mut committed = 0;
+                for disk in disks {
+                    match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
+                        Ok(_) => committed += 1,
+                        Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
+                        Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
+                    }
+                }
+                if committed == 3 {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("three disks must publish metadata while the fourth rename remains paused");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for size in [4096, 1024 * 1024] {
+                let (_dirs, disks, set) = hermetic_set_disks(4).await;
+                let bucket = "put-full-tail-cas";
+                let object = "full-tail-cas-object";
+                make_completion_test_bucket(&disks, bucket).await;
+                let tasks = rename_fanout_barrier::observe_tasks(object);
+                let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+                let writer = Arc::clone(&set);
+                let put = tokio::spawn(async move {
+                    let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
+                    writer
+                        .put_object(
+                            bucket,
+                            object,
+                            &mut reader,
+                            &ObjectOptions {
+                                write_completion: WriteCompletion::TailDrained,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                });
+                tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                    .await
+                    .expect("full-tail PUT must reach the rename barrier");
+                wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+                assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
+                let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
+                assert!(
+                    futures::poll!(lock_probe.as_mut()).is_pending(),
+                    "the owned namespace guard must remain held"
+                );
+                barrier.release();
+                let written = tokio::time::timeout(Duration::from_secs(30), put)
+                    .await
+                    .expect("full-tail PUT should finish after release")
+                    .expect("full-tail PUT task should join")
+                    .expect("full-tail PUT must commit");
+                assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
+                drop(
+                    tokio::time::timeout(Duration::from_secs(5), lock_probe)
+                        .await
+                        .expect("same-key lock should be available on return")
+                        .expect("same-key lock probe should succeed"),
+                );
+                for disk in &disks {
+                    disk.read_version("", bucket, object, "", &ReadOptions::default())
+                        .await
+                        .expect("successful full-tail PUT must publish on every healthy disk");
+                }
+                drop(barrier);
+                let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
+                set.put_object(
+                    bucket,
+                    object,
+                    &mut replacement,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        http_preconditions: Some(HTTPPreconditions {
+                            if_match: written.etag,
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("immediate same-key CAS must acquire the namespace guard");
+                let mut read = set
+                    .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("CAS successor must be immediately readable");
+                let mut body = Vec::new();
+                read.stream.read_to_end(&mut body).await.expect("successor body must drain");
+                assert_eq!(body, b"cas successor");
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "put-full-tail-heal";
+        let object = "full-tail-heal-object";
+        make_completion_test_bucket(&disks, bucket).await;
+        let mut heals = set.capture_test_rename_tail_heals();
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
+        let writer = Arc::clone(&set);
+        let put = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+            writer
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("failed tail must first reach the rename barrier");
+        wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+        assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
+        barrier.release();
+        tokio::time::timeout(Duration::from_secs(30), put)
+            .await
+            .expect("failed tail should drain")
+            .expect("PUT task should join")
+            .expect("a minority tail error must not negate committed quorum");
+        assert_eq!(tasks.running(), 0);
+        let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
+            .await
+            .expect("failed tail must schedule heal")
+            .expect("heal capture must remain connected");
+        assert_eq!(heal.bucket, bucket);
+        assert_eq!(heal.object_prefix.as_deref(), Some(object));
+        let info = set
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("committed object must remain readable despite the failed tail");
+        assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_rejects_quorum_minus_one() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "put-full-tail-no-quorum";
+        let object = "full-tail-no-quorum-object";
+        make_completion_test_bucket(&disks, bucket).await;
+        let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+        let err = set
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    write_completion: WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("draining two successful disks cannot satisfy write quorum three");
+        assert!(
+            matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
+            "original quorum error expected: {err}"
+        );
+        assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
+        assert!(
+            set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
+            "failed fresh write must not become visible"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
+        use crate::set_disk::core::io_primitives::rollback_fault_injection;
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
+                for fault in [
+                    rollback_fault_injection::Fault::Io,
+                    rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
+                ] {
+                    let (dirs, disks, set) = hermetic_set_disks(4).await;
+                    let bucket = "put-incomplete-undo";
+                    let object = "incomplete-undo-object";
+                    make_completion_test_bucket(&disks, bucket).await;
+                    let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
+                    set.put_object(
+                        bucket,
+                        object,
+                        &mut old_reader,
+                        &ObjectOptions {
+                            write_completion: WriteCompletion::TailDrained,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("old generation should be completely committed");
+                    wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
+                    let old = disks[0]
+                        .read_version("", bucket, object, "", &ReadOptions::default())
+                        .await
+                        .expect("old metadata must be readable");
+                    let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
+                    let tasks = rename_fanout_barrier::observe_tasks(object);
+                    let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+                    let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
+                    let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
+                    let writer = Arc::clone(&set);
+                    let put = tokio::spawn(async move {
+                        let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+                        writer
+                            .put_object(
+                                bucket,
+                                object,
+                                &mut reader,
+                                &ObjectOptions {
+                                    write_completion,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                    });
+                    tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                        .await
+                        .expect("overwrite must enter the actual rename fan-out before failure injection");
+                    barrier.release();
+                    let err = tokio::time::timeout(Duration::from_secs(30), put)
+                        .await
+                        .expect("incomplete undo must return without hanging")
+                        .expect("PUT task should join")
+                        .expect_err("two renamed disks cannot satisfy write quorum three");
+                    assert!(
+                        matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
+                        "original quorum error expected: {err}"
+                    );
+                    assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
+                    let leftovers = non_trash_tmp_entries(&dirs).await;
+                    assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
+                    let backups = dirs
+                        .iter()
+                        .filter(|dir| {
+                            dir.path()
+                                .join(bucket)
+                                .join(object)
+                                .join(old_data_dir.to_string())
+                                .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
+                                .exists()
+                        })
+                        .count();
+                    assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
+                    // The remaining three disks still serve the old generation;
+                    // the failed minority must never become an acknowledged write.
+                    let mut read = set
+                        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                        .await
+                        .expect("old generation must remain readable after incomplete rollback");
+                    let mut body = Vec::new();
+                    read.stream
+                        .read_to_end(&mut body)
+                        .await
+                        .expect("old generation should stream");
+                    assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
+                }
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
+        let (dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = RUSTFS_META_BUCKET;
+        let object = "full-tail-cancelled-receipt";
+        // Internal config writes do not own a bucket lifecycle guard. The object
+        // guard alone must keep the full-tail coordinator alive after cancellation.
+        let tasks = rename_fanout_barrier::observe_tasks(object);
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let writer = Arc::clone(&set);
+        let put = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+            writer
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("cancelled receipt must first reach the rename barrier");
+        wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
+        put.abort();
+        assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
+        let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
+        assert!(
+            futures::poll!(lock_probe.as_mut()).is_pending(),
+            "owned coordinator must retain the namespace guard after waiter cancellation"
+        );
+        barrier.release();
+        drop(
+            tokio::time::timeout(Duration::from_secs(30), lock_probe)
+                .await
+                .expect("cancelled coordinator must eventually release its guard")
+                .expect("post-commit lock probe should succeed"),
+        );
+        assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
+        for disk in &disks {
+            disk.read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("caller cancellation must not interrupt committed receipt materialization");
+        }
+        wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
@@ -18184,6 +18588,7 @@ mod put_object_tmp_cleanup_tests {
                         &mut reader,
                         &ObjectOptions {
                             no_lock: true,
+                            write_completion: WriteCompletion::TailDrained,
                             ..Default::default()
                         },
                     )
@@ -18209,7 +18614,18 @@ mod put_object_tmp_cleanup_tests {
             put.await
                 .expect("no-lock PUT task should join")
                 .expect("no-lock PUT should commit after the rename tail releases");
+            let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
+            assert!(
+                futures::poll!(lock_probe.as_mut()).is_pending(),
+                "full-tail PUT must not release the caller's outer guard"
+            );
             drop(outer_guard);
+            drop(
+                tokio::time::timeout(Duration::from_secs(5), lock_probe)
+                    .await
+                    .expect("outer owner releasing its guard should unblock the probe")
+                    .expect("post-outer-guard probe should succeed"),
+            );
         })
         .await;
     }

crates/ecstore/src/set_disk/transition_matrix_tests.rs

@@ -18,6 +18,7 @@ use super::{
 };
 use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
 use crate::ecstore_validation_blackbox::make_local_set_disks;
+use crate::object_api::WriteCompletion;
 use crate::services::tier::test_util::register_mock_tier;
 use crate::storage_api_contracts::bucket::BucketOperations;
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -25,19 +26,24 @@ use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
 use tokio::io::AsyncReadExt;
 
 async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
-    set_disks
-        .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
-        .await
-        .expect("object metadata should resolve");
-    let generation = set_disks
-        .get_object_metadata_cache_generation(bucket, object)
-        .expect("metadata generation should be active");
-    let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
-    assert!(
-        set_disks.get_object_metadata_cache.get(&key).await.is_some(),
-        "metadata read should publish the generation under test"
-    );
-    key
+    tokio::time::timeout(Duration::from_secs(30), async {
+        loop {
+            set_disks
+                .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+                .await
+                .expect("object metadata should resolve");
+            let generation = set_disks
+                .get_object_metadata_cache_generation(bucket, object)
+                .expect("metadata generation should be active");
+            let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+            if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
+                return key;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("metadata read should publish the generation under test")
 }
 
 async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
@@ -60,8 +66,17 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
         .await
         .expect("bucket should be created");
     let mut reader = PutObjReader::from_vec(payload.clone());
+    // Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
     let original = set_disks
-        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+        .put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
         .await
         .expect("source object should be written");
     let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;
@@ -164,8 +179,17 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
         .await
         .expect("bucket should be created");
     let mut reader = PutObjReader::from_vec(payload.clone());
+    // Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
     let original = set_disks
-        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+        .put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
         .await
         .expect("source object should be written");
 

crates/ecstore/src/store/bucket.rs

@@ -329,11 +329,11 @@ impl ECStore {
     /// reuse its result, which is sound because bucket deletion/recreation
     /// requires the lifecycle WRITE lock and therefore cannot have run while
     /// any read guard was continuously held.
-    pub(crate) async fn acquire_bucket_incarnation_fence(
+    pub async fn acquire_bucket_incarnation_fence(
         &self,
         bucket: &str,
         expected: uuid::Uuid,
-    ) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
+    ) -> Result<super::BucketIncarnationFenceGuard> {
         let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
         let pieces = super::bucket_fence::FencePieces {
             registry: self.bucket_fence_registry.clone(),
@@ -1059,6 +1059,7 @@ mod tests {
     use crate::storage_api_contracts::{
         bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
         list::ListOperations as _,
+        namespace::NamespaceLocking as _,
         object::{ObjectIO as _, ObjectOperations as _},
     };
     use crate::store::{ECStore, init_local_disks_with_instance_ctx};
@@ -1486,10 +1487,19 @@ mod tests {
             .put_object(bucket, object, &mut reader, &ObjectOptions::default())
             .await
             .expect("object should be written");
+        let lock = ecstore.pools[0].disk_set[0]
+            .new_ns_lock(bucket, object)
+            .await
+            .expect("fixture namespace lock should be created");
+        drop(
+            lock.get_write_lock(Duration::from_secs(30))
+                .await
+                .expect("fixture rename tail should finish before checking its generation"),
+        );
         assert_eq!(
             ecstore.scanner_namespace_mutation_generation(),
-            generation_before_put.saturating_add(1),
-            "successful object creation should advance scanner namespace activity"
+            generation_before_put.saturating_add(3),
+            "successful object creation must observe the logical mutation and both fanout boundaries"
         );
         ecstore
             .get_object_info(bucket, object, &ObjectOptions::default())

crates/ecstore/src/store/bucket_fence.rs

@@ -150,7 +150,7 @@ impl BucketFenceRegistry {
 /// A held bucket lifecycle read lock plus its registration in the fence
 /// registry. Dropping the guard deregisters it; the memo is cleared when the
 /// last guard for the bucket drops (or a lost lock is observed).
-pub(crate) struct BucketIncarnationFenceGuard {
+pub struct BucketIncarnationFenceGuard {
     inner: Option<NamespaceLockGuard>,
     registry: Arc<BucketFenceRegistry>,
     bucket: String,
@@ -158,6 +158,14 @@ pub(crate) struct BucketIncarnationFenceGuard {
 }
 
 impl BucketIncarnationFenceGuard {
+    /// Propagate lifecycle lock loss into the storage commit checks.
+    /// The caller still owns this guard until the complete write tail drains.
+    pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
+        if let Some(guard) = self.namespace_lock_guard() {
+            opts.add_bucket_lifecycle_lock_guard(guard);
+        }
+    }
+
     pub(crate) fn is_lock_lost(&self) -> bool {
         self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
     }
@@ -346,6 +354,36 @@ mod tests {
         first_pieces.abandon("b", first.token);
     }
 
+    #[tokio::test]
+    async fn checkpoint_options_inherit_bucket_fence_lock_loss() {
+        let lock = NamespaceLock::new("bucket-fence-options".to_string(), Arc::new(LocalClient::new()));
+        let inner = lock
+            .acquire_guard(&lock_request("options"))
+            .await
+            .expect("acquire")
+            .expect("quorum");
+        let pieces = FencePieces {
+            registry: Arc::default(),
+            inner,
+        };
+        let registration = pieces.enter("b");
+        let fence = pieces.into_guard("b", registration.token);
+        let mut opts = crate::object_api::ObjectOptions::default();
+        fence.attach_to_object_options(&mut opts);
+        let inherited = opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .expect("checkpoint inherits lifecycle guard");
+        assert!(!inherited.is_lock_lost());
+        tokio::time::timeout(
+            Duration::from_secs(2),
+            fence.namespace_lock_guard().expect("held guard").lock_lost_notified(),
+        )
+        .await
+        .expect("distributed guard expires");
+        assert!(inherited.is_lock_lost(), "the actual pre-rename options must observe lifecycle lock loss");
+    }
+
     #[test]
     fn buckets_are_isolated() {
         let reg = BucketFenceRegistry::default();

crates/ecstore/src/store/mod.rs

@@ -417,6 +417,7 @@ const MAX_UPLOADS_LIST: usize = 10000;
 mod bucket;
 mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
+pub use bucket_fence::BucketIncarnationFenceGuard;
 mod heal;
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
@@ -425,7 +426,7 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 pub use object::DeleteAfterObjectLockSnapshotBarrier;
 pub(crate) use object::{
     DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
@@ -848,7 +849,7 @@ impl ECStore {
     }
 
     pub fn scanner_namespace_mutation_generation(&self) -> u64 {
-        list_objects::scanner_namespace_mutation_generation()
+        list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
     }
 
     pub async fn scanner_data_movement_active(&self) -> bool {
@@ -857,7 +858,7 @@ impl ECStore {
     }
 
     /// Return the storage-owned movement state and generation as one
-    /// authenticated activity snapshot.  The read lock is acquired before
+    /// authenticated activity snapshot. The read lock is acquired before
     /// the state locks (cancelers, pool metadata, then rebalance metadata),
     /// matching the transition writer order and preventing a terminal state
     /// from being reported with the preceding generation.
@@ -886,11 +887,12 @@ impl ECStore {
     /// Returns whether scanner metadata may still be hidden by a local
     /// data-movement state. Terminal failed/canceled decommission entries
     /// remain suspended until an operator clears or retries them, so they are
-    /// a publication barrier even after the worker has stopped.
+    /// a publication barrier even after the worker has stopped. Active PUT
+    /// rename fanouts also defer publication, including post-ACK tails.
     pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
         let operation_gate = self.ctx.data_movement_operation_gate();
         let _operation_guard = operation_gate.read_owned().await;
-        self.scanner_data_usage_publication_snapshot_blocked().await
+        self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
     }
 
     pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
@@ -1070,7 +1072,7 @@ impl ECStore {
         {
             return Err(Error::other("scanner publication lease generation is stale"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
 
@@ -1109,7 +1111,7 @@ impl ECStore {
         {
             return Err(Error::other("scanner publication lease generation is stale"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
         if !self.ctx.scanner_publication_lease_is_active(token).await {
@@ -1129,7 +1131,7 @@ impl ECStore {
         if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
             return Err(Error::other("scanner publication lease generation is exhausted"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
         let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {

crates/filemeta/src/filemeta/version.rs

@@ -297,6 +297,20 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio
     }
 }
 
+fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String {
+    decoded.map(str::to_owned).unwrap_or_else(|| {
+        if raw.is_empty() {
+            String::new()
+        } else {
+            String::from_utf8_lossy(raw).into_owned()
+        }
+    })
+}
+
+fn is_transition_version_metadata_key(key: &str) -> bool {
+    strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
+}
+
 fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
     let valid = match state {
         TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
@@ -366,14 +380,26 @@ impl<'a> DerivedInternalMetadata<'a> {
             }
             *slot = Some(value.as_slice());
         }
+        fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result<Option<&'a [u8]>> {
+            if let (Some(canonical), Some(legacy)) = (canonical, legacy)
+                && canonical != legacy
+            {
+                return Err(Error::FileCorrupt);
+            }
+            Ok(canonical.or(legacy))
+        }
+
         Ok(Self {
             checksum: canonical.checksum.or(legacy.checksum),
             part_checksums: canonical.part_checksums.or(legacy.part_checksums),
-            transition_status: canonical.transition_status.or(legacy.transition_status),
-            transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
-            transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
-            transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
-            transition_tier: canonical.transition_tier.or(legacy.transition_tier),
+            transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?,
+            transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?,
+            transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?,
+            transitioned_version_state: merge_consistent(
+                canonical.transitioned_version_state,
+                legacy.transitioned_version_state,
+            )?,
+            transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?,
         })
     }
 }
@@ -438,8 +464,14 @@ impl FileInfo {
     }
 }
 
-fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
-    if state == TransitionVersionState::Unknown {
+fn set_transition_version_state(
+    meta_sys: &mut HashMap<String, Vec<u8>>,
+    state: TransitionVersionState,
+    source_metadata: &HashMap<String, String>,
+) {
+    if state == TransitionVersionState::Unknown
+        && !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE)
+    {
         remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
     } else {
         insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
@@ -2643,6 +2675,11 @@ impl MetaObject {
         if derived_metadata.transitioned_version_state.is_some() {
             validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
         }
+        for (key, value) in &self.meta_sys {
+            if is_transition_version_metadata_key(key) {
+                metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref()));
+            }
+        }
         let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
         let transition_tier = derived_metadata
             .transition_tier
@@ -2689,7 +2726,7 @@ impl MetaObject {
         } else {
             remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
         }
-        set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
+        set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata);
         insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
         if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
             insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
@@ -2830,7 +2867,7 @@ impl From<FileInfo> for MetaObject {
             insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
         }
         if !value.transition_status.is_empty() {
-            set_transition_version_state(&mut meta_sys, value.transition_version_state);
+            set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
         }
 
         if !value.transition_tier.is_empty() {
@@ -2985,6 +3022,12 @@ impl MetaDeleteMarker {
             fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
             fi.transition_version =
                 transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
+            for (key, value) in &self.meta_sys {
+                if is_transition_version_metadata_key(key) {
+                    fi.metadata
+                        .insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref()));
+                }
+            }
             fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
             if derived_metadata.transitioned_version_state.is_some() {
                 validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
@@ -3152,7 +3195,7 @@ impl From<FileInfo> for MetaDeleteMarker {
             insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
         }
         if !value.transition_status.is_empty() || value.tier_free_version() {
-            set_transition_version_state(&mut meta_sys, value.transition_version_state);
+            set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
         }
         if !value.transition_tier.is_empty() {
             insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
@@ -4574,6 +4617,7 @@ mod tests {
             .into_fileinfo("b", "k", false)
             .expect("into_fileinfo");
         assert_eq!(fi.transition_version_id, None);
+        assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new()));
     }
 
     #[test]
@@ -4585,6 +4629,10 @@ mod tests {
             .into_fileinfo("b", "k", false)
             .expect("into_fileinfo");
         assert_eq!(fi.transition_version_id, None);
+        assert!(
+            get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
+            "nil UUID bytes must remain distinguishable from an empty MinIO version"
+        );
     }
 
     #[test]
@@ -4598,6 +4646,7 @@ mod tests {
         assert_eq!(fi.transition_version_id, Some(id));
         assert_eq!(fi.transition_version, Some(id.to_string()));
         assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
+        assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
     }
 
     #[test]
@@ -4637,6 +4686,36 @@ mod tests {
         assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
     }
 
+    #[test]
+    fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            SUFFIX_TRANSITIONED_VERSION_STATE,
+            TransitionVersionState::Unknown.as_str().to_string(),
+        );
+        let fi = FileInfo {
+            transition_status: "complete".to_string(),
+            transition_version_state: TransitionVersionState::Unknown,
+            metadata,
+            ..Default::default()
+        };
+
+        let object = MetaObject::from(fi);
+        assert_eq!(
+            get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE),
+            Some(b"unknown".as_slice())
+        );
+        let decoded = object
+            .into_fileinfo("b", "k", false)
+            .expect("explicit unknown state should decode");
+        assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown);
+        assert_eq!(
+            rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,),
+            Some("unknown")
+        );
+    }
+
     #[test]
     fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
         let id = sample_version_id();
@@ -4753,6 +4832,10 @@ mod tests {
             .expect("invalid transition version bytes must not fail the object read");
         assert_eq!(fi.transition_version_id, None);
         assert_eq!(fi.transition_version, None);
+        assert!(
+            get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
+            "invalid raw bytes must remain distinguishable from an empty MinIO version"
+        );
     }
 
     #[test]
@@ -4795,6 +4878,10 @@ mod tests {
         .into_fileinfo("b", "k", false)
         .expect("nil tier version should remain an absent remote version");
         assert_eq!(fi.transition_version_id, None);
+        assert!(
+            get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
+            "nil UUID bytes must remain distinguishable from an empty MinIO version"
+        );
     }
 
     #[test]
@@ -4812,6 +4899,7 @@ mod tests {
         .expect("legacy binary UUID tier version should decode");
         assert_eq!(fi.transition_version_id, Some(id));
         assert_eq!(fi.transition_version, Some(id.to_string()));
+        assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
     }
 
     #[test]
@@ -4910,6 +4998,23 @@ mod tests {
         assert_eq!(err, Error::FileCorrupt);
     }
 
+    #[test]
+    fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() {
+        let sys = HashMap::from([
+            (
+                format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"),
+                b"unknown".to_vec(),
+            ),
+            ("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()),
+        ]);
+
+        let err = make_meta_object_with_sys(sys)
+            .into_fileinfo("b", "k", false)
+            .expect_err("mixed-case transition state aliases must agree");
+
+        assert_eq!(err, Error::FileCorrupt);
+    }
+
     #[test]
     fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
         let object = FileMetaVersionHeader {

crates/heal/src/heal/manager.rs

@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
 use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
 
 const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
+// Each cache includes alias tokens in its count and byte budget. Eviction
+// removes every token sharing a snapshot; neither cache retains repair state.
+const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
+const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
+const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
 const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
 const LOG_COMPONENT_HEAL: &str = "heal";
 const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
     request: &HealRequest,
 ) -> Arc<CompletedHealStatus> {
     let terminal = Arc::new(CompletedHealStatus {
+        progress: None,
+        retained_bytes: std::sync::OnceLock::new(),
         heal_type: request.heal_type.clone(),
         status: HealTaskStatus::Failed {
             error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
     let mut terminals = lock_displaced_terminals(registry);
     prune_completed_heal_statuses(&mut terminals);
     terminals.insert(request.id.clone(), Arc::clone(&terminal));
+    prune_completed_heal_statuses(&mut terminals);
     terminal
 }
 
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
         .collect::<Vec<_>>();
     let mut displaced_terminals = lock_displaced_terminals(terminals);
     prune_completed_heal_statuses(&mut displaced_terminals);
-    for alias_id in alias_ids {
-        displaced_terminals.insert(alias_id, Arc::clone(terminal));
+    if displaced_terminals
+        .get(task_id)
+        .is_some_and(|current| Arc::ptr_eq(current, terminal))
+    {
+        for alias_id in alias_ids {
+            displaced_terminals.insert(alias_id, Arc::clone(terminal));
+        }
     }
+    prune_completed_heal_statuses(&mut displaced_terminals);
     aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
 }
 
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
         .retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
 }
 
+// Callers hold active ownership until publication. Lock order is active ->
+// retrying (when needed) -> aliases -> completed; queries release aliases
+// before looking up active state. Publishing aliases before removing their
+// mapping keeps both an already-resolved token and a new lookup valid.
+async fn publish_completed_heal(
+    completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
+    task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
+    task_id: &str,
+    completed: CompletedHealStatus,
+    terminal: bool,
+) {
+    let completed = Arc::new(completed);
+    completed.retained_bytes();
+    let mut aliases = task_aliases.lock().await;
+    let mut retained = completed_heals.lock().await;
+    if let Some(previous) = retained.get(task_id).cloned() {
+        for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
+            *entry = Arc::clone(&completed);
+        }
+    }
+    retained.insert(task_id.to_owned(), Arc::clone(&completed));
+    if terminal {
+        for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
+            retained.insert(alias_id.clone(), Arc::clone(&completed));
+        }
+        aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
+    }
+    prune_completed_heal_statuses(&mut retained);
+}
+
 #[derive(Debug, Clone)]
 pub struct HealTaskReport {
     pub status: HealTaskStatus,
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
     let result_items = match since {
         None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
         Some(cursor) => {
-            if cursor + 1 < completed.min_seq {
+            if cursor.saturating_add(1) < completed.min_seq {
                 lagged = true;
             }
             completed
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
         status: completed.status.clone(),
         result_items,
         result_items_truncated: completed.result_items_truncated || lagged,
-        progress: None,
+        progress: completed.progress.clone(),
         next_seq: completed.next_seq,
         min_seq: completed.min_seq,
     }
@@ -1847,14 +1891,14 @@ impl HealManager {
 
     pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
         let canonical_task_id = self.canonical_task_id(task_id).await;
-        let active_heals = self.active_heals.lock().await;
-        if let Some(task) = active_heals.get(&canonical_task_id) {
-            Ok(task.get_progress().await)
-        } else {
-            Err(Error::TaskNotFound {
-                task_id: task_id.to_string(),
-            })
-        }
+        let progress = match self.lookup_task_state(&canonical_task_id, None).await {
+            TaskStateLookup::Active(task) => Some(task.get_progress().await),
+            TaskStateLookup::Completed(completed) => completed.progress.clone(),
+            _ => None,
+        };
+        progress.ok_or_else(|| Error::TaskNotFound {
+            task_id: task_id.to_string(),
+        })
     }
 
     /// Cancel task
@@ -1864,6 +1908,8 @@ impl HealManager {
             let mut active_heals = self.active_heals.lock().await;
             if let Some(task) = active_heals.get(&canonical_task_id) {
                 task.cancel().await?;
+                let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
+                publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
                 active_heals.remove(&canonical_task_id);
                 publish_active_heal_count(&active_heals);
                 info!(
@@ -1940,6 +1986,8 @@ impl HealManager {
             for task_id in &task_ids {
                 if let Some(task) = active_heals.get(task_id) {
                     task.cancel().await?;
+                    let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
+                    publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
                 }
                 active_heals.remove(task_id);
                 cancelled += 1;

crates/heal/src/heal/manager/queue.rs

@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
 pub(super) struct CompletedHealStatus {
     pub(super) heal_type: HealType,
     pub(super) status: HealTaskStatus,
+    pub(super) progress: Option<HealProgress>,
+    pub(super) retained_bytes: std::sync::OnceLock<usize>,
     pub(super) result_items_truncated: bool,
     pub(super) completed_at: SystemTime,
     /// Sequence-stamped retained window, archived with the completion so
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
     pub(super) min_seq: u64,
 }
 
+impl CompletedHealStatus {
+    // Account for owned capacities, including nested drive arrays. Aliases
+    // conservatively charge the shared allocation again, keeping both token
+    // count and retained payload bounded without a second ownership index.
+    pub(super) fn retained_bytes(&self) -> usize {
+        *self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
+    }
+
+    fn measure_retained_bytes(&self) -> usize {
+        let mut bytes = size_of::<Self>();
+        let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
+        match &self.heal_type {
+            HealType::Cluster => {}
+            HealType::Bucket { bucket } => add(bucket.capacity()),
+            HealType::Object {
+                bucket,
+                object,
+                version_id,
+            }
+            | HealType::ECDecode {
+                bucket,
+                object,
+                version_id,
+            } => {
+                add(bucket.capacity());
+                add(object.capacity());
+                add(version_id.as_ref().map_or(0, String::capacity));
+            }
+            HealType::Prefix { bucket, prefix } => {
+                add(bucket.capacity());
+                add(prefix.capacity());
+            }
+            HealType::Metadata { bucket, object } => {
+                add(bucket.capacity());
+                add(object.capacity());
+            }
+            HealType::ErasureSet { buckets, set_disk_id } => {
+                add(buckets.capacity().saturating_mul(size_of::<String>()));
+                for bucket in buckets {
+                    add(bucket.capacity());
+                }
+                add(set_disk_id.capacity());
+            }
+        }
+        if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
+            add(error.capacity());
+        }
+        add(self
+            .progress
+            .as_ref()
+            .and_then(|progress| progress.current_object.as_ref())
+            .map_or(0, String::capacity));
+        add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
+        for (_, item) in &self.seqed_items {
+            add(Self::result_item_heap_bytes(item));
+        }
+        bytes
+    }
+
+    fn result_item_heap_bytes(item: &HealResultItem) -> usize {
+        let mut bytes = 0usize;
+        let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
+        for value in [
+            &item.heal_item_type,
+            &item.bucket,
+            &item.object,
+            &item.version_id,
+            &item.detail,
+        ] {
+            add(value.capacity());
+        }
+        for infos in [&item.before, &item.after] {
+            add(infos
+                .drives
+                .capacity()
+                .saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
+            for drive in &infos.drives {
+                add(drive.uuid.capacity());
+                add(drive.endpoint.capacity());
+                add(drive.state.capacity());
+            }
+        }
+        bytes
+    }
+
+    pub(super) fn bound_result_window(&mut self) {
+        let mut bytes = 0usize;
+        let retained = self
+            .seqed_items
+            .iter()
+            .rev()
+            .take_while(|(_, item)| {
+                bytes = bytes
+                    .saturating_add(size_of::<(u64, HealResultItem)>())
+                    .saturating_add(Self::result_item_heap_bytes(item));
+                bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
+            })
+            .count();
+        let truncated = retained < self.seqed_items.len();
+        if truncated {
+            self.seqed_items.drain(..self.seqed_items.len() - retained);
+            self.seqed_items.shrink_to_fit();
+            self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
+            self.result_items_truncated = true;
+            self.retained_bytes.take();
+        }
+    }
+
+    pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
+        let seqed_items = task.get_seqed_result_items().await;
+        let (next_seq, min_seq) = task.result_seq_cursors();
+        let mut snapshot = Self {
+            heal_type: task.heal_type.clone(),
+            status,
+            progress: Some(task.get_progress().await),
+            retained_bytes: std::sync::OnceLock::new(),
+            result_items_truncated: task.result_items_truncated(),
+            completed_at: SystemTime::now(),
+            seqed_items,
+            next_seq,
+            min_seq,
+        };
+        snapshot.bound_result_window();
+        snapshot
+    }
+}
+
 #[derive(Debug, Clone)]
 pub(super) struct HealTaskAlias {
     pub(super) task_id: String,

crates/heal/src/heal/manager/scheduler.rs

@@ -264,7 +264,7 @@ impl HealManager {
                         error: error.clone(),
                         retry_attempt: request.retry_attempts,
                     });
-                    let retry_request_for_queue = retry_request;
+                    let mut retry_request_for_queue = retry_request;
                     let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
                     if retry_request_for_queue.is_none() {
                         replacement_recovery_anchors_clone
@@ -272,7 +272,35 @@ impl HealManager {
                             .unwrap_or_else(|poisoned| poisoned.into_inner())
                             .remove(&task_id);
                     }
+                    let mut completed_status = match retry_request_for_status {
+                        Some(status) => status,
+                        None => task.get_status().await,
+                    };
+                    let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
+                    let completed_progress = task.get_progress().await;
+                    #[cfg(test)]
+                    tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
                     let mut active_heals_guard = active_heals_clone.lock().await;
+                    let owns_completion = active_heals_guard.contains_key(&task_id);
+                    let cancelled_completion = if owns_completion {
+                        false
+                    } else {
+                        // Cancellation can win while a finished worker waits
+                        // for active ownership. It must not resurrect a retry
+                        // or replace an acknowledged cancellation with success.
+                        retry_request_for_queue = None;
+                        completed_heals_clone
+                            .lock()
+                            .await
+                            .get(&task_id)
+                            .is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
+                    };
+                    if cancelled_completion {
+                        completed_status = HealTaskStatus::Cancelled;
+                        completed_status_entry.status = HealTaskStatus::Cancelled;
+                    }
+                    let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
+                    let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
                     // Keep retry ownership continuous: status snapshots acquire
                     // these locks in the same active -> retrying order.
                     let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -295,6 +323,16 @@ impl HealManager {
                     } else {
                         None
                     };
+                    if owns_completion || cancelled_completion {
+                        publish_completed_heal(
+                            &completed_heals_clone,
+                            &task_aliases_clone,
+                            &task_id,
+                            completed_status_entry,
+                            terminal_completion,
+                        )
+                        .await;
+                    }
                     let completed_task = active_heals_guard.remove(&task_id);
                     if let Some(completed_task) = completed_task.as_ref() {
                         publish_active_heal_count(&active_heals_guard);
@@ -304,33 +342,10 @@ impl HealManager {
                     drop(retrying_heals_guard.take());
                     drop(active_heals_guard);
 
-                    if let Some(completed_task) = completed_task {
-                        let completed_status = if let Some(status) = retry_request_for_status {
-                            status
-                        } else {
-                            completed_task.get_status().await
-                        };
-                        let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
-                        let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
-                        let completed_progress = completed_task.get_progress().await;
-                        // Single snapshot of the retained window: the task is
-                        // finished and already off the active map, so there is
-                        // no concurrent writer to race with.
-                        let seqed_items = completed_task.get_seqed_result_items().await;
-                        let (next_seq, min_seq) = completed_task.result_seq_cursors();
-                        let completed_status_entry = CompletedHealStatus {
-                            heal_type: completed_task.heal_type.clone(),
-                            status: completed_status.clone(),
-                            result_items_truncated: completed_task.result_items_truncated(),
-                            completed_at: SystemTime::now(),
-                            seqed_items,
-                            next_seq,
-                            min_seq,
-                        };
-                        let mut completed_heals_guard = completed_heals_clone.lock().await;
-                        prune_completed_heal_statuses(&mut completed_heals_guard);
-                        completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
-                        drop(completed_heals_guard);
+                    #[cfg(test)]
+                    tests::pause_completed_retention_handoff(&task_id).await;
+
+                    if completed_task.is_some() {
                         // update statistics
                         let mut stats = statistics_clone.write().await;
                         match completed_status {
@@ -352,10 +367,6 @@ impl HealManager {
                             } else {
                                 release_mrf_repair_notice_targets(notice_targets);
                             }
-                            task_aliases_clone
-                                .lock()
-                                .await
-                                .retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
                         }
                     }
 
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
 }
 
 pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
-    let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
-        return;
-    };
+    prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
+}
 
+pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
     completed_heals.retain(|_, completed| {
-        completed
-            .completed_at
-            .duration_since(SystemTime::UNIX_EPOCH)
-            .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
+        now.duration_since(completed.completed_at)
+            .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
             .unwrap_or(false)
     });
+    let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
+        key.capacity()
+            .saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
+            .saturating_add(value.retained_bytes())
+    };
+    let mut bytes = completed_heals
+        .iter()
+        .fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
+    while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
+        let Some(oldest) = completed_heals
+            .iter()
+            .min_by(|(left_id, left), (right_id, right)| {
+                left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
+            })
+            .map(|(_, value)| Arc::clone(value))
+        else {
+            break;
+        };
+        completed_heals.retain(|key, value| {
+            if Arc::ptr_eq(value, &oldest) {
+                bytes = bytes.saturating_sub(entry_bytes(key, value));
+                false
+            } else {
+                true
+            }
+        });
+    }
 }
 
 pub(super) fn can_schedule_request(

crates/heal/src/heal/manager/tests.rs

@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
 
 struct MockStorage;
 
+fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
+    CompletedHealStatus {
+        heal_type: HealType::Cluster,
+        status: HealTaskStatus::Completed,
+        progress: Some(HealProgress {
+            objects_scanned: 9,
+            objects_healed: 8,
+            objects_failed: 1,
+            ..Default::default()
+        }),
+        retained_bytes: std::sync::OnceLock::new(),
+        result_items_truncated: false,
+        completed_at,
+        seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
+        next_seq: 5,
+        min_seq: 3,
+    }
+}
+
+#[test]
+fn completed_retention_cursor_boundaries_preserve_progress() {
+    let completed = completed_retention_fixture(SystemTime::now());
+    for (cursor, count, lagged) in [
+        (0, 2, true),
+        (1, 2, true),
+        (2, 2, false),
+        (3, 1, false),
+        (4, 0, false),
+        (5, 0, false),
+        (u64::MAX, 0, false),
+    ] {
+        let report = completed_task_report(&completed, Some(cursor));
+        assert_eq!(report.result_items.len(), count, "cursor={cursor}");
+        assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
+        assert_eq!(report.progress, completed.progress);
+        assert_eq!((report.next_seq, report.min_seq), (5, 3));
+    }
+    assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
+}
+
+#[tokio::test]
+async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::bucket("bucket".to_string());
+    manager.insert_task_alias("alias", &request.id).await;
+    let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
+    lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
+    remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
+    for token in [&request.id, &"alias".to_string()] {
+        assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
+    }
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
+}
+
+#[test]
+fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
+    let now = SystemTime::now();
+    let mut entries = HashMap::new();
+    let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
+    entries.insert("oldest".to_string(), Arc::clone(&oldest));
+    entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
+    for index in 2..MAX_COMPLETED_HEAL_TOKENS {
+        entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
+    }
+    prune_completed_heal_statuses_at(&mut entries, now);
+    assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
+    entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
+    prune_completed_heal_statuses_at(&mut entries, now);
+    assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
+    assert!(!entries.contains_key("oldest"));
+    assert!(!entries.contains_key("oldest-alias"));
+    entries.clear();
+    entries.insert("ttl-boundary".to_string(), oldest);
+    entries.insert(
+        "expired".to_string(),
+        Arc::new(completed_retention_fixture(
+            now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
+        )),
+    );
+    entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
+    prune_completed_heal_statuses_at(&mut entries, now);
+    assert_eq!(entries.len(), 1);
+    assert!(entries.contains_key("ttl-boundary"));
+    prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
+    assert!(entries.is_empty());
+}
+
+#[test]
+fn completed_retention_total_byte_cap_and_cap_plus_one() {
+    let now = SystemTime::now();
+    let key = "large".to_string();
+    let mut entry = completed_retention_fixture(now);
+    let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
+    entry.retained_bytes.take();
+    entry.status = HealTaskStatus::Failed {
+        error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
+    };
+    assert_eq!(
+        entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
+        MAX_COMPLETED_HEAL_BYTES
+    );
+    let mut entries = HashMap::from([(key, Arc::new(entry))]);
+    prune_completed_heal_statuses_at(&mut entries, now);
+    assert_eq!(entries.len(), 1, "exact byte cap remains retained");
+    let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
+    over.retained_bytes.take();
+    if let HealTaskStatus::Failed { error } = &mut over.status {
+        *error = "x".repeat(error.len() + 1);
+    }
+    entries.insert("large".to_string(), Arc::new(over));
+    prune_completed_heal_statuses_at(&mut entries, now);
+    assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
+}
+
+#[tokio::test]
+async fn completed_retention_large_window_keeps_cursors_and_progress() {
+    let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
+    let mut snapshot = completed_retention_fixture(SystemTime::now());
+    snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
+    snapshot.bound_result_window();
+    assert_eq!(snapshot.seqed_items.len(), 1);
+    assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
+    assert!(snapshot.result_items_truncated);
+    assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
+    let report = completed_task_report(&snapshot, Some(0));
+    assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
+    assert!(report.result_items_truncated);
+    let active_max = task.get_result_items_since(Some(u64::MAX)).await;
+    assert!(active_max.items.is_empty());
+    assert!(!active_max.lagged);
+}
+
+#[test]
+fn completed_retention_result_byte_cap_and_cap_plus_one() {
+    for extra in [0, 1] {
+        let mut snapshot = completed_retention_fixture(SystemTime::now());
+        snapshot.seqed_items = vec![(
+            4,
+            HealResultItem {
+                detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
+                ..Default::default()
+            },
+        )];
+        snapshot.min_seq = 4;
+        snapshot.bound_result_window();
+        assert_eq!(snapshot.seqed_items.len(), 1 - extra);
+        assert_eq!(snapshot.result_items_truncated, extra == 1);
+        assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
+        assert_eq!(snapshot.next_seq, 5);
+        assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
+    }
+}
+
+#[derive(Default)]
+struct CompletedRetentionHook {
+    started: Notify,
+    execute: Notify,
+    handoff: Notify,
+    finish: Notify,
+    pause_before_publish: bool,
+    before_publish: Notify,
+    publish: Notify,
+    prepared_status: Mutex<Option<HealTaskStatus>>,
+}
+
+static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
+    let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
+    if let Some(hook) = hook {
+        hook.handoff.notify_one();
+        hook.finish.notified().await;
+    }
+}
+
+pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
+    let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
+    if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
+        *hook.prepared_status.lock().await = Some(status.clone());
+        hook.before_publish.notify_one();
+        hook.publish.notified().await;
+    }
+}
+
+#[tokio::test]
+async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
+    let bucket = "completed-retention-retry-cancel";
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
+    let alias = duplicate.id.clone();
+    let hook = Arc::new(CompletedRetentionHook {
+        pause_before_publish: true,
+        ..Default::default()
+    });
+    {
+        let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
+        hooks.insert(bucket.to_string(), Arc::clone(&hook));
+        hooks.insert(task_id.clone(), Arc::clone(&hook));
+    }
+    manager.submit_heal_request(request).await.expect("admit original");
+    manager.submit_heal_request(duplicate).await.expect("admit alias");
+    process_manager_queue_once(&manager).await;
+    tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
+        .await
+        .expect("scheduler starts");
+    let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
+    task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
+    hook.execute.notify_one();
+    tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
+        .await
+        .expect("retry snapshot prepared");
+    manager.cancel_task(&alias).await.expect("cancel wins active ownership");
+    assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
+    hook.publish.notify_one();
+    tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+        .await
+        .expect("scheduler finishes handoff");
+    for token in [&task_id, &alias] {
+        let report = manager.get_task_report(token).await.expect("cancelled token retained");
+        assert_eq!(report.status, HealTaskStatus::Cancelled);
+        assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
+    }
+    assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
+    assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
+    hook.finish.notify_one();
+    COMPLETED_RETENTION_HOOKS
+        .lock()
+        .await
+        .retain(|key, _| key != bucket && key != &task_id);
+}
+
+#[tokio::test]
+async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
+    for outcome in ["success", "failed", "cancelled"] {
+        let bucket = format!("completed-retention-{outcome}");
+        let hook = Arc::new(CompletedRetentionHook::default());
+        let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
+        let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
+        let task_id = request.id.clone();
+        let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
+        let alias = duplicate.id.clone();
+        {
+            let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
+            hooks.insert(bucket.clone(), Arc::clone(&hook));
+            hooks.insert(task_id.clone(), Arc::clone(&hook));
+        }
+        manager.submit_heal_request(request).await.expect("admit original");
+        manager.submit_heal_request(duplicate).await.expect("admit alias");
+        process_manager_queue_once(&manager).await;
+        tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
+            .await
+            .expect("scheduler reaches storage");
+        let task = manager
+            .active_heals
+            .lock()
+            .await
+            .get(&task_id)
+            .cloned()
+            .expect("task is active");
+        task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
+        let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
+        assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
+        let poll_manager = Arc::clone(&manager);
+        let poll_alias = alias.clone();
+        let stop = CancellationToken::new();
+        let poll_stop = stop.clone();
+        let polling = tokio::spawn(async move {
+            while !poll_stop.is_cancelled() {
+                let report = poll_manager
+                    .get_task_report(&poll_alias)
+                    .await
+                    .expect("handoff must never return NotFound");
+                assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
+                tokio::task::yield_now().await;
+            }
+        });
+        if outcome == "cancelled" {
+            manager.cancel_task(&alias).await.expect("cancel active task by alias");
+        } else {
+            hook.execute.notify_one();
+        }
+        tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+            .await
+            .expect("scheduler archives terminal");
+        assert!(!manager.active_heals.lock().await.contains_key(&task_id));
+        let expected = task.get_progress().await;
+        for token in [&task_id, &alias] {
+            assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
+            let report = manager
+                .get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
+                .await
+                .expect("terminal token remains queryable at handoff");
+            assert_eq!(report.progress.as_ref(), Some(&expected));
+            assert!(report.result_items.is_empty());
+            match outcome {
+                "success" => assert_eq!(report.status, HealTaskStatus::Completed),
+                "failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
+                _ => assert_eq!(report.status, HealTaskStatus::Cancelled),
+            }
+        }
+        let retained = manager.completed_heals.lock().await;
+        assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
+        drop(retained);
+        stop.cancel();
+        polling.await.expect("concurrent polling succeeds");
+        // Archived progress must not alias a mutable live progress object.
+        task.progress.write().await.objects_scanned = 999;
+        assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
+        hook.finish.notify_one();
+        COMPLETED_RETENTION_HOOKS
+            .lock()
+            .await
+            .retain(|key, _| key != &bucket && key != &task_id);
+    }
+}
+
 #[async_trait::async_trait]
 impl HealStorageAPI for MockStorage {
     async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
     }
 
     async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
+        let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
+        if let Some(hook) = hook {
+            hook.started.notify_one();
+            hook.execute.notified().await;
+            return Ok(true);
+        }
         Ok(bucket == "retry-transition")
     }
 
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
         _version_id: Option<&str>,
         _opts: &HealOpts,
     ) -> Result<(HealResultItem, Option<Error>)> {
+        if bucket == "completed-retention-failed" {
+            return Err(Error::TaskExecutionFailed {
+                message: "retention fixture failure".to_string(),
+            });
+        }
         if let Some(hook) = manager_recovery_test_hook() {
             *hook
                 .heal_object_calls
                 .lock()
                 .expect("manager recovery object call lock should not poison") += 1;
         }
-        if bucket == "retry-transition" {
+        if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
             return Ok((
                 HealResultItem::default(),
                 Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
         .expect("duplicate token should cancel merged active task");
 
     assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
-    assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
+    assert_eq!(
+        manager
+            .get_task_status(&active_task_id)
+            .await
+            .expect("cancelled task remains queryable"),
+        HealTaskStatus::Cancelled
+    );
 }
 
 #[tokio::test]
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
     manager.completed_heals.lock().await.insert(
         task_id,
         Arc::new(CompletedHealStatus {
+            progress: None,
+            retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type,
             status: HealTaskStatus::Retrying {
                 error: "Lock acquisition timeout".to_string(),
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
         "the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
     );
     assert!(
-        matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
+        matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
         "a cancelled task must no longer resolve as an active heal"
     );
 }
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
     manager.completed_heals.lock().await.insert(
         task_id.clone(),
         Arc::new(CompletedHealStatus {
+            progress: None,
+            retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type.clone(),
             status: HealTaskStatus::Retrying {
                 error: "transient disk failure".to_string(),
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
     manager.completed_heals.lock().await.insert(
         "completed-token".to_string(),
         Arc::new(CompletedHealStatus {
+            progress: None,
+            retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Bucket {
                 bucket: "bucket".to_string(),
             },
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
     manager.completed_heals.lock().await.insert(
         "completed-token".to_string(),
         Arc::new(CompletedHealStatus {
+            progress: None,
+            retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Object {
                 bucket: "bucket".to_string(),
                 object: "object".to_string(),

crates/heal/src/heal/mrf_queue.rs

@@ -45,6 +45,10 @@ use uuid::Uuid;
 
 use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
 
+/// Read-only inspection of committed MRF checkpoints. The legacy consumer
+/// remains unchanged until ownership-aware replay is deployed.
+pub mod snapshot;
+
 /// Journal location inside the metadata bucket, following the resume-state
 /// layout.
 pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";

crates/heal/src/heal/mrf_queue/snapshot.rs

@@ -0,0 +1,681 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Reader-first support for owner-local MRF checkpoints.
+//!
+//! Each of two slots has a payload and a commit manifest. The manifest binds
+//! the writer identity, persistent sequence, length and whole-payload digest.
+//! Replacing the inactive slot must leave the previous committed slot intact.
+//! Production publication and reclamation are deliberately not enabled here.
+//! An unreadable commit path cannot prove that only legacy data exists. This
+//! explicit inspection API fails closed and never mutates recovery anchors.
+//! It is not wired into the legacy consumer: that transition requires the
+//! ownership-aware replay and producer handoff before writer activation.
+//! One surviving committed replica supports process restart recovery only;
+//! this reader does not establish a replication quorum or a power-loss policy.
+
+use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal};
+use crate::heal::RUSTFS_META_BUCKET;
+use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore};
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+use tokio::io::AsyncReadExt;
+use uuid::Uuid;
+
+// Root-level control files avoid requiring a new directory before the first
+// atomic commit. They remain inside the storage owner's metadata volume.
+const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
+const MANIFEST_PATHS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
+const MAGIC: &[u8; 8] = b"RFMRFC01";
+const MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
+const VERSION: u8 = 1;
+
+#[derive(Debug, thiserror::Error)]
+pub enum SnapshotError {
+    #[error("MRF checkpoint has an invalid or incomplete commit record")]
+    Corrupt,
+    #[error("MRF checkpoint format is unsupported")]
+    Unsupported,
+    #[error("MRF checkpoint exceeds the configured byte limit")]
+    TooLarge,
+    #[error("MRF checkpoint replicas disagree at the same sequence")]
+    Conflict,
+    #[error("MRF checkpoint storage is unavailable")]
+    Disk(#[source] EcstoreDiskError),
+    #[error("MRF checkpoint body could not be read")]
+    Read(#[source] std::io::Error),
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct Manifest {
+    owner: Uuid,
+    sequence: u64,
+    payload_len: usize,
+    payload_digest: [u8; 32],
+}
+
+impl Manifest {
+    fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
+        if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
+            return Err(SnapshotError::Corrupt);
+        }
+        if bytes[8] != VERSION {
+            return Err(SnapshotError::Unsupported);
+        }
+        let signed = MANIFEST_LEN - 32;
+        let checksum: [u8; 32] = Sha256::digest(&bytes[..signed]).into();
+        if checksum != bytes[signed..] {
+            return Err(SnapshotError::Corrupt);
+        }
+        let owner = Uuid::from_slice(&bytes[9..25]).map_err(|_| SnapshotError::Corrupt)?;
+        let sequence = u64::from_le_bytes(bytes[25..33].try_into().map_err(|_| SnapshotError::Corrupt)?);
+        let payload_len = u64::from_le_bytes(bytes[33..41].try_into().map_err(|_| SnapshotError::Corrupt)?);
+        let payload_len = usize::try_from(payload_len).map_err(|_| SnapshotError::TooLarge)?;
+        if owner.is_nil() || sequence == 0 || sequence == u64::MAX {
+            return Err(SnapshotError::Corrupt);
+        }
+        if payload_len > limit {
+            return Err(SnapshotError::TooLarge);
+        }
+        Ok(Self {
+            owner,
+            sequence,
+            payload_len,
+            payload_digest: bytes[41..73].try_into().map_err(|_| SnapshotError::Corrupt)?,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct CommittedSnapshot {
+    manifest: Manifest,
+    payload: Vec<u8>,
+}
+
+impl CommittedSnapshot {
+    /// Persistent single-writer sequence, not a process UUID ordering.
+    pub fn sequence(&self) -> u64 {
+        self.manifest.sequence
+    }
+
+    /// Identity recorded by the committed checkpoint's writer.
+    pub fn owner(&self) -> Uuid {
+        self.manifest.owner
+    }
+
+    /// Complete, checksum-validated record bytes. Inspection does not consume
+    /// these records or acknowledge completion to any producer.
+    pub fn payload(&self) -> &[u8] {
+        &self.payload
+    }
+
+    fn decode(manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
+        let manifest = Manifest::decode(manifest, limit)?;
+        let checksum: [u8; 32] = Sha256::digest(&payload).into();
+        if payload.len() != manifest.payload_len || checksum != manifest.payload_digest {
+            return Err(SnapshotError::Corrupt);
+        }
+        if decode_journal(&payload).1 != 0 {
+            return Err(SnapshotError::Corrupt);
+        }
+        Ok(Self { manifest, payload })
+    }
+}
+
+#[derive(Debug)]
+pub enum RecoverySnapshot {
+    /// An intact legacy snapshot, without a comparable commit sequence.
+    Legacy(Vec<u8>),
+    /// A committed checkpoint requiring ownership-aware replay before use.
+    Committed(CommittedSnapshot),
+}
+
+async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
+    let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
+        Ok(reader) => reader,
+        Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
+        Err(error) => return Err(SnapshotError::Disk(error)),
+    };
+    let maximum = limit.checked_add(1).ok_or(SnapshotError::TooLarge)?;
+    let maximum = u64::try_from(maximum).map_err(|_| SnapshotError::TooLarge)?;
+    let mut bytes = Vec::new();
+    reader
+        .take(maximum)
+        .read_to_end(&mut bytes)
+        .await
+        .map_err(SnapshotError::Read)?;
+    if bytes.len() > limit {
+        return Err(SnapshotError::TooLarge);
+    }
+    Ok(Some(bytes))
+}
+
+fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: CommittedSnapshot) -> Result<(), SnapshotError> {
+    if let Some(current) = selected {
+        if current.manifest.sequence == candidate.manifest.sequence
+            && (current.manifest != candidate.manifest || current.payload != candidate.payload)
+        {
+            return Err(SnapshotError::Conflict);
+        }
+        if current.manifest.sequence >= candidate.manifest.sequence {
+            return Ok(());
+        }
+    }
+    *selected = Some(candidate);
+    Ok(())
+}
+
+async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
+    let mut selected = None;
+    let mut damaged = None;
+    let mut identities = HashMap::new();
+    for disk in disks {
+        for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
+            let candidate = async {
+                let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
+                    return Ok(None);
+                };
+                let header = Manifest::decode(&manifest, limit)?;
+                let payload = read_bounded(disk, payload_path, header.payload_len)
+                    .await?
+                    .ok_or(SnapshotError::Corrupt)?;
+                CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
+            }
+            .await;
+            match candidate {
+                Ok(Some(candidate)) => {
+                    let identity = (
+                        candidate.manifest.owner,
+                        candidate.manifest.payload_len,
+                        candidate.manifest.payload_digest,
+                    );
+                    if identities
+                        .insert(candidate.manifest.sequence, identity)
+                        .is_some_and(|previous| previous != identity)
+                    {
+                        return Err(SnapshotError::Conflict);
+                    }
+                    select_snapshot(&mut selected, candidate)?;
+                }
+                Ok(None) => {}
+                // A future committed format may supersede all readable slots.
+                Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported),
+                Err(error) => damaged = Some(error),
+            }
+        }
+    }
+    match (selected, damaged) {
+        (Some(snapshot), _) => Ok(Some(snapshot)),
+        (None, Some(error)) => Err(error),
+        (None, None) => Ok(None),
+    }
+}
+
+async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
+    let mut selected = None;
+    let mut incomplete: Option<Vec<u8>> = None;
+    for disk in disks {
+        match read_bounded(disk, path, limit).await {
+            Ok(Some(payload)) if decode_journal(&payload).1 == 0 => {
+                if selected.as_ref().is_some_and(|current| *current != payload) {
+                    // Legacy snapshots have no sequence. There is no evidence
+                    // that the first, longest or nonempty replica is newest.
+                    return Err(SnapshotError::Conflict);
+                }
+                selected = Some(payload);
+            }
+            Ok(Some(payload)) => {
+                if let Some(previous) = &incomplete {
+                    if previous.starts_with(&payload) {
+                        continue;
+                    }
+                    if !payload.starts_with(previous) {
+                        return Err(SnapshotError::Corrupt);
+                    }
+                }
+                incomplete = Some(payload);
+            }
+            Ok(None) => {}
+            Err(error) => return Err(error),
+        }
+    }
+    if let Some(prefix) = incomplete
+        && !selected.as_ref().is_some_and(|payload| payload.starts_with(&prefix))
+    {
+        // In particular, an empty O_TRUNC replica cannot supersede another
+        // replica containing intact records followed by a torn tail.
+        return Err(SnapshotError::Corrupt);
+    }
+    Ok(selected)
+}
+
+/// Inspect local MRF checkpoints without replaying, acknowledging or deleting.
+///
+/// `max_bytes` bounds each payload read. Every local replica is examined and
+/// ambiguous identities, unavailable proof or unsupported formats return a
+/// typed error. This API must not authorize a writer without the separate
+/// ownership and mixed-version activation checks.
+pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
+    read_recovery_snapshot(&super::journal_disks().await, max_bytes).await
+}
+
+async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
+    if let Some(snapshot) = read_committed(disks, limit).await? {
+        return Ok(Some(RecoverySnapshot::Committed(snapshot)));
+    }
+    // RUSTFS_COMPAT_TODO(backlog-2263): inspect retained legacy MRF journals. Remove after all supported upgrade and rollback readers understand committed snapshots and retained journals have migrated.
+    if let Some(payload) = read_legacy(disks, MRF_SCOPED_JOURNAL_PATH, limit).await? {
+        return Ok(Some(RecoverySnapshot::Legacy(payload)));
+    }
+    Ok(read_legacy(disks, MRF_JOURNAL_PATH, limit)
+        .await?
+        .map(RecoverySnapshot::Legacy))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::heal::mrf_queue::encode_intent;
+    use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
+    use crate::heal::{DiskOption, Endpoint, new_disk};
+    use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
+    use std::sync::Arc;
+    use tempfile::TempDir;
+
+    fn payload(object: &str) -> Vec<u8> {
+        let intent = MrfIntent {
+            bucket: Arc::from("bucket"),
+            object: Arc::from(object),
+            version_id: None,
+            kind: MrfKind::PartialWrite,
+            scope: None,
+            lease: None,
+            enqueued_at_ms: 1234,
+            attempts: 0,
+        };
+        let mut bytes = Vec::new();
+        assert!(encode_intent(&intent, &mut bytes), "fixture must encode a full record");
+        bytes
+    }
+
+    fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
+        let mut bytes = Vec::with_capacity(MANIFEST_LEN);
+        bytes.extend_from_slice(MAGIC);
+        bytes.push(VERSION);
+        bytes.extend_from_slice(owner.as_bytes());
+        bytes.extend_from_slice(&sequence.to_le_bytes());
+        bytes.extend_from_slice(&u64::try_from(payload.len()).expect("fixture length fits").to_le_bytes());
+        bytes.extend_from_slice(&Sha256::digest(payload));
+        bytes.extend_from_slice(&Sha256::digest(&bytes));
+        bytes
+    }
+
+    async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore {
+        let path = root.path().join(name);
+        std::fs::create_dir_all(&path).expect("create disk directory");
+        let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("valid disk endpoint");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("open disk");
+        let result = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await;
+        assert!(
+            matches!(result, Ok(()) | Err(EcstoreDiskError::VolumeExists)),
+            "metadata volume: {result:?}"
+        );
+        disk
+    }
+
+    // Exercise the existing storage owner's atomic CAS primitive. No production
+    // caller publishes this format until ownership-aware replay is available.
+    async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8]) {
+        let expected = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await.ok();
+        let result = EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            path,
+            expected,
+            Some(EcstoreDiskBytes::copy_from_slice(bytes)),
+        )
+        .await
+        .expect("atomic snapshot slot write");
+        assert_eq!(result, EcstoreConditionalFileUpdate::Updated);
+    }
+
+    async fn commit(disk: &EcstoreDiskStore, slot: usize, owner: Uuid, sequence: u64, bytes: &[u8]) {
+        install(disk, PAYLOAD_PATHS[slot], bytes).await;
+        install(disk, MANIFEST_PATHS[slot], &manifest(owner, sequence, bytes)).await;
+    }
+
+    #[test]
+    fn manifest_validates_identity_sequence_length_and_digest() {
+        let bytes = payload("object");
+        let owner = Uuid::new_v4();
+        assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
+        for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] {
+            assert!(matches!(
+                Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()),
+                Err(SnapshotError::Corrupt)
+            ));
+        }
+        assert!(matches!(
+            Manifest::decode(&manifest(owner, 1, &bytes), bytes.len() - 1),
+            Err(SnapshotError::TooLarge)
+        ));
+        let mut corrupt = manifest(owner, 1, &bytes);
+        corrupt[25] ^= 1;
+        assert!(matches!(Manifest::decode(&corrupt, bytes.len()), Err(SnapshotError::Corrupt)));
+        let mut unsupported = manifest(owner, 1, &bytes);
+        unsupported[8] = 2;
+        assert!(matches!(Manifest::decode(&unsupported, bytes.len()), Err(SnapshotError::Unsupported)));
+    }
+
+    #[test]
+    fn whole_payload_integrity_is_required_even_with_a_valid_manifest() {
+        let bytes = payload("object");
+        let owner = Uuid::new_v4();
+        let header = manifest(owner, 1, &bytes);
+        assert!(matches!(
+            CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
+            Err(SnapshotError::Corrupt)
+        ));
+        let invalid = b"not an MRF record".to_vec();
+        assert!(matches!(
+            CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()),
+            Err(SnapshotError::Corrupt)
+        ));
+    }
+
+    #[tokio::test]
+    async fn newest_complete_replica_wins_in_both_disk_orders() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 1, &payload("old")).await;
+        commit(&second, 1, owner, 2, &payload("new")).await;
+        for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+            let recovered = read_committed(&disks, 4096)
+                .await
+                .expect("read replicas")
+                .expect("committed snapshot");
+            assert_eq!(recovered.manifest.sequence, 2);
+            assert_eq!(recovered.payload, payload("new"));
+        }
+    }
+
+    #[tokio::test]
+    async fn divergent_commits_at_same_sequence_fail_closed() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 7, &payload("a")).await;
+        commit(&second, 1, owner, 7, &payload("b")).await;
+        assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn newer_slot_does_not_hide_a_conflicting_commit_history() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 8, &payload("newest")).await;
+        commit(&first, 1, owner, 7, &payload("a")).await;
+        commit(&second, 1, owner, 7, &payload("b")).await;
+        assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn uncommitted_or_torn_successor_preserves_previous_slot() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let old = payload("old");
+        let next = payload("next");
+        commit(&disk, 0, owner, 1, &old).await;
+        install(&disk, PAYLOAD_PATHS[1], &next).await;
+        let recovered = read_committed(std::slice::from_ref(&disk), 4096)
+            .await
+            .expect("staged payload is not a commit")
+            .expect("old snapshot");
+        assert_eq!(recovered.payload, old);
+        install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)[..20]).await;
+        let recovered = read_committed(std::slice::from_ref(&disk), 4096)
+            .await
+            .expect("torn manifest preserves old slot")
+            .expect("old snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+        install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)).await;
+        install(&disk, PAYLOAD_PATHS[1], b"torn").await;
+        let recovered = read_committed(&[disk], 4096)
+            .await
+            .expect("torn payload preserves old slot")
+            .expect("old snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+    }
+
+    #[tokio::test]
+    async fn stale_manifest_cas_cannot_replace_committed_anchor() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let bytes = payload("object");
+        commit(&disk, 0, owner, 1, &bytes).await;
+        let result = EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            MANIFEST_PATHS[0],
+            None,
+            Some(manifest(owner, 2, &bytes).into()),
+        )
+        .await
+        .expect("CAS call");
+        assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch);
+        let recovered = read_committed(&[disk], 4096)
+            .await
+            .expect("read old anchor")
+            .expect("snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+    }
+
+    #[tokio::test]
+    async fn legacy_import_requires_complete_consistent_replicas() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let bytes = payload("object");
+        for (disk, data) in [(&first, &bytes[..bytes.len() - 1]), (&second, bytes.as_slice())] {
+            EcstoreDiskAPI::write_all(
+                disk.as_ref(),
+                RUSTFS_META_BUCKET,
+                MRF_SCOPED_JOURNAL_PATH,
+                EcstoreDiskBytes::copy_from_slice(data),
+            )
+            .await
+            .expect("legacy fixture");
+        }
+        let disks = [first.clone(), second];
+        assert!(
+            matches!(read_recovery_snapshot(&disks, 4096).await.expect("intact legacy replica"), Some(RecoverySnapshot::Legacy(data)) if data == bytes)
+        );
+        EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, payload("different").into())
+            .await
+            .expect("divergent fixture");
+        assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn committed_inspection_leaves_payload_and_manifest_unchanged() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let bytes = payload("object");
+        commit(&disk, 0, owner, 3, &bytes).await;
+        assert!(matches!(
+            read_recovery_snapshot(std::slice::from_ref(&disk), 4096)
+                .await
+                .expect("new snapshot"),
+            Some(RecoverySnapshot::Committed(_))
+        ));
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
+                .await
+                .expect("manifest retained")
+                .as_ref(),
+            manifest(owner, 3, &bytes)
+        );
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
+                .await
+                .expect("payload retained")
+                .as_ref(),
+            bytes
+        );
+    }
+
+    #[tokio::test]
+    async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
+        let scoped = |set_index| {
+            let intent = MrfIntent {
+                bucket: Arc::from("bucket"),
+                object: Arc::from("a"),
+                version_id: None,
+                kind: MrfKind::PartialWrite,
+                scope: Some(MrfScope {
+                    pool_index: 0,
+                    set_index,
+                }),
+                lease: None,
+                enqueued_at_ms: 1234,
+                attempts: 0,
+            };
+            let mut bytes = Vec::new();
+            assert!(encode_intent(&intent, &mut bytes), "scoped fixture must encode");
+            bytes
+        };
+        let mut superset = payload("a");
+        superset.extend_from_slice(&payload("b"));
+        for (case, first_bytes, second_bytes) in [
+            ("complete-subset", payload("a"), superset),
+            ("different-set", scoped(1), scoped(2)),
+            ("unknown-scope", payload("a"), scoped(1)),
+        ] {
+            let root = TempDir::new().expect("test directory");
+            let first = disk(&root, "first").await;
+            let second = disk(&root, "second").await;
+            for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
+                assert_eq!(decode_journal(bytes).1, 0, "{case}: complete fixture");
+                EcstoreDiskAPI::write_all(
+                    disk.as_ref(),
+                    RUSTFS_META_BUCKET,
+                    MRF_SCOPED_JOURNAL_PATH,
+                    EcstoreDiskBytes::copy_from_slice(bytes),
+                )
+                .await
+                .expect("write legacy replica");
+            }
+            for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+                assert!(
+                    matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)),
+                    "{case}: neither replica order proves a latest snapshot"
+                );
+            }
+            for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
+                assert_eq!(
+                    EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
+                        .await
+                        .expect("legacy evidence retained")
+                        .as_ref(),
+                    bytes.as_slice(),
+                    "{case}: inspection must preserve both source replicas"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn oversized_or_corrupt_scoped_snapshot_never_falls_back_to_legacy() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, vec![0; 1025].into())
+            .await
+            .expect("oversized fixture");
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload("old").into())
+            .await
+            .expect("legacy fixture");
+        assert!(matches!(
+            read_recovery_snapshot(std::slice::from_ref(&disk), 1024).await,
+            Err(SnapshotError::TooLarge)
+        ));
+        assert!(matches!(read_recovery_snapshot(&[disk], 2048).await, Err(SnapshotError::Corrupt)));
+    }
+
+    #[tokio::test]
+    async fn empty_legacy_replica_cannot_erase_records_in_a_torn_replica() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let mut incomplete = payload("durable-object");
+        incomplete.extend_from_slice(b"torn");
+        EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, Vec::new().into())
+            .await
+            .expect("empty truncated replica");
+        EcstoreDiskAPI::write_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, incomplete.clone().into())
+            .await
+            .expect("records and torn tail");
+        for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+            assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Corrupt)));
+        }
+        assert_eq!(
+            EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
+                .await
+                .expect("recovery anchor preserved")
+                .as_ref(),
+            incomplete
+        );
+    }
+
+    #[tokio::test]
+    async fn unreadable_commit_record_never_implies_legacy_only() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let legacy = payload("old");
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, legacy.clone().into())
+            .await
+            .expect("legacy fixture");
+        // Opening a directory as a record either fails at open or at read,
+        // depending on the platform. Neither outcome proves absence.
+        std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[0]))
+            .expect("unreadable manifest fixture");
+        let recovered = read_recovery_snapshot(std::slice::from_ref(&disk), 4096).await;
+        assert!(
+            matches!(recovered, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))),
+            "must preserve unavailable proof: {recovered:?}"
+        );
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH)
+                .await
+                .expect("legacy remains")
+                .as_ref(),
+            legacy
+        );
+    }
+}

crates/heal/src/heal/task.rs

@@ -999,7 +999,7 @@ impl HealTask {
         let items = match since {
             None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
             Some(cursor) => {
-                if cursor + 1 < min_seq {
+                if cursor.saturating_add(1) < min_seq {
                     lagged = true;
                 }
                 result_items