Merge branch 'main' into houseme/test/scanner-heal-v2-w20

* test(odm): keep listing header import test scoped

* fix(build): gate GCS-only migration HTTP helpers

.config/make/pre-commit.mak

@@ -3,9 +3,10 @@
 .NOTPARALLEL: pre-commit pre-pr dev-check
 
 .PHONY: setup-hooks
-setup-hooks: ## Set up git hooks
+setup-hooks: ## Install the configured pre-commit hooks
 	@echo "🔧 Setting up git hooks..."
-	chmod +x .git/hooks/pre-commit
+	pre-commit validate-config
+	pre-commit install
 	@echo "✅ Git hooks setup complete!"
 
 .PHONY: doc-paths-check

.config/make/tests.mak

@@ -31,6 +31,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
+	./scripts/test_scanner_validation_harness.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
@@ -41,6 +42,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test

.github/actions/quick-checks/action.yml

@@ -0,0 +1,120 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Quick Checks
+description: Run the shared compile-free RustFS quality checks.
+
+runs:
+  using: composite
+  steps:
+    - name: Install quality tools
+      uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+      with:
+        tool: |
+          ripgrep@15.2.0
+          shellcheck@0.11.0
+
+    - name: Install actionlint
+      shell: bash
+      run: |
+        actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
+        curl --fail --location --silent --show-error \
+          --output "$actionlint_dir/actionlint.tar.gz" \
+          https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
+        echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8  $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
+        tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
+        rm "$actionlint_dir/actionlint.tar.gz"
+        echo "$actionlint_dir" >> "$GITHUB_PATH"
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+      with:
+        components: rustfmt
+
+    - name: Check workflow syntax and shell scripts
+      shell: bash
+      run: shellcheck --version && actionlint
+
+    - name: Check code formatting
+      shell: bash
+      run: cargo fmt --all --check
+
+    - name: Check unsafe code allowances
+      shell: bash
+      run: ./scripts/check_unsafe_code_allowances.sh
+
+    - name: Check layered dependencies
+      shell: bash
+      run: ./scripts/check_layer_dependencies.sh
+
+    - name: Check architecture migration rules
+      shell: bash
+      run: ./scripts/check_architecture_migration_rules.sh
+
+    - name: Check logging guardrails
+      shell: bash
+      run: ./scripts/check_logging_guardrails.sh
+
+    - name: Check error other(format!) ratchet
+      shell: bash
+      run: ./scripts/check_error_other_format_ratchet.sh
+
+    - name: Check tokio io-uring feature guard
+      shell: bash
+      run: ./scripts/check_no_tokio_io_uring.sh
+
+    - name: Check extension schema boundaries
+      shell: bash
+      run: ./scripts/check_extension_schema_boundaries.sh
+
+    - name: Check body-cache whitelist guard
+      shell: bash
+      run: ./scripts/check_body_cache_whitelist.sh
+
+    - name: Check s3s footprint ratchet
+      shell: bash
+      run: ./scripts/check_s3s_footprint.sh
+
+    - name: Check cryptographic capability wording
+      shell: bash
+      run: ./scripts/check_fips_wording.sh
+
+    - name: Check no embedded secret material
+      shell: bash
+      run: ./scripts/check_embedded_secrets.sh
+
+    - name: Run script contract tests
+      shell: bash
+      run: make script-tests
+
+    - name: Check test wiring
+      shell: bash
+      run: |
+        python3 ./scripts/check_test_wiring.py --self-test
+        python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+        python3 ./scripts/test_security_workflow.py
+        python3 ./scripts/test_nightly_candidate.py
+        python3 ./scripts/check_test_wiring.py
+
+    - name: Check no planning docs committed
+      shell: bash
+      run: ./scripts/check_no_planning_docs.sh
+
+    - name: Check CI paths stay in sync
+      shell: bash
+      run: ./scripts/check_ci_paths_sync.sh
+
+    - name: Check io_uring lane --lib precondition
+      shell: bash
+      run: ./scripts/check_uring_lane_lib_only.sh

.github/workflows/ci-docs-only.yml

@@ -12,24 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Companion to ci.yml for required status checks.
-#
-# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
-# requires a check named "Test and Lint" — without this workflow a docs-only PR
-# would wait on it forever. This workflow triggers on exactly the paths ci.yml
-# ignores and reports success under the same job name. Mixed PRs trigger both
-# workflows and the real check still gates: a required check with any failing
-# run blocks the merge.
-# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
-#
-# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
-# required too (rustfs/backlog#1599). Until that change lands this job is
-# inert; mirroring it first is what lets the ruleset change happen without
-# stranding docs-only PRs on a check nobody reports.
-#
-# Keep the paths list below in sync with the pull_request paths-ignore list
-# in ci.yml, and keep the quick-checks steps below byte-identical to the
-# quick-checks job in ci.yml.
+# Reports the existing required checks for paths excluded by ci.yml.
+# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
+# action to keep validation coverage aligned. Keep this paths list in sync with
+# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
 
 name: Continuous Integration (docs only)
 
@@ -59,19 +45,6 @@ permissions:
   contents: read
 
 jobs:
-  # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
-  # check, ci.yml gates every expensive job behind it, so a mixed PR reports
-  # two check runs with this name: the real one (45-51s) and this companion.
-  # GitHub has no written contract for how it picks between same-named
-  # required check runs ("latest wins" vs "any failure blocks"), so instead of
-  # relying on ordering we make both runs execute the same commands against
-  # the same merge ref — their conclusions are then necessarily identical and
-  # the choice does not matter. Keep these steps byte-identical to the
-  # quick-checks job in ci.yml (a guard script that asserts this, and the paths
-  # sync below, is tracked in rustfs/backlog#1603).
-  #
-  # For a genuinely docs-only PR this adds no strictness (no code changed, so
-  # fmt and the guards always pass) and costs ~50s of ubuntu-latest.
   quick-checks:
     name: Quick Checks
     runs-on: ubuntu-latest
@@ -82,64 +55,8 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-        with:
-          tool: ripgrep@15.2.0
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          components: rustfmt
-
-      - name: Check code formatting
-        run: cargo fmt --all --check
-
-      - name: Check unsafe code allowances
-        run: ./scripts/check_unsafe_code_allowances.sh
-
-      - name: Check layered dependencies
-        run: ./scripts/check_layer_dependencies.sh
-
-      - name: Check architecture migration rules
-        run: ./scripts/check_architecture_migration_rules.sh
-
-      - name: Check logging guardrails
-        run: ./scripts/check_logging_guardrails.sh
-
-      - name: Check tokio io-uring feature guard
-        run: ./scripts/check_no_tokio_io_uring.sh
-
-      - name: Check extension schema boundaries
-        run: ./scripts/check_extension_schema_boundaries.sh
-
-      - name: Check body-cache whitelist guard
-        run: ./scripts/check_body_cache_whitelist.sh
-
-      - name: Check s3s footprint ratchet
-        run: ./scripts/check_s3s_footprint.sh
-
-      - name: Check cryptographic capability wording
-        run: ./scripts/check_fips_wording.sh
-
-      - name: Check no embedded secret material
-        run: ./scripts/check_embedded_secrets.sh
-
-      - name: Check test wiring
-        run: |
-          python3 ./scripts/check_test_wiring.py --self-test
-          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-          python3 ./scripts/test_security_workflow.py
-          python3 ./scripts/check_test_wiring.py
-
-      - name: Check no planning docs committed
-        run: ./scripts/check_no_planning_docs.sh
-
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
+      - name: Run shared quick checks
+        uses: ./.github/actions/quick-checks
 
   test-and-lint:
     name: Test and Lint

.github/workflows/ci.yml

@@ -100,12 +100,7 @@ jobs:
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
-  # Fast, compile-free checks that fail early so contributors get feedback in
-  # ~1 minute instead of waiting for the full test job.
-  #
-  # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
-  # PR, which reports two check runs named "Quick Checks", cannot get one red
-  # and one green. Edit both jobs together.
+  # Fail early with compile-free checks shared with docs-only CI.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -117,67 +112,8 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-        with:
-          tool: ripgrep@15.2.0
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          components: rustfmt
-
-      - name: Check code formatting
-        run: cargo fmt --all --check
-
-      - name: Check unsafe code allowances
-        run: ./scripts/check_unsafe_code_allowances.sh
-
-      - name: Check layered dependencies
-        run: ./scripts/check_layer_dependencies.sh
-
-      - name: Check architecture migration rules
-        run: ./scripts/check_architecture_migration_rules.sh
-
-      - name: Check logging guardrails
-        run: ./scripts/check_logging_guardrails.sh
-
-      - name: Check error other(format!) ratchet
-        run: ./scripts/check_error_other_format_ratchet.sh
-
-      - name: Check tokio io-uring feature guard
-        run: ./scripts/check_no_tokio_io_uring.sh
-
-      - name: Check extension schema boundaries
-        run: ./scripts/check_extension_schema_boundaries.sh
-
-      - name: Check body-cache whitelist guard
-        run: ./scripts/check_body_cache_whitelist.sh
-
-      - name: Check s3s footprint ratchet
-        run: ./scripts/check_s3s_footprint.sh
-
-      - name: Check cryptographic capability wording
-        run: ./scripts/check_fips_wording.sh
-
-      - name: Check no embedded secret material
-        run: ./scripts/check_embedded_secrets.sh
-
-      - name: Check test wiring
-        run: |
-          python3 ./scripts/check_test_wiring.py --self-test
-          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-          python3 ./scripts/test_security_workflow.py
-          python3 ./scripts/check_test_wiring.py
-
-      - name: Check no planning docs committed
-        run: ./scripts/check_no_planning_docs.sh
-
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
+      - name: Run shared quick checks
+        uses: ./.github/actions/quick-checks
 
   test-and-lint:
     name: Test and Lint

.github/workflows/e2e-upgrade.yml

@@ -19,7 +19,9 @@ on:
     paths:
       - ".github/workflows/e2e-upgrade.yml"
       - "crates/e2e_test/src/common.rs"
+      - "crates/e2e_test/src/fake_s3_target/**"
       - "crates/e2e_test/src/lib.rs"
+      - "crates/e2e_test/src/replication_extension_test.rs"
       - "crates/e2e_test/src/upgrade_compatibility_test.rs"
       - "crates/ecstore/**"
       - "crates/filemeta/**"
@@ -44,9 +46,9 @@ concurrency:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
-  UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
-  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
-  UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+  UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
+  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
+  UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
 
 jobs:
   upgrade:
@@ -55,14 +57,31 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Direct upgrade from rc.2
+          # The two `_from_rc2_` tests keep their names: they assert
+          # release-independent object contracts and pass unchanged against the
+          # newer pinned source, so renaming them would only churn history and
+          # the CI required-check names. UPGRADE_SOURCE_VERSION above is the
+          # single source of truth for which release they actually run against.
+          - name: Direct upgrade from the previous release
             cache_key: e2e-direct-upgrade
             test: direct_upgrade_from_rc2_preserves_object_contracts
             artifact: direct-upgrade
-          - name: Mixed-version rolling upgrade from rc.2
+          - name: Mixed-version rolling upgrade from the previous release
             cache_key: e2e-mixed-version-upgrade
             test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
             artifact: mixed-version-upgrade
+          - name: Bucket configuration survives the upgrade
+            cache_key: e2e-bucket-config-upgrade
+            test: direct_upgrade_from_previous_release_preserves_bucket_configuration
+            artifact: bucket-config-upgrade
+          - name: Rollback reads current bucket metadata
+            cache_key: e2e-bucket-config-rollback
+            test: rollback_to_previous_release_reads_current_bucket_metadata
+            artifact: bucket-config-rollback
+          - name: ODM configuration recovery after rc.5 rollback
+            cache_key: e2e-odm-config-rollback
+            test: rc5_rollback_requires_restoring_odm_configuration
+            artifact: odm-config-rollback
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:

.github/workflows/nightly-gnu.yml

@@ -166,8 +166,9 @@ jobs:
       # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
       # Skipped when the R2 secrets are not configured (artifact-only mode).
       - name: Upload DEB to Cloudflare R2
-        if: env.R2_ACCESS_KEY_ID != ''
+        id: publish
         env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
             exit 0
           fi
 
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y -qq awscli
-          fi
-
           export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          SOURCE_SHA="$(git rev-parse HEAD)"
+          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+            echo "Checkout SHA does not match the nightly build run" >&2
+            exit 1
+          fi
+          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
+          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
+          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
+
+          # Old AWS CLI models lack conditional PutObject support. Never fall
+          # back to an overwriting upload for a candidate.
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
+            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          "${AWS_CLI}" --version
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
+            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
+          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
+            echo "Published candidate checksum does not match the built package" >&2
+            exit 1
+          fi
+
           R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
 
           echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           # Stable "latest" alias so tests can fetch the newest nightly
           # without knowing today's date.
           echo "📤 Uploading latest alias"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
             --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           echo "✅ R2 upload complete"
 
+          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg source_sha "${SOURCE_SHA}" \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
+            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            > "${CANDIDATE_FILE}"
+          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload nightly candidate manifest
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error
+
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-functional-chain.yml

@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -54,14 +54,26 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 480
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -117,7 +129,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -127,7 +139,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -137,13 +149,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -152,8 +161,9 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
           import re
           import sys
 
@@ -165,6 +175,7 @@ jobs:
 
           steps = {}
           order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
           version = None
           version_node = None
           verdict = None
@@ -178,14 +189,15 @@ jobs:
                           n, desc, status = m.group(1), m.group(2), m.group(3)
                           if n not in steps:
                               order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
                           continue
                       m = ver_re.match(line)
                       if m:
                           version, version_node = m.group(1), m.group(2)
                           continue
                       m = result_re.match(line)
-                      if m:
+                      if m and verdict != 'FAIL':
                           verdict, verdict_detail = m.group(1), m.group(2)
           except FileNotFoundError:
               pass
@@ -205,30 +217,43 @@ jobs:
                   out.write(f'| {n} | {desc} | {status} |\n')
               if not order:
                   out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
           PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${STEPS_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -238,28 +263,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -287,14 +316,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -310,14 +341,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-heal-test*.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}

.github/workflows/rustfs-kms-test.yml

@@ -49,10 +49,28 @@ env:
 jobs:
   kms-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -109,9 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -141,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -156,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -238,28 +214,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -287,14 +267,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -310,14 +292,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-kms.log
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-performance-test.yml

@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -76,22 +75,33 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
   performance-test:
     runs-on: pf-testing
-    # Requirement: a failing benchmark must not fail the workflow;
-    # failures are filed to rustfs/backlog.
-    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -123,7 +133,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -133,7 +143,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -143,7 +153,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -156,17 +166,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -186,7 +194,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -194,7 +201,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -202,6 +209,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -211,8 +220,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          } > "${REPORT_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -231,11 +240,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -263,14 +271,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -286,20 +296,26 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-perf-test*.log
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+          if-no-files-found: error
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-pool-expand-test.yml

@@ -76,9 +76,6 @@ jobs:
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
@@ -542,17 +539,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}

.github/workflows/rustfs-replication-test.yml

@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -62,12 +61,28 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -116,9 +131,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -141,10 +153,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -166,80 +175,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -249,28 +222,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -298,14 +275,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -321,14 +300,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
@@ -349,13 +329,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-s3-compat-test.yml

@@ -37,10 +37,28 @@ env:
 jobs:
   s3-compat-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -88,9 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -107,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -132,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -218,28 +191,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -267,14 +244,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -290,14 +269,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-s3-compat.log
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-security-test.yml

@@ -77,10 +77,14 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      # Checkout the repository into its own subdirectory. Checking out at
+      # the workspace root would wipe the auto-testing clone above (that is
+      # exactly how run 33934141181 lost rustfs-security-test.sh).
       - name: Checkout repository (for the OIDC live gate script)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          path: rustfs-repo
 
       - name: Initialize security evidence
         id: evidence
@@ -88,7 +92,7 @@ jobs:
           set -euo pipefail
           umask 077
           SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${SECURITY_ARTIFACTS_DIR}"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
       # auto-testing is private: clone it with the dedicated PF token (not
@@ -144,8 +148,8 @@ jobs:
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
-          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-security-test.sh
@@ -168,7 +172,7 @@ jobs:
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
-          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
 
       - name: Generate report
         id: report
@@ -219,17 +223,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -296,7 +305,10 @@ jobs:
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
+          path: |
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
           if-no-files-found: error
           retention-days: 3
 

.github/workflows/rustfs-storage-test.yml

@@ -46,10 +46,28 @@ env:
 jobs:
   storage-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -97,9 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -122,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -147,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -233,28 +206,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -282,14 +259,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -305,14 +284,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-tier-test.yml

@@ -61,9 +61,6 @@ env:
 jobs:
   tier-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
@@ -380,17 +377,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: Verify required tier evidence
         id: evidence_verify

.github/workflows/rustfs-upgrade-test.yml

@@ -79,10 +79,28 @@ env:
 jobs:
   upgrade-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,9 +160,7 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
-        continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
@@ -202,10 +218,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -226,103 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
-          rows = []
-          index = {}
-          topo_rows = []
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${MATRIX_TABLE}"
+              echo ""
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -332,28 +289,32 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -381,14 +342,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -404,14 +367,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)

.pre-commit-config.yaml

@@ -3,9 +3,9 @@
 repos:
   - repo: local
     hooks:
-      - id: rustfs-dev-check
-        name: rustfs dev-check
-        entry: make dev-check
+      - id: rustfs-fmt-check
+        name: Rust formatting
+        entry: cargo fmt --all --check
         language: system
         types: [rust]
         pass_filenames: false

CHANGELOG.md

@@ -18,7 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Read paths: an object at or below `policy.inline_max_bytes` (16 MiB by default) is teed to the client and to the local store in a single source read; a larger object or a Range read streams through and a background pull stores the whole object. A HEAD miss is proxied to the source and stores nothing (`policy.head = local_only` disables it). Every source-backed response carries `x-rustfs-on-demand-migration: source`
   - Protections: a per-source circuit breaker, a per-key negative cache, singleflight per key, a concurrency limit and a bounded pull queue shared by the inline and background paths, an optional bandwidth limit, an anti-loop request marker, and the shared outbound-endpoint (SSRF) policy
   - Metrics under `rustfs_on_demand_migration_*` (`requests_total`, `pulled_bytes_total`, `pulled_objects_total`, `pull_failures_total`, `inflight_pulls`, `queue_depth`, `source_latency_seconds_*`, `breaker_state`), mirrored per node by the admin status route
-  - Limitations: listings show only local objects (the source is not merged into `ListObjectsV2`); PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
+  - Listings: `ListObjects` v1 remains local with ordinary key markers. `ListObjectsV2` can merge source objects when `policy.list_through = true`; this is off by default
+  - Upgrade and rollback: finish upgrading every node before enabling ODM. An rc.5 node that writes bucket configuration drops the ODM fields from metadata; neither a later restart nor moving the service out of ECStore recovers them. Before rollback, disable ODM and securely retain the original full configuration and credentials. After every node returns to a compatible version, restore and validate that configuration. Redacted exports cannot replace the credential backup; source-only objects are unavailable through RustFS while ODM is disabled. See the upgrade and rollback section of `docs/operations/on-demand-migration.md`
+  - Optional Google dependencies: default and `full` server builds retain native GCS support. `cargo build -p rustfs --no-default-features --features ftps,webdav` excludes Google SDKs while preserving configuration decoding and redaction; native GCS ODM and tier operations require the `gcs` feature. Do not use that build with existing GCS-tiered data
+  - Limitations: PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
 - **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
   - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
   - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window

CONTRIBUTING.md

@@ -109,24 +109,17 @@ affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
-Git hooks are **not** versioned in this repository, so a fresh clone has no
-active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
-choice is a one-liner that runs `make pre-commit`), you can mark it executable
-with:
+The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
 
 ```bash
 make setup-hooks
 ```
 
-Or manually:
+The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
 
-```bash
-chmod +x .git/hooks/pre-commit
-```
+`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
 
-With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
-applicable scoped checks, and reserve `make pre-pr` for broad cross-module
-changes whose impact cannot be bounded by those checks.
+A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
 
 ### 📝 Formatting Configuration
 
@@ -138,31 +131,11 @@ fn_call_width = 90
 single_line_let_else_max_width = 100
 ```
 
-### 🚫 Commit Prevention
-
-If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
-
-1. **Block the commit** and show clear error messages
-2. **Provide exact commands** to fix the issues
-3. **Guide you through** the resolution process
-
-Example output when formatting fails:
-
-```
-❌ Code formatting check failed!
-💡 Please run 'cargo fmt --all' to format your code before committing.
-
-🔧 Quick fix:
-   cargo fmt --all
-   git add .
-   git commit
-```
-
 ### 🔄 Development Workflow
 
 1. **Make your changes**
 2. **Format your code**: `make fmt` or `cargo fmt --all`
-3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
+3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
 6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -206,11 +179,12 @@ Configure your IDE to:
 #### Pre-commit hook not running?
 
 ```bash
-# Check if hook is executable
-ls -la .git/hooks/pre-commit
-
-# Make it executable if needed
-chmod +x .git/hooks/pre-commit
+pre-commit validate-config
+pre-commit run --all-files
+# Inspect any configured hook manager; do not overwrite it.
+git config --get core.hooksPath
+# Install if no separate hook manager is configured.
+make setup-hooks
 ```
 
 #### Formatting issues?

Cargo.lock

@@ -315,7 +315,7 @@ dependencies = [
  "strum",
  "thiserror 2.0.20",
  "uuid",
- "zstd",
+ "zstd 0.13.3",
 ]
 
 [[package]]
@@ -508,7 +508,7 @@ dependencies = [
  "arrow-select",
  "flatbuffers",
  "lz4_flex",
- "zstd",
+ "zstd 0.13.3",
 ]
 
 [[package]]
@@ -1679,7 +1679,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -1698,7 +1698,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -2110,7 +2110,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "inout 0.1.4",
 ]
 
@@ -2249,8 +2249,8 @@ dependencies = [
  "liblzma",
  "lz4",
  "memchr",
- "zstd",
- "zstd-safe",
+ "zstd 0.13.3",
+ "zstd-safe 7.3.0",
 ]
 
 [[package]]
@@ -2580,7 +2580,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rand_core 0.6.4",
  "subtle",
  "zeroize",
@@ -2605,11 +2605,11 @@ dependencies = [
 
 [[package]]
 name = "crypto-common"
-version = "0.1.7"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "typenum",
 ]
 
@@ -3901,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer 0.10.4",
  "const-oid 0.9.6",
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "subtle",
 ]
 
@@ -4067,7 +4067,7 @@ dependencies = [
  "uuid",
  "walkdir",
  "zip",
- "zstd",
+ "zstd 0.14.0",
 ]
 
 [[package]]
@@ -4166,7 +4166,7 @@ dependencies = [
  "crypto-bigint 0.5.5",
  "digest 0.10.7",
  "ff 0.13.1",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "group 0.13.0",
  "hkdf 0.12.4",
  "pem-rfc7468 0.7.0",
@@ -4499,7 +4499,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
 dependencies = [
  "io-lifetimes 2.0.4",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4622,9 +4622,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.7"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
 dependencies = [
  "typenum",
  "version_check",
@@ -4637,7 +4637,7 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rustversion",
  "typenum",
 ]
@@ -5319,9 +5319,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
+checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5660,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
  "block-padding 0.3.3",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
 dependencies = [
  "io-lifetimes 3.0.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -5971,7 +5971,7 @@ dependencies = [
  "lz4",
  "snap",
  "uuid",
- "zstd",
+ "zstd 0.13.3",
 ]
 
 [[package]]
@@ -7115,7 +7115,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.1",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -7658,7 +7658,7 @@ dependencies = [
  "snap",
  "tokio",
  "twox-hash",
- "zstd",
+ "zstd 0.13.3",
 ]
 
 [[package]]
@@ -9493,6 +9493,8 @@ dependencies = [
  "atomic_enum",
  "aws-config",
  "aws-sdk-s3",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
  "axum",
  "base64-simd",
  "bytes",
@@ -9501,11 +9503,13 @@ dependencies = [
  "clap",
  "const-str",
  "datafusion",
+ "faster-hex",
  "flatbuffers",
  "flate2",
  "futures",
  "futures-lite",
  "futures-util",
+ "google-cloud-auth",
  "hashbrown 0.17.1",
  "hex-simd",
  "hmac 0.13.0",
@@ -9524,6 +9528,7 @@ dependencies = [
  "metrics",
  "metrics-util",
  "mime_guess",
+ "moka",
  "opentelemetry",
  "opentelemetry_sdk",
  "p256 0.14.0",
@@ -9616,9 +9621,10 @@ dependencies = [
  "urlencoding",
  "uuid",
  "x509-parser",
+ "xxhash-rust",
  "zeroize",
  "zip",
- "zstd",
+ "zstd 0.14.0",
 ]
 
 [[package]]
@@ -10228,7 +10234,7 @@ dependencies = [
  "thiserror 2.0.20",
  "walkdir",
  "zip",
- "zstd",
+ "zstd 0.14.0",
 ]
 
 [[package]]
@@ -10395,7 +10401,7 @@ dependencies = [
  "tracing-opentelemetry",
  "tracing-subscriber",
  "url",
- "zstd",
+ "zstd 0.14.0",
 ]
 
 [[package]]
@@ -10985,7 +10991,7 @@ dependencies = [
  "transform-stream",
  "url",
  "windows",
- "zstd",
+ "zstd 0.14.0",
 ]
 
 [[package]]
@@ -11399,7 +11405,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
  "base16ct 0.2.0",
  "der 0.7.10",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "pkcs8 0.10.2",
  "subtle",
  "zeroize",
@@ -12399,7 +12405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -13651,6 +13657,15 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
@@ -13823,7 +13838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
 dependencies = [
  "bitflags 2.13.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -14095,7 +14110,7 @@ dependencies = [
  "typed-path",
  "zeroize",
  "zopfli",
- "zstd",
+ "zstd 0.13.3",
 ]
 
 [[package]]
@@ -14128,7 +14143,16 @@ version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
 dependencies = [
- "zstd-safe",
+ "zstd-safe 7.3.0",
+]
+
+[[package]]
+name = "zstd"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e"
+dependencies = [
+ "zstd-safe 8.0.0",
 ]
 
 [[package]]
@@ -14140,6 +14164,15 @@ dependencies = [
  "zstd-sys",
 ]
 
+[[package]]
+name = "zstd-safe"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab"
+dependencies = [
+ "zstd-sys",
+]
+
 [[package]]
 name = "zstd-sys"
 version = "2.1.0+zstd.1.5.7"

Cargo.toml

@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.1" }
-argon2 = { version = "=0.6.0" }
-blake2 = "=0.11.0"
-chacha20poly1305 = { version = "=0.11.0" }
+aes-gcm = { version = "0.11.1" }
+argon2 = { version = "0.6.0" }
+blake2 = "0.11.0"
+chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.0.0" }
@@ -343,7 +343,7 @@ windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.18" }
 zip = "8.6.0"
-zstd = "0.13.3"
+zstd = "0.14.0"
 
 # Observability and Metrics
 metrics = "0.24.6"
@@ -371,7 +371,8 @@ dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
-hotpath = { version = "0.25.0", default-features = false }
+# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
+hotpath = { version = "=0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

crates/config/README.md

@@ -130,6 +130,21 @@ Scanner cycle budget controls:
   - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
   - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
 
+## Remote tier timeout environment variables
+
+- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
+  - remote tier TCP connect timeout.
+  - default is `10`.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
+- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
+  - remote tier request timeout through response headers.
+  - default is `86400` so large transition uploads keep a production-safe budget.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
+- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
+  - maximum idle time between remote tier response-body chunks.
+  - default is `60`; the timer resets only when non-empty body data keeps progressing.
+  - must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
+
 ## Drive timeout environment variables
 
 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`

crates/config/src/constants/object.rs

@@ -137,6 +137,28 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
 const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
 
+/// Environment variable for remote tier TCP connect timeout in seconds.
+pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
+/// Default remote tier TCP connect timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+/// Environment variable for the remote tier request timeout in seconds.
+///
+/// This bounds upload/download request progress through response headers. The
+/// default is intentionally large so multi-TiB transition uploads keep their
+/// previous production budget while black-hole remotes no longer wait forever.
+pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
+/// Default remote tier request timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
+
+/// Environment variable for remote tier response-body idle timeout in seconds.
+///
+/// The timer is re-armed on every non-empty response-body chunk, so slow but
+/// progressing remotes can continue while silent response bodies are cancelled.
+pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
+/// Default remote tier response-body idle timeout in seconds.
+pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
+
 /// Request the object-transaction fencing contract used by storage-owned
 /// cleanup receipts and lock-window optimizations.
 ///
@@ -812,6 +834,16 @@ mod remote_version_state_tests {
         );
     }
 
+    #[test]
+    fn remote_tier_timeout_env_names_are_stable() {
+        assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
+        assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
+        assert_eq!(
+            super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+            "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
+        );
+    }
+
     #[test]
     fn data_movement_part_checksum_gate_uses_stable_environment_names() {
         assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,21 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use crate::common::{
+    RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
+};
+use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
+use crate::replication_extension_test::{
+    LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
+    BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
+    ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
+    ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
+    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
 };
+use http::{Method, StatusCode};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use tokio::task::JoinSet;
 use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -40,6 +53,32 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
 // comfortably covers that window plus CI scheduling jitter.
 const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
+// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
+const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
+const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
+const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
+const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
+const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
+const ROLLBACK_BUCKET: &str = "rollback-config-data";
+const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
+const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
+const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
+const LIFECYCLE_PREFIX: &str = "logs/";
+const LIFECYCLE_DAYS: i32 = 30;
+const BUCKET_TAG_KEY: &str = "owner";
+const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
+const OBJECT_LOCK_DAYS: i32 = 1;
+// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
+// authoritative; the quota test uses the same 30s budget.
+const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
+// Quota admission fails closed while a freshly started server has neither
+// authoritative usage nor a persisted degraded baseline for the bucket
+// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
+// window. It is a restart property, not an upgrade property — the same window
+// opens on the very first start — so the write assertions ride it out instead
+// of treating it as an upgrade failure.
+const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
+
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
         .map(PathBuf::from)
@@ -240,6 +279,93 @@ async fn exercise_mixed_cluster(
     Ok(())
 }
 
+/// Pins the published old writer's limitation and the supported recovery
+/// procedure. This is not a promise that mixed-version ODM is supported.
+/// Replace the loss assertion when ODM gains independent persistence;
+/// preserving configuration across rc.5 writes is then an improvement.
+#[tokio::test]
+#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
+async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let version = tokio::process::Command::new(&previous_binary)
+        .arg("--version")
+        .output()
+        .await?;
+    assert!(version.status.success(), "previous binary must report its version");
+    assert!(
+        String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
+        "this compatibility scenario requires the published rc.5 writer"
+    );
+    let mut env = OdmTestEnv::start().await?;
+    let bucket = "odm-rc5-rollback";
+    let source_bucket = "odm-rc5-source";
+    env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
+    env.seed_source(
+        source_bucket,
+        &[SeedObject::new(
+            "source-only",
+            bytes::Bytes::from_static(b"source read after recovery"),
+        )],
+    );
+    env.rustfs.create_test_bucket(bucket).await?;
+    let saved_config = env.fake_source_spec(source_bucket);
+    assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
+    let before = env.get_config(bucket).await?;
+    assert_eq!(before.status, 200);
+    let expected_config = before
+        .json()?
+        .get("config")
+        .cloned()
+        .ok_or("configuration response omitted config")?;
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key("local")
+        .body(ByteStream::from_static(b"local data survives rollback"))
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restarted = env.get_config(bucket).await?;
+    assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
+    assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
+
+    restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(
+            Tagging::builder()
+                .tag_set(Tag::builder().key("writer").value("rc5").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let missing = env.get_config(bucket).await?;
+    assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
+    assert!(missing.body.contains("NoSuchConfiguration"));
+    assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
+
+    assert_eq!(
+        env.configure_source(bucket, &saved_config).await?.status,
+        200,
+        "restore from saved full configuration"
+    );
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restored = env.get_config(bucket).await?;
+    assert_eq!(restored.status, 200, "restored ODM configuration persists");
+    assert_eq!(restored.json()?.get("config"), Some(&expected_config));
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        read_object(&env.client, bucket, "source-only", None).await?.1,
+        b"source read after recovery"
+    );
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -429,3 +555,653 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
 
     Ok(())
 }
+
+/// Child-process environment shared by both bucket-configuration scenarios.
+///
+/// The replication target is an in-process fake bound to `127.0.0.1`, which
+/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
+/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
+/// server's outbound health check.
+fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
+    let mut env = vec![
+        (SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
+        ("NO_PROXY", "127.0.0.1,localhost"),
+        ("HTTP_PROXY", ""),
+        ("HTTPS_PROXY", ""),
+        // Shorten the scanner cycle so the bucket's usage becomes authoritative
+        // in seconds; both `set-bucket-quota` and quota admission block on it.
+        ("RUSTFS_SCANNER_CYCLE", "1"),
+        ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
+    ];
+    env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env.extend(replication_fast_env());
+    env
+}
+
+/// Restart `env` in place on the same data directory using an explicit binary.
+///
+/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
+/// the workspace build, which is the upgrade direction only. The rollback
+/// scenario needs the reverse: stop the current build and bring the pinned
+/// previous release up on the metadata that build just wrote.
+async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
+    env.stop_server();
+    env.start_rustfs_server_from_binary(binary, vec![], server_env).await
+}
+
+async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
+    let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
+    loop {
+        let (status, response) =
+            admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
+        if status.is_success() {
+            return Ok(());
+        }
+        if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
+            return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
+/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
+///
+/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
+/// never ends, is a genuine regression and surfaces as an error.
+async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
+    let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
+    loop {
+        let result = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(body))
+            .send()
+            .await;
+        let error = match result {
+            Ok(_) => return Ok(()),
+            Err(error) => error,
+        };
+        let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
+        if !retryable || Instant::now() >= deadline {
+            return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
+    }
+    let quota: serde_json::Value = serde_json::from_str(&response)?;
+    Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
+}
+
+/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
+///
+/// Returns an error for any non-200, because rustfs#7172 made this endpoint
+/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
+/// An upgrade that misreads a blob written by the previous release therefore
+/// shows up here as an error, and a silently dropped target shows up as an
+/// empty list — the caller must distinguish the two.
+async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
+    let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
+    }
+    Ok(serde_json::from_str(&response)?)
+}
+
+/// Assert that `bucket` still carries exactly the replication target `arn`.
+async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
+    let targets = list_remote_targets(env, bucket).await?;
+    assert_eq!(
+        targets.len(),
+        1,
+        "{context}: list-remote-targets must still report the single configured target, got {targets:?}"
+    );
+    assert_eq!(
+        targets[0].get("arn").and_then(serde_json::Value::as_str),
+        Some(arn),
+        "{context}: the target ARN changed across the restart: {targets:?}"
+    );
+    Ok(())
+}
+
+/// Configure a replication target on `bucket` pointing at the in-process fake,
+/// then attach an enabled replication rule for it. Returns the target ARN.
+async fn configure_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target: &FakeS3Target,
+    target_bucket: &str,
+) -> Result<String, BoxError> {
+    let arn = set_replication_target_with_options(
+        env,
+        bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(env, bucket, &arn).await?;
+    Ok(arn)
+}
+
+async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
+    let configuration = ServerSideEncryptionConfiguration::builder()
+        .rules(
+            ServerSideEncryptionRule::builder()
+                .apply_server_side_encryption_by_default(
+                    ServerSideEncryptionByDefault::builder()
+                        .sse_algorithm(ServerSideEncryption::Aes256)
+                        .build()?,
+                )
+                .build(),
+        )
+        .build()?;
+    client
+        .put_bucket_encryption()
+        .bucket(bucket)
+        .server_side_encryption_configuration(configuration)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let response = client.get_bucket_encryption().bucket(bucket).send().await?;
+    let rules = response
+        .server_side_encryption_configuration()
+        .ok_or("GetBucketEncryption omitted the configuration")?
+        .rules();
+    assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
+    assert_eq!(
+        rules[0]
+            .apply_server_side_encryption_by_default()
+            .map(ServerSideEncryptionByDefault::sse_algorithm),
+        Some(&ServerSideEncryption::Aes256),
+        "{context}: the default encryption algorithm changed"
+    );
+    Ok(())
+}
+
+async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
+    let tagging = Tagging::builder()
+        .tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
+        .build()?;
+    client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
+    Ok(())
+}
+
+async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
+    let tag_set = tags.tag_set();
+    assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
+    assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
+    assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
+    Ok(())
+}
+
+async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
+    assert_eq!(
+        versioning.status(),
+        Some(&BucketVersioningStatus::Enabled),
+        "{context}: versioning is no longer Enabled on {bucket}"
+    );
+    Ok(())
+}
+
+fn bucket_policy_document(bucket: &str) -> serde_json::Value {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Sid": "UpgradePublicRead",
+            "Effect": "Allow",
+            "Principal": { "AWS": ["*"] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
+        }]
+    })
+}
+
+/// `GET .../on-demand-migration/{bucket}/status`.
+///
+/// The migration module defaults on from rustfs#7089, so a bucket that never
+/// configured a source must still answer `configured: false` rather than
+/// engaging the migration path.
+async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
+    let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
+    );
+    let body: serde_json::Value = serde_json::from_str(&response)?;
+    assert_eq!(
+        body.get("configured"),
+        Some(&serde_json::Value::Bool(false)),
+        "a bucket upgraded from the previous release must not look migration-configured: {body}"
+    );
+    Ok(())
+}
+
+/// A GET for a key that was never written must be a plain `NoSuchKey`.
+///
+/// With the migration module on by default this is the cheap proof that an
+/// unconfigured bucket never consults a source: any migration engagement would
+/// surface as a different status or error code here.
+async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
+    let error = client
+        .get_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("a key that was never written must not be readable");
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "a missing key must stay a 404 on a bucket with no migration configuration"
+    );
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("NoSuchKey"),
+        "a missing key must stay NoSuchKey on a bucket with no migration configuration"
+    );
+    Ok(())
+}
+
+/// Bucket configuration written by the pinned previous release must survive an
+/// upgrade to the current build unchanged, and must keep working.
+///
+/// This pins the three on-disk surfaces the on-demand-migration series moved:
+///
+/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
+///   every configuration read below decodes a 44-key blob on 46-key code.
+/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
+///   public-access-block / quota blob "present but unreadable" instead of
+///   silently defaulting, and made `list-remote-targets` fail closed on it. A
+///   replication target configured by the old release must therefore still be
+///   *listed*, not dropped and not an error.
+/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
+///   encryption configuration cannot be read, so a misparsed SSE config would
+///   turn every PUT to that bucket into a 500.
+///
+/// Not covered on purpose: on-demand-migration configuration itself, which the
+/// previous release has no public API for — the reverse direction is asserted
+/// instead (an upgraded bucket reports `configured: false`).
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    // In-process: the fake target outlives both server processes, so the
+    // replication target stays reachable across the upgrade.
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
+        .await?;
+    let old_client = env.create_s3_client();
+
+    env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
+    env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
+    env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
+    old_client
+        .create_bucket()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    // Plain bucket: policy, tags, lifecycle, quota.
+    let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
+    old_client
+        .put_bucket_policy()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .policy(policy.to_string())
+        .send()
+        .await?;
+    put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
+    old_client
+        .put_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .lifecycle_configuration(
+            BucketLifecycleConfiguration::builder()
+                .rules(
+                    LifecycleRule::builder()
+                        .id(LIFECYCLE_RULE_ID)
+                        .status(ExpirationStatus::Enabled)
+                        .filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
+                        .expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
+                        .build()?,
+                )
+                .build()?,
+        )
+        .send()
+        .await?;
+    set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
+
+    // Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
+    // public access block, both of which rustfs#7172 now fails closed on.
+    put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
+    old_client
+        .put_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .block_public_acls(true)
+                .ignore_public_acls(true)
+                .block_public_policy(true)
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // Replicated bucket: versioning, a validated remote target, a rule.
+    enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
+    let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
+
+    // Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
+    old_client
+        .put_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_configuration(
+            ObjectLockConfiguration::builder()
+                .object_lock_enabled(ObjectLockEnabled::Enabled)
+                .rule(
+                    ObjectLockRule::builder()
+                        .default_retention(
+                            DefaultRetention::builder()
+                                .mode(ObjectLockRetentionMode::Governance)
+                                .days(OBJECT_LOCK_DAYS)
+                                .build(),
+                        )
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let plain_key = "plain/written-by-previous";
+    let plain_bytes = b"plain object written by the previous RustFS release";
+    put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
+
+    let encrypted_key = "encrypted/written-by-previous";
+    let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
+    old_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(encrypted_key)
+        .body(ByteStream::from_static(encrypted_bytes))
+        .send()
+        .await?;
+    assert_eq!(
+        read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
+            .await?
+            .0,
+        Some(ServerSideEncryption::Aes256),
+        "the previous release must apply the bucket default encryption it just accepted"
+    );
+
+    // The multipart object lives in the default-encrypted bucket so the
+    // upgraded build has to reassemble parts *and* re-derive the object key.
+    let multipart_key = "encrypted/multipart-written-by-previous";
+    let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
+
+    let versioned_key = "versioned/written-by-previous";
+    let versioned_bytes = b"versioned object written by the previous RustFS release";
+    let versioned_id = old_client
+        .put_object()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .key(versioned_key)
+        .body(ByteStream::from_static(versioned_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    env.restart_server_preserving_data(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    // Every configuration must read back unchanged on the upgraded build.
+    let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
+    let upgraded_policy: serde_json::Value =
+        serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
+    assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
+    assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
+
+    let lifecycle = new_client
+        .get_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .send()
+        .await?;
+    let rules = lifecycle.rules();
+    assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
+    assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
+    assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
+    assert_eq!(
+        rules[0].expiration().and_then(LifecycleExpiration::days),
+        Some(LIFECYCLE_DAYS),
+        "the lifecycle expiration changed across the upgrade"
+    );
+
+    assert_eq!(
+        get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
+        Some(BUCKET_QUOTA_BYTES),
+        "the bucket quota changed across the upgrade"
+    );
+
+    assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
+    let public_access_block = new_client
+        .get_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .send()
+        .await?;
+    let public_access_block = public_access_block
+        .public_access_block_configuration()
+        .ok_or("GetPublicAccessBlock omitted the configuration")?;
+    assert_eq!(public_access_block.block_public_acls(), Some(true));
+    assert_eq!(public_access_block.ignore_public_acls(), Some(true));
+    assert_eq!(public_access_block.block_public_policy(), Some(true));
+    assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
+
+    assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
+    // rustfs#7172: neither an empty list nor an error is acceptable here.
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
+    let replication = new_client
+        .get_bucket_replication()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .send()
+        .await?;
+    let replication_rules = replication
+        .replication_configuration()
+        .ok_or("GetBucketReplication omitted the configuration")?
+        .rules();
+    assert_eq!(
+        replication_rules.len(),
+        1,
+        "the replication rule count changed across the upgrade: {replication_rules:?}"
+    );
+    assert_eq!(
+        replication_rules[0].destination().map(|destination| destination.bucket()),
+        Some(target_arn.as_str()),
+        "the replication rule no longer points at the configured target"
+    );
+
+    let object_lock = new_client
+        .get_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .send()
+        .await?;
+    let object_lock = object_lock
+        .object_lock_configuration()
+        .ok_or("GetObjectLockConfiguration omitted the configuration")?;
+    assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
+    let retention = object_lock
+        .rule()
+        .and_then(ObjectLockRule::default_retention)
+        .ok_or("the object lock configuration lost its default retention")?;
+    assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
+    assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
+
+    // rustfs#7183: a PUT into the default-encrypted bucket must still succeed
+    // and still come back encrypted.
+    let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
+    let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
+    new_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(post_upgrade_encrypted_key)
+        .body(ByteStream::from_static(post_upgrade_encrypted_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "a PUT after the upgrade lost the bucket default encryption"
+    );
+    assert_eq!(body, post_upgrade_encrypted_bytes);
+
+    let post_upgrade_plain_key = "plain/written-after-upgrade";
+    let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
+    put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
+    assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
+    assert_eq!(body, post_upgrade_plain_bytes);
+
+    // Every object written by the previous release reads back byte-identical.
+    assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, encrypted_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
+    assert_eq!(
+        read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
+            .await?
+            .1,
+        versioned_bytes
+    );
+
+    // rustfs#7089: the migration module is on by default, but a bucket that
+    // never configured a source behaves exactly as before.
+    assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
+    assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
+
+    replication_target.shutdown().await;
+    Ok(())
+}
+
+/// Rolling back to the pinned previous release must still read the bucket
+/// metadata the current build wrote.
+///
+/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
+/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
+/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
+/// has to skip those two unknown keys instead of failing the whole blob. If it
+/// did not, every configuration read below would come back empty or error and
+/// the rollback would silently discard the bucket's configuration.
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_with_env(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    env.create_test_bucket(ROLLBACK_BUCKET).await?;
+    enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
+    put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
+    put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
+    let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
+
+    let single_key = "rollback/single";
+    let single_bytes = b"single-part object written by the current RustFS build";
+    let single_version = new_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(single_key)
+        .body(ByteStream::from_static(single_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "rollback/multipart";
+    let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
+
+    restart_from_binary(&mut env, &previous_binary, &server_env).await?;
+    let old_client = env.create_s3_client();
+
+    assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
+
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, single_bytes);
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
+
+    // A PUT on the rolled-back release must still honour the encryption
+    // configuration it decoded out of the current build's metadata blob.
+    let post_rollback_key = "rollback/written-after-rollback";
+    let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
+    old_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(post_rollback_key)
+        .body(ByteStream::from_static(post_rollback_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "the rolled-back release lost the bucket default encryption"
+    );
+    assert_eq!(body, post_rollback_bytes);
+
+    replication_target.shutdown().await;
+    Ok(())
+}

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
 # Compiles the controlled list-objects namespace-journal chaos injector into a
 # production binary (it is always available to tests). Off by default so the
 # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
@@ -212,8 +213,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
-google-cloud-storage = { workspace = true }
-google-cloud-auth = { workspace = true }
+google-cloud-storage = { workspace = true, optional = true }
+google-cloud-auth = { workspace = true, optional = true }
 faster-hex = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }

crates/ecstore/src/api/mod.rs

@@ -146,67 +146,23 @@ pub mod bucket {
         };
     }
 
-    pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
-            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
-            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
-            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
-            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
-            SourceLatencySnapshot, source_client_spec,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-            OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
-            SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
-            PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
-            WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
-            idle_guarded_body,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
-            ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
-            SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
-        };
-        pub mod backfill {
-            pub use crate::bucket::on_demand_migration::backfill::{
-                BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
-                BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
-                BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
-                BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
-                BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
-                StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
-                read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
-            };
-        }
-        pub mod source_client {
-            pub use crate::bucket::on_demand_migration::source_client::{
-                SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
-                SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
-                resolve_path_style,
-            };
-        }
-    }
-
     pub mod metadata_sys {
-        #[cfg(feature = "test-util")]
-        pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
         pub use crate::bucket::metadata_sys::{
-            BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
+            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
             capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
             get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
             get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
             get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
-            get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
-            get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
-            reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
-            update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
-            update_under_transaction_lock,
+            get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
+            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
+            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
+            update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
+            update_quota_if_incarnation, update_under_transaction_lock,
         };
+        #[cfg(feature = "test-util")]
+        pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
     }
 
     pub mod migration {
@@ -249,7 +205,7 @@ pub mod bucket {
     pub mod remote_s3_client {
         pub use crate::bucket::remote_s3_client::{
             PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
-            validate_remote_endpoint,
+            build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
         };
     }
 
@@ -495,9 +451,9 @@ pub mod object {
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
         SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
-        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
-        unregister_object_mutation_hook,
+        ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
+        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
+        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -561,6 +517,12 @@ pub mod set_disk {
     pub mod test_util {
         pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
+
+        /// Keep a namespace commit pending until the returned owner is dropped.
+        #[must_use]
+        pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync {
+            store.ctx.begin_namespace_commit()
+        }
     }
 }
 

crates/ecstore/src/bucket/metadata.rs

@@ -489,28 +489,17 @@ impl BucketMetadata {
         !self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
     }
 
-    /// Parsed per-bucket durability override, if a valid one is stored.
-    ///
-    /// Absent/empty/unparsable payloads all mean "no override" (the bucket
-    /// follows the global durability mode); a parse failure is logged so a
-    /// corrupted entry cannot silently change fsync behavior.
-    /// Parsed on-demand migration config, if one is stored.
-    ///
-    /// `Ok(None)` means no config (absent or cleared). A stored payload that
-    /// does not parse is an error, never a default: the runtime must not
-    /// pull from a source it cannot describe.
-    pub fn on_demand_migration_config(
-        &self,
-    ) -> std::result::Result<
-        Option<super::on_demand_migration::OnDemandMigrationConfig>,
-        super::on_demand_migration::OnDemandMigrationConfigError,
-    > {
-        if self.on_demand_migration_config_json.is_empty() {
-            return Ok(None);
-        }
-        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
+    /// Opaque application-owned configuration with its persisted update time.
+    /// Empty bytes mean absent or cleared; decoding belongs to the consumer.
+    pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
+        (!self.on_demand_migration_config_json.is_empty()).then_some((
+            self.on_demand_migration_config_json.as_slice(),
+            self.on_demand_migration_config_updated_at,
+        ))
     }
 
+    /// Parsed per-bucket durability override, if a valid one is stored.
+    /// Invalid payloads follow the global mode after logging a parse failure.
     pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
         if self.durability_config_json.is_empty() {
             return None;
@@ -916,13 +905,6 @@ impl BucketMetadata {
                 self.durability_config_updated_at = updated;
             }
             BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
-                // Structural check only (shape, unknown fields); the
-                // deployment-relative rules run in the admin handler with a
-                // `ValidationContext`. A blob this build cannot read must not
-                // be persisted for every later reader to trip over.
-                if !data.is_empty() {
-                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
-                }
                 self.on_demand_migration_config_json = data;
                 self.on_demand_migration_config_updated_at = updated;
             }
@@ -1978,51 +1960,30 @@ mod test {
 
     const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
-    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
-    /// extension entry that round-trips through `update_config` and the
-    /// msgpack codec, clears on delete, and never parses corruption into a
-    /// default.
+    /// The metadata codec preserves application-owned bytes and timestamps.
     #[test]
     fn on_demand_migration_config_round_trips_and_tracks_updates() {
-        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
-
         let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
-
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+        assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("valid config is accepted");
-        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
-
+            .expect("opaque config is accepted");
+        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
+        assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
         let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
         assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(
-            back.on_demand_migration_config_updated_at.unix_timestamp(),
-            bm.on_demand_migration_config_updated_at.unix_timestamp()
-        );
-        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
-
-        // A blob this build cannot read is rejected at the write boundary
-        // rather than persisted for every reader to trip over.
-        let before = bm.on_demand_migration_config_json.clone();
-        assert!(
-            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
-                .is_err()
-        );
-        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
-
-        // Delete clears the entry.
-        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
         assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
         assert!(bm.on_demand_migration_config_updated_at >= stamped);
-
-        // Corruption that bypassed `update_config` (disk, another writer)
-        // is a typed error, never a default.
-        bm.on_demand_migration_config_json = b"not-json".to_vec();
-        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
+        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
+            .unwrap();
+        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
+        assert_eq!(
+            back.on_demand_migration_config_json, b"not-json",
+            "metadata must not reinterpret application bytes"
+        );
     }
 
     /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
@@ -2034,7 +1995,7 @@ mod test {
         let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
         assert!(bm.on_demand_migration_config_json.is_empty());
         assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
 
         bm.default_timestamps();
         assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");

crates/ecstore/src/bucket/metadata_sys.rs

@@ -19,7 +19,6 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
-use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
@@ -49,6 +48,11 @@ use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};
 use uuid::Uuid;
 
+/// Opaque bucket configuration notifications for application-owned services.
+/// `None` withdraws a configuration; consumers validate nonempty bytes.
+pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
+pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
+
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
 
 #[cfg(any(test, feature = "test-util"))]
@@ -395,39 +399,21 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
-/// Publish the bucket's on-demand migration config (or its absence) to the
-/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
-///
-/// Called from the same five cache-install paths as
-/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
-/// published as `None`: the runtime must stop pulling for that bucket rather
-/// than keep an older config or guess.
+/// Publish application-owned bytes on every cache install path.
 fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
-        return;
-    };
-    match bm.on_demand_migration_config() {
-        Ok(config) => hook(bucket, config.as_ref()),
-        Err(err) => {
-            warn!(
-                event = "bucket_metadata_parse_failed",
-                component = "ecstore",
-                subsystem = "bucket_metadata",
-                bucket = %bucket,
-                config = "on_demand_migration",
-                error = %err,
-                "Failed to parse bucket metadata config"
-            );
-            hook(bucket, None);
-        }
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(
+            bucket,
+            super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
+            bm.on_demand_migration_config()
+                .map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
+        );
     }
 }
 
-/// Withdraw a bucket's on-demand migration config when its metadata leaves
-/// the cache.
 fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
-        hook(bucket, None);
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
     }
 }
 
@@ -1049,15 +1035,21 @@ pub async fn get_durability_config(
 }
 
 /// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. A stored payload that does not parse
-/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
+pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
     bucket_meta_sys.get_on_demand_migration_config(bucket).await
 }
 
+/// Resolve opaque configuration from the store's own metadata system.
+pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+    let sys = bucket_metadata_sys_of(&api.ctx)?;
+    let lock = sys.read().await;
+    lock.get_on_demand_migration_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2579,29 +2571,27 @@ impl BucketMetadataSys {
     }
 
     /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(
-        &self,
-        bucket: &str,
-    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+    pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        let config = bm.on_demand_migration_config().map_err(Error::other)?;
-        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
+        Ok(bm
+            .on_demand_migration_config()
+            .map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
     }
 }
 
 /// Test-only fixture shared with sibling modules (e.g. the quota checker
 /// tests): a 4-disk `ECStore` on an isolated instance context, so tests
 /// exercising the metadata system never touch ambient process state.
-#[cfg(test)]
-pub(crate) mod test_support {
+#[cfg(any(test, feature = "test-util"))]
+pub mod test_support {
     use super::*;
     use crate::disk::endpoint::Endpoint;
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::store::init_local_disks_with_instance_ctx;
 
-    pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+    pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
         let mut dirs = Vec::with_capacity(4);
         let mut endpoints = Vec::with_capacity(4);
         for disk_idx in 0..4 {
@@ -4385,19 +4375,26 @@ mod tests {
 
     const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
+    type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
+    type RecordedOdmHookCall = (String, RecordedOdmConfig);
+
     /// Every `(bucket, config)` the recording hook has seen. Tests filter by
     /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
+    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
 
     fn install_recording_odm_hook() {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
-            Box::new(|bucket, config| {
-                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
+        BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
+            Box::new(|bucket, config_file, config| {
+                assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
+                ODM_HOOK_CALLS.lock().unwrap().push((
+                    bucket.to_string(),
+                    config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
+                ));
             })
         });
     }
 
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
+    fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
         ODM_HOOK_CALLS
             .lock()
             .unwrap()
@@ -4407,54 +4404,6 @@ mod tests {
             .collect()
     }
 
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
-
     /// rustfs/backlog#2148: the publish hook fires on every path that
     /// installs bucket metadata into the cache (set, initial load, peer
     /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
@@ -4468,15 +4417,22 @@ mod tests {
         for dir in &dirs {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
         }
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+
+        let incarnation = Uuid::new_v4();
         let expect_publish = |before: usize, label: &str| {
             let calls = odm_hook_calls(bucket);
             assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
+            assert_eq!(
+                calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
+                Some(ODM_JSON),
+                "{label} must publish the stored bytes"
+            );
+            assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
         };
 
         // set (via persist_new_and_set, which installs through `set`).
         let mut bm = BucketMetadata::new(bucket);
+        bm.bucket_incarnation_id = incarnation;
         bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
             .unwrap();
         let writer = BucketMetadataSys::new(ecstore.clone());
@@ -4518,14 +4474,18 @@ mod tests {
         assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
         assert_eq!(calls.last().unwrap(), &None);
 
-        // A corrupt payload is withdrawn, never published as a config.
+        // Opaque bytes reach the application even if they are not valid JSON.
         let mut corrupt = BucketMetadata::new(bucket);
         corrupt.on_demand_migration_config_json = b"not-json".to_vec();
         let before = odm_hook_calls(bucket).len();
         lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
         let calls = odm_hook_calls(bucket);
         assert_eq!(calls.len(), before + 1);
-        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
+        assert_eq!(
+            calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
+            Some(b"not-json".as_slice()),
+            "the application validates opaque config bytes"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/bucket/mod.rs

@@ -26,7 +26,6 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
-pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec {
 
 #[derive(Debug, thiserror::Error)]
 pub enum RemoteS3ClientError {
+    #[error("the {0} backend is not included in this build")]
+    BackendNotCompiled(&'static str),
     #[error("remote endpoint requires credentials")]
     MissingCredentials,
     #[error("{0}")]
@@ -281,9 +283,7 @@ impl Intercept for UserAgentSuffixInterceptor {
 
 /// Builds the SDK config for `spec` without finalizing it, so callers can add
 /// interceptors or (in tests) swap the HTTP client before `build()`.
-pub(crate) async fn build_remote_s3_config(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
+pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
     let Some(credentials) = &spec.credentials else {
         return Err(RemoteS3ClientError::MissingCredentials);
     };
@@ -523,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
     Ok(())
 }
 
-pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
+pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
     validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
 }
 

crates/ecstore/src/object_api/types.rs

@@ -956,6 +956,9 @@ pub struct ObjectOptions {
     pub preserve_etag: Option<String>,
     pub metadata_chg: bool,
     pub http_preconditions: Option<HTTPPreconditions>,
+    /// Internal create-only writes may also preserve an acknowledged deletion.
+    /// Evaluated with `http_preconditions` under the namespace commit lock.
+    pub preserve_delete_marker: bool,
 
     pub delete_replication: Option<ReplicationState>,
     pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,

crates/ecstore/src/runtime/instance.rs

@@ -78,6 +78,21 @@ pub(crate) struct ScannerPublicationLeaseEntry {
     pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
 }
 
+pub(crate) struct NamespaceCommitGuard {
+    ctx: Arc<InstanceContext>,
+    counted: bool,
+}
+
+impl Drop for NamespaceCommitGuard {
+    fn drop(&mut self) {
+        if self.counted {
+            // Publish the new generation before a zero-pending publication probe.
+            self.ctx.advance_namespace_commit_generation();
+            self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
+        }
+    }
+}
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -209,9 +224,13 @@ pub struct InstanceContext {
     /// Last storage-owned movement snapshot observed under the operation
     /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
     scanner_publication_state: AtomicU8,
+    namespace_commits: AtomicU64,
+    namespace_commit_generation: AtomicU64,
     /// Resolves object-encryption material at the application boundary.
     object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
+    #[cfg(test)]
+    suppress_tier_delete_journal_recovery: bool,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
 }
@@ -256,8 +275,12 @@ impl InstanceContext {
             data_movement_generation_exhausted: AtomicBool::new(false),
             data_movement_generation_notify: Arc::new(Notify::new()),
             scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
+            namespace_commits: AtomicU64::new(0),
+            namespace_commit_generation: AtomicU64::new(0),
             object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
+            #[cfg(test)]
+            suppress_tier_delete_journal_recovery: false,
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
         }
@@ -385,6 +408,36 @@ impl InstanceContext {
             && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
     }
 
+    pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
+        let counted = self
+            .namespace_commits
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
+            .is_ok();
+        if counted {
+            self.advance_namespace_commit_generation();
+        } else {
+            self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
+        }
+        Arc::new(NamespaceCommitGuard {
+            ctx: Arc::clone(self),
+            counted,
+        })
+    }
+
+    fn advance_namespace_commit_generation(&self) {
+        let _ = self
+            .namespace_commit_generation
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
+    }
+
+    pub(crate) fn namespace_commit_generation(&self) -> u64 {
+        self.namespace_commit_generation.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn namespace_commits_pending(&self) -> bool {
+        self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
+    }
+
     pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
         self.scanner_publication_state.store(
             if blocked {
@@ -640,12 +693,21 @@ impl InstanceContext {
     }
 
     pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
+        #[cfg(test)]
+        if self.suppress_tier_delete_journal_recovery {
+            return false;
+        }
         self.tier_delete_journal_recovery_stores
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner)
             .insert(store_id)
     }
 
+    #[cfg(test)]
+    pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
+        self.suppress_tier_delete_journal_recovery = true;
+    }
+
     pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
         self.transition_transaction_recovery_stores
             .lock()
@@ -756,6 +818,50 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
+        let first = Arc::new(InstanceContext::new());
+        let other = Arc::new(InstanceContext::new());
+        first.set_scanner_publication_state(false);
+        other.set_scanner_publication_state(false);
+        assert!(first.scanner_publication_state_allowed());
+        let one = first.begin_namespace_commit();
+        let shared_owner = Arc::clone(&one);
+        let two = first.begin_namespace_commit();
+        assert!(first.namespace_commits_pending());
+        assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
+        assert_eq!(first.namespace_commit_generation(), 2);
+        assert!(!other.namespace_commits_pending());
+        assert_eq!(other.namespace_commit_generation(), 0);
+        assert!(other.scanner_publication_state_allowed());
+        drop(one);
+        assert_eq!(first.namespace_commit_generation(), 2);
+        drop(shared_owner);
+        assert!(first.namespace_commits_pending());
+        assert_eq!(first.namespace_commit_generation(), 3);
+        drop(two);
+        assert!(!first.namespace_commits_pending());
+        assert_eq!(first.namespace_commit_generation(), 4);
+        assert!(first.scanner_publication_state_allowed());
+    }
+
+    #[test]
+    fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
+        for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
+            let ctx = Arc::new(InstanceContext::new());
+            ctx.set_scanner_publication_state(false);
+            ctx.namespace_commits.store(count, Ordering::Release);
+            ctx.namespace_commit_generation.store(generation, Ordering::Release);
+            let guard = ctx.begin_namespace_commit();
+            assert!(ctx.namespace_commits_pending());
+            assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
+            drop(guard);
+            assert!(ctx.namespace_commits_pending());
+            assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
+            assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
+        }
+    }
+
     // The SetupType inputs must derive the exact (is_erasure,
     // is_dist_erasure, is_erasure_sd) triples that the original three
     // process-global erasure bools produced via update_erasure_type().
@@ -1073,6 +1179,12 @@ mod tests {
         assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
         assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
         assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
+
+        let mut manual_ctx = InstanceContext::new();
+        manual_ctx.suppress_tier_delete_journal_recovery_for_test();
+        assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
+        assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
+        assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
     }
 
     #[test]

crates/ecstore/src/services/tier/mod.rs

@@ -25,6 +25,7 @@ pub(crate) mod tier_probe_intent;
 pub mod warm_backend;
 pub mod warm_backend_aliyun;
 pub mod warm_backend_azure;
+#[cfg(feature = "gcs")]
 pub mod warm_backend_gcs;
 pub mod warm_backend_huaweicloud;
 pub mod warm_backend_minio;

crates/ecstore/src/services/tier/tier.rs

@@ -3541,7 +3541,7 @@ impl TierConfigMgr {
         // Get tier configuration and create new driver
         let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
 
-        let driver = new_warm_backend(tier_config, false).await?;
+        let driver = construct_warm_backend(tier_config).await?;
 
         self.replace_driver(tier_name, driver)?;
         Ok(self
@@ -4486,6 +4486,11 @@ impl TierConfigMgr {
                     let committed_coordinator_intent =
                         committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag)
                             .map_err(TierConfigUpdateError::Save)?;
+                    // Persist Committed before notifying refresh; a Prepared disk record
+                    // would restore the prepared block and invalidate our publish allowance.
+                    let coordinator_commit =
+                        commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
+                            .await;
                     if let Some(intent) = committed_coordinator_intent.as_ref() {
                         TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent)
                             .await
@@ -4496,9 +4501,9 @@ impl TierConfigMgr {
                                 .map_err(TierConfigUpdateError::Publish)?,
                         );
                     }
-                    commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
-                        .await
-                        .map_err(TierConfigUpdateError::Save)?;
+                    // Config is already saved: retain the committed fence and wake recovery
+                    // even when the coordinator commit failed or its outcome is unknown.
+                    coordinator_commit.map_err(TierConfigUpdateError::Save)?;
                     if coordinated_config_update {
                         drop(update.take());
                         drop(config_lock.take());
@@ -10603,6 +10608,11 @@ mod tests {
             .expect_err("coordinator committed-state CAS failure must be observable");
         assert!(matches!(err, TierConfigUpdateError::Save(_)));
         assert!(manager.read().await.tiers.contains_key("COLD-A"));
+        assert!(TierConfigMgr::has_committed_mutation_block(&manager).await);
+        let refresh = TierConfigMgr::mutation_refresh_notifier(&manager).await;
+        tokio::time::timeout(Duration::from_secs(1), refresh.notified())
+            .await
+            .expect("failed coordinator commit must notify recovery after saving config");
         let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
             Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"),
             Err(err) => err,
@@ -14329,6 +14339,12 @@ mod tests {
         after_commit: bool,
     }
 
+    #[derive(Debug, Default)]
+    struct CasCoordinatorCommitBarrier {
+        arrived: tokio::sync::Notify,
+        release: tokio::sync::Notify,
+    }
+
     #[derive(Debug)]
     struct CasConfigStore {
         objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
@@ -14341,6 +14357,7 @@ mod tests {
         fail_delete_prefix: tokio::sync::Mutex<Option<(String, usize)>>,
         delete_log: tokio::sync::Mutex<Vec<String>>,
         list_barrier: tokio::sync::Mutex<Option<Arc<CasListBarrier>>>,
+        coordinator_commit_barrier: tokio::sync::Mutex<Option<Arc<CasCoordinatorCommitBarrier>>>,
         intent_list_calls: AtomicUsize,
         fail_reference_walk: AtomicBool,
         reference_walk_send_count: AtomicUsize,
@@ -14363,6 +14380,7 @@ mod tests {
                 fail_delete_prefix: tokio::sync::Mutex::new(None),
                 delete_log: tokio::sync::Mutex::new(Vec::new()),
                 list_barrier: tokio::sync::Mutex::new(None),
+                coordinator_commit_barrier: tokio::sync::Mutex::new(None),
                 intent_list_calls: AtomicUsize::new(0),
                 fail_reference_walk: AtomicBool::new(false),
                 reference_walk_send_count: AtomicUsize::new(0),
@@ -14554,6 +14572,19 @@ mod tests {
             }
             let mut payload = Vec::new();
             tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
+            if object.starts_with(crate::services::tier::tier_mutation_intent::TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX)
+                && opts
+                    .http_preconditions
+                    .as_ref()
+                    .and_then(HTTPPreconditions::if_match_value)
+                    .is_some()
+            {
+                let barrier = self.coordinator_commit_barrier.lock().await.take();
+                if let Some(barrier) = barrier {
+                    barrier.arrived.notify_one();
+                    barrier.release.notified().await;
+                }
+            }
             let race_rewrite = if opts
                 .http_preconditions
                 .as_ref()
@@ -15651,14 +15682,7 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn force_remove_and_save_bypasses_lifecycle_only_reference() {
-        // rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level
-        // reference-proof function) for a tier with zero transitioned objects but a lifecycle
-        // rule still pointing at it — the exact shape of
-        // `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test,
-        // which force-removes a tier a lifecycle rule still references to simulate a
-        // decommissioned backend.
+    async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
         let store = Arc::new(CasConfigStore::default());
         let tier = build_rustfs_tier("COLD-A");
         let mut persisted = empty_mgr();
@@ -15699,22 +15723,55 @@ mod tests {
 
         let manager = TierConfigMgr::new();
         manager.write().await.tiers.insert("COLD-A".to_string(), tier);
-        TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
-            .await
-            .expect("force remove must bypass a lifecycle-config-only reference");
+        let mutation = if clear {
+            TierCandidateMutation::Clear(force)
+        } else {
+            TierCandidateMutation::Remove("COLD-A".to_string(), force)
+        };
+        let result = TIER_DRIVER_TEST_FACTORY
+            .scope(
+                healthy_driver_factory(),
+                TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation),
+            )
+            .await;
+        if force {
+            result.expect("force mutation must bypass a lifecycle-config-only reference");
+        } else {
+            let err = result.expect_err("non-force mutation must reject a lifecycle-only reference");
+            let TierConfigUpdateError::Publish(err) = err else {
+                panic!("non-force mutation must fail during reference proof: {err:?}");
+            };
+            assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
+            assert!(err.message.contains("move-current"), "{err}");
+        }
 
-        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
-        assert!(
-            !load_tier_config_for_update(store)
+        assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), !force);
+        assert_eq!(
+            load_tier_config_for_update(store)
                 .await
                 .expect("config should still reload")
                 .0
                 .tiers
                 .contains_key("COLD-A"),
-            "force removal must persist the empty candidate"
+            !force,
+            "persisted state must match the force mutation result"
         );
     }
 
+    #[tokio::test]
+    async fn remove_with_config_lock_obeys_force_for_lifecycle_only_reference() {
+        for force in [false, true] {
+            assert_lifecycle_only_reference_obeys_force(false, force).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn clear_with_config_lock_obeys_force_for_lifecycle_only_reference() {
+        for force in [false, true] {
+            assert_lifecycle_only_reference_obeys_force(true, force).await;
+        }
+    }
+
     #[tokio::test]
     async fn zero_reference_proof_blocks_clear_before_config_save() {
         let store = Arc::new(CasConfigStore::default());
@@ -17255,6 +17312,98 @@ mod tests {
         assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
     }
 
+    async fn assert_coordinator_commit_refresh_succeeds(mutation: TierCandidateMutation) {
+        let adding = matches!(mutation, TierCandidateMutation::Add(..));
+        let manager = TierConfigMgr::new();
+        let store = Arc::new(CasConfigStore::default());
+        if !adding {
+            let mut persisted = empty_mgr();
+            persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
+            persisted
+                .save_tiering_config_if_current(store.clone(), None)
+                .await
+                .expect("existing tier fixture should persist");
+            let mut guard = manager.write().await;
+            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
+        }
+        let barrier = Arc::new(CasCoordinatorCommitBarrier::default());
+        *store.coordinator_commit_barrier.lock().await = Some(barrier.clone());
+        let update_manager = manager.clone();
+        let update_store = store.clone();
+        let update = tokio::spawn(async move {
+            TIER_DRIVER_TEST_FACTORY
+                .scope(
+                    healthy_driver_factory(),
+                    TIER_MUTATION_TEST_PEERS.scope(
+                        Vec::new(),
+                        TierConfigMgr::update_candidate_with_config_lock(&update_manager, update_store, mutation),
+                    ),
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(5), barrier.arrived.notified())
+            .await
+            .expect("mutation should reach coordinator commit after saving config");
+        assert_eq!(
+            load_tier_config_for_update(store.clone())
+                .await
+                .expect("saved config should be readable before coordinator commit")
+                .0
+                .tiers
+                .contains_key("COLD-A"),
+            adding
+        );
+        assert_eq!(
+            TierConfigMgr::load_coordinator_mutation_intents(store.clone())
+                .await
+                .expect("coordinator intent should remain readable")[0]
+                .state,
+            TierMutationIntentState::Prepared
+        );
+
+        let lock_requests = lock_unpoisoned(&store.lock_requests).len();
+        // Also exercise an independently scheduled refresh while the durable
+        // coordinator record is still Prepared, before its commit notification.
+        TierConfigMgr::request_committed_mutation_refresh(&manager).await;
+        TIER_MUTATION_TEST_PEERS
+            .scope(Vec::new(), async {
+                let worker = TierConfigMgr::refresh_tier_config_handle_with(manager.clone(), store.clone());
+                tokio::pin!(worker);
+                tokio::time::timeout(Duration::from_secs(5), async {
+                    while lock_unpoisoned(&store.lock_requests).len() == lock_requests {
+                        tokio::select! {
+                            _ = &mut worker => panic!("refresh worker must remain available"),
+                            _ = tokio::task::yield_now() => {}
+                        }
+                    }
+                })
+                .await
+                .expect("refresh should reconcile the Prepared record before waiting for the config lock");
+                barrier.release.notify_one();
+                let result = tokio::time::timeout(Duration::from_secs(5), async {
+                    tokio::select! {
+                        _ = &mut worker => panic!("refresh worker must remain available"),
+                        result = update => result.expect("tier mutation task should join"),
+                    }
+                })
+                .await
+                .expect("tier mutation should finish with refresh running");
+                result.expect("saved tier mutation must publish successfully on the first attempt");
+            })
+            .await;
+        assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), adding);
+    }
+
+    #[tokio::test]
+    async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
+        assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
+    }
+
+    #[tokio::test]
+    async fn tier_remove_succeeds_with_refresh_during_coordinator_commit() {
+        assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Remove("COLD-A".to_string(), true)).await;
+    }
+
     async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc<RwLock<TierConfigMgr>>, Arc<CasConfigStore>, uuid::Uuid) {
         let manager = TierConfigMgr::new();
         {

crates/ecstore/src/services/tier/warm_backend.rs

@@ -19,13 +19,14 @@
 #![allow(clippy::all)]
 
 use crate::error::is_err_bucket_not_found;
+#[cfg(feature = "gcs")]
+use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
 use crate::services::tier::{
     tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
     tier_config::{TierConfig, TierType},
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
     warm_backend_azure::WarmBackendAzure,
-    warm_backend_gcs::WarmBackendGCS,
     warm_backend_huaweicloud::WarmBackendHuaweicloud,
     warm_backend_minio::WarmBackendMinIO,
     warm_backend_r2::WarmBackendR2,
@@ -37,7 +38,7 @@ use crate::services::tier::{
 use bytes::Bytes;
 use http::StatusCode;
 use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
-use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
+use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
 use rustfs_s3_client::{
     admin_handler_utils::AdminError,
     api_error_response::to_error_response,
@@ -320,6 +321,27 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
     }
 }
 
+fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
+}
+
+pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
+    TransitionClientTimeouts::new(
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
+        ),
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
+        ),
+        transition_timeout_from_env(
+            rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+            rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+        ),
+    )
+}
+
 /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
 ///
 /// Credential, bucket, and endpoint validation run in this order because the
@@ -350,6 +372,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
         signer_type: SignatureType::SignatureV4,
         ..Default::default()
     }));
+    let timeouts = transition_client_timeouts_from_env();
     let opts = Options {
         creds,
         secure: u.scheme() == "https",
@@ -362,7 +385,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
     // Run the SSRF guard after the host-presence check so a host-less endpoint
     // keeps this constructor's stable error text.
     (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
-    let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
+    let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
 
     let client = Arc::new(client);
     let core = TransitionCore(Arc::clone(&client));
@@ -890,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
                 });
             }
         }
+        #[cfg(not(feature = "gcs"))]
+        TierType::GCS => {
+            return Err(AdminError {
+                code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
+                message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
+                status_code: StatusCode::NOT_IMPLEMENTED,
+            });
+        }
+        #[cfg(feature = "gcs")]
         TierType::GCS => {
             if let Some(gcs_config) = tier.gcs.as_ref() {
                 let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
@@ -1006,6 +1038,27 @@ mod tests {
 
     const PROBE_VERSION: &str = "remote-v2";
 
+    #[cfg(not(feature = "gcs"))]
+    #[tokio::test]
+    async fn gcs_backend_not_compiled_preserves_config() {
+        let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
+        let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
+        assert_eq!(tier.tier_type, TierType::GCS);
+        let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
+        let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
+        assert_eq!(restored.tier_type, TierType::GCS);
+        let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
+        assert_eq!(restored_gcs.bucket, "archive");
+        assert_eq!(restored_gcs.creds, "secret");
+        assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
+        let error = match new_warm_backend(&tier, false).await {
+            Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
+            Err(error) => error,
+        };
+        assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
+        assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
+    }
+
     struct CountingBackend {
         put_result: fn() -> Result<String, std::io::Error>,
         removes: Arc<AtomicUsize>,

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -26,7 +26,7 @@ use crate::services::tier::{
     tier_config::TierS3,
     warm_backend::{
         TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
-        build_transition_put_options, endpoint_authority,
+        build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
     },
 };
 use http::HeaderMap;
@@ -139,6 +139,7 @@ impl WarmBackendS3 {
         } else {
             return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
         }
+        let timeouts = transition_client_timeouts_from_env();
         let opts = Options {
             creds,
             secure: u.scheme() == "https",
@@ -147,7 +148,7 @@ impl WarmBackendS3 {
             ..Default::default()
         };
         let endpoint = endpoint_authority(&u)?;
-        let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
+        let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3558,6 +3558,11 @@ impl RenameRollbackReceipt {
     }
 }
 
+struct RenameRollbackOwnership {
+    receipt: Option<RenameRollbackReceipt>,
+    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
+}
+
 async fn inspect_incomplete_rename_rollback(
     disks: &[Option<DiskStore>],
     bucket: &str,
@@ -3604,8 +3609,12 @@ async fn rollback_failed_rename(
     dispatch_states: &[RenameDispatchState],
     rollback_dirs: &[Option<Uuid>],
     dst: (&str, &str),
-    receipt: Option<RenameRollbackReceipt>,
+    ownership: RenameRollbackOwnership,
 ) {
+    let RenameRollbackOwnership {
+        receipt,
+        namespace_commit_guard,
+    } = ownership;
     let owned_disks = disks.to_vec();
     let owned_errs = errs.to_vec();
     let owned_dispatch_states = dispatch_states.to_vec();
@@ -3651,7 +3660,9 @@ async fn rollback_failed_rename(
             let fi = std::mem::take(&mut file_infos[disk_index]);
             let bucket = bucket.to_string();
             let object = object.to_string();
+            let disk_namespace_commit_guard = namespace_commit_guard.clone();
             let task = tokio::spawn(async move {
+                let _namespace_commit_guard = disk_namespace_commit_guard;
                 #[allow(clippy::let_unit_value)]
                 let _task_guard = SetDisks::rename_fanout_task_guard(&object);
                 SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
@@ -3672,6 +3683,9 @@ async fn rollback_failed_rename(
             });
             tasks.push(async move { (disk_index, task.await) });
         }
+        #[cfg(test)]
+        rollback_fault_injection::after_undo_dispatch(object);
+        let _namespace_commit_guard = namespace_commit_guard;
         for (disk_index, result) in join_all(tasks).await {
             outcomes[disk_index].outcome = rename_rollback_task_outcome(result);
         }
@@ -3778,6 +3792,7 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
     write_quorum: usize,
     scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
     scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
+    namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
     rollback_receipt: Option<RenameRollbackReceipt>,
 }
 
@@ -3790,6 +3805,7 @@ impl<'a> RenameDataFenceOptions<'a> {
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: None,
+            namespace_commit_guard: None,
             rollback_receipt: None,
         }
     }
@@ -3806,6 +3822,14 @@ impl<'a> RenameDataFenceOptions<'a> {
         self.scanner_publication_commit_scope = scanner_publication_commit_scope;
         self
     }
+
+    pub(in crate::set_disk) fn with_namespace_commit_guard(
+        mut self,
+        namespace_commit_guard: Option<Arc<crate::runtime::instance::NamespaceCommitGuard>>,
+    ) -> Self {
+        self.namespace_commit_guard = namespace_commit_guard;
+        self
+    }
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -4164,6 +4188,7 @@ impl SetDisks {
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope: _scanner_publication_commit_scope,
+            namespace_commit_guard,
             rollback_receipt,
         } = fence_options;
         if let Some(file_info) = disks
@@ -4210,7 +4235,9 @@ impl SetDisks {
                     let dst_object = fanout_dst_object.clone();
                     let file_info = file_info.clone();
                     let successful_rename_completion_rank = successful_rename_completion_rank.clone();
+                    let namespace_commit_guard = namespace_commit_guard.clone();
                     tasks.spawn(async move {
+                        let _namespace_commit_guard = namespace_commit_guard;
                         let mut dispatch_state = RenameDispatchState::NotDispatched;
                         let result = std::panic::AssertUnwindSafe(async {
                             #[allow(clippy::let_unit_value)]
@@ -4372,7 +4399,10 @@ impl SetDisks {
                         &dispatch_states,
                         &data_dirs,
                         (&fanout_dst_bucket, &fanout_dst_object),
-                        rollback_receipt,
+                        RenameRollbackOwnership {
+                            receipt: rollback_receipt,
+                            namespace_commit_guard,
+                        },
                     )
                     .await;
                     if let Some(commit_tx) = commit_tx.take() {
@@ -4528,6 +4558,7 @@ impl SetDisks {
             write_quorum,
             scanner_publication_lease_tokens,
             scanner_publication_commit_scope,
+            namespace_commit_guard,
             rollback_receipt,
         } = fence_options;
         if let Some(file_info) = disks
@@ -4561,6 +4592,7 @@ impl SetDisks {
         let fanout_dst_bucket = dst_bucket.clone();
         let fanout_dst_object = dst_object.clone();
         let fanout_publication_scope = scanner_publication_commit_scope.clone();
+        let fanout_namespace_commit_guard = namespace_commit_guard.clone();
         // Keep one coordinator task so a cancelled caller cannot drop partially
         // completed disk mutations. Per-disk futures stay ordered in `join_all`,
         // preserving slot-indexed quorum and convergence accounting without a
@@ -4569,6 +4601,7 @@ impl SetDisks {
             // Keep the storage-owned movement permit attached to the actual
             // fan-out owner, even if the caller future is cancelled.
             let _fanout_publication_scope = fanout_publication_scope;
+            let _namespace_commit_guard = fanout_namespace_commit_guard;
             let successful_rename_completion_rank =
                 rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
             let futures = fanout_disks
@@ -4790,7 +4823,10 @@ impl SetDisks {
                 &dispatch_states,
                 &data_dirs,
                 (&dst_bucket, &dst_object),
-                rollback_receipt,
+                RenameRollbackOwnership {
+                    receipt: rollback_receipt,
+                    namespace_commit_guard,
+                },
             )
             .await;
             return Err(ret_err);
@@ -6503,9 +6539,9 @@ impl SetDisks {
         match oi {
             Ok(oi) => {
                 // Ordinary writes may proceed past a top-level delete marker;
-                // data movement must not replace an acknowledged deletion.
+                // data movement and guarded internal writes must preserve it.
                 if oi.delete_marker {
-                    return opts.data_movement.then_some(StorageError::PreconditionFailed);
+                    return (opts.data_movement || opts.preserve_delete_marker).then_some(StorageError::PreconditionFailed);
                 }
                 let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
                 let if_match = http_preconditions.if_match_value().map(str::to_owned);
@@ -6754,6 +6790,7 @@ pub(in crate::set_disk) mod rollback_fault_injection {
         VolumeNotFoundAfterRename,
         PanicAfterRename,
         CoordinatorPanic,
+        RollbackCoordinatorPanic,
     }
 
     fn registry() -> &'static Mutex<HashMap<String, (usize, Fault)>> {
@@ -6816,6 +6853,17 @@ pub(in crate::set_disk) mod rollback_fault_injection {
             panic!("injected rename coordinator panic");
         }
     }
+
+    pub(super) fn after_undo_dispatch(object: &str) {
+        let fault = registry()
+            .lock()
+            .expect("rollback registry should not poison")
+            .get(object)
+            .copied();
+        if matches!(fault, Some((_, Fault::RollbackCoordinatorPanic))) {
+            panic!("injected rollback coordinator panic");
+        }
+    }
 }
 
 /// Test-only per-disk call counters for the metadata fan-out (backlog#1325,
@@ -6977,7 +7025,7 @@ pub(crate) mod rename_fanout_barrier {
     use tokio::sync::Notify;
 
     pub use super::rename_fanout_barrier_phase::{
-        CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME,
+        CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, ROLLBACK as PHASE_ROLLBACK,
     };
 
     /// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses.
@@ -10814,79 +10862,177 @@ mod tests {
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
-        for cancel_caller in [false, true] {
-            let bucket = "rename-rollback-barrier";
-            let object = if cancel_caller {
-                "rollback-barrier-cancelled"
-            } else {
-                "rollback-barrier-object"
-            };
-            let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
-            prepare_rename_source_dirs(&dirs, &disks, "source").await;
-            let mut old = metadata_test_fileinfo(object);
-            old.mod_time = Some(OffsetDateTime::now_utc());
-            old.data = Some(Bytes::from_static(b"old-inline-body"));
-            old.set_inline_data();
-            old.metadata.insert("etag".to_string(), "old-etag".to_string());
-            for disk in disks.iter().flatten() {
-                disk.write_metadata(bucket, bucket, object, old.clone())
-                    .await
-                    .expect("old metadata should be staged");
-            }
-            let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
-            let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io);
-            let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
-            let receipt = RenameRollbackReceipt::default();
-            let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
-                &disks,
-                (RUSTFS_META_TMP_BUCKET, "source"),
-                rename_commit_fileinfos(object, 4, "new-etag"),
-                (bucket, object),
-                false,
-                RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()),
-            ));
-            tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
-                tokio::select! {
-                    () = barrier.wait_until_paused() => {}
-                    _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"),
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for (allow_early_ack, cancel_caller, object) in [
+                (false, false, "rollback-barrier-object"),
+                (false, true, "rollback-barrier-cancelled"),
+                (true, false, "rollback-barrier-early-object"),
+                (true, true, "rollback-barrier-early-cancelled"),
+            ] {
+                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+                let bucket = "rename-rollback-barrier";
+                let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
+                prepare_rename_source_dirs(&dirs, &disks, "source").await;
+                let mut old = metadata_test_fileinfo(object);
+                old.mod_time = Some(OffsetDateTime::now_utc());
+                old.data = Some(Bytes::from_static(b"old-inline-body"));
+                old.set_inline_data();
+                old.metadata.insert("etag".to_string(), "old-etag".to_string());
+                for disk in disks.iter().flatten() {
+                    disk.write_metadata(bucket, bucket, object, old.clone())
+                        .await
+                        .expect("old metadata should be staged");
                 }
-            })
-            .await
-            .expect("undo must reach its disk barrier");
-            assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success");
-            if cancel_caller {
-                drop(rename);
+                let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
+                let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io);
+                let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
+                let receipt = RenameRollbackReceipt::default();
+                let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
+                    &disks,
+                    (RUSTFS_META_TMP_BUCKET, "source"),
+                    rename_commit_fileinfos(object, 4, "new-etag"),
+                    (bucket, object),
+                    allow_early_ack,
+                    RenameDataFenceOptions::new(3, None)
+                        .with_rollback_receipt(receipt.clone())
+                        .with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
+                ));
+                tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
+                    tokio::select! {
+                        () = barrier.wait_until_paused() => {}
+                        _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"),
+                    }
+                })
+                .await
+                .expect("undo must reach its disk barrier");
+                assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success");
+                assert!(ctx.namespace_commits_pending());
+                assert_eq!(ctx.namespace_commit_generation(), 1);
+                if cancel_caller {
+                    drop(rename);
+                    assert!(ctx.namespace_commits_pending(), "caller cancellation must not retire pending undo work");
+                    assert_eq!(ctx.namespace_commit_generation(), 1);
+                    barrier.release();
+                    tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
+                        while receipt.0.get().is_none() || ctx.namespace_commits_pending() {
+                            tokio::task::yield_now().await;
+                        }
+                    })
+                    .await
+                    .expect("cancelled caller must not cancel rollback accounting");
+                } else {
+                    barrier.release();
+                    assert!(rename.await.is_err());
+                }
+                assert!(
+                    !ctx.namespace_commits_pending(),
+                    "the completed rollback must release its namespace ownership"
+                );
+                assert_eq!(ctx.namespace_commit_generation(), 2);
+                assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt");
+                for dir in dirs.iter().skip(1) {
+                    let reopened = reopen_local_disk(dir).await;
+                    let restored = reopened
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("old version must remain readable after caller cancellation");
+                    assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
+                }
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn rename_rollback_children_keep_namespace_ownership_after_coordinator_panic() {
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            for (allow_early_ack, object) in [
+                (false, "rollback-coordinator-panic"),
+                (true, "rollback-coordinator-panic-early"),
+            ] {
+                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+                let bucket = "rename-rollback-coordinator-panic";
+                let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
+                prepare_rename_source_dirs(&dirs, &disks, "source").await;
+                let mut old = metadata_test_fileinfo(object);
+                old.mod_time = Some(OffsetDateTime::now_utc());
+                old.data = Some(Bytes::from_static(b"old-inline-body"));
+                old.set_inline_data();
+                old.metadata.insert("etag".to_string(), "old-etag".to_string());
+                for disk in disks.iter().flatten() {
+                    disk.write_metadata(bucket, bucket, object, old.clone())
+                        .await
+                        .expect("old metadata should be staged");
+                }
+                let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
+                let _rollback_fault =
+                    rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::RollbackCoordinatorPanic);
+                let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK);
+                let receipt = RenameRollbackReceipt::default();
+                let result = tokio::time::timeout(
+                    BARRIER_PAUSE_GUARD,
+                    SetDisks::rename_data_owned_with_fence(
+                        &disks,
+                        (RUSTFS_META_TMP_BUCKET, "source"),
+                        rename_commit_fileinfos(object, 4, "new-etag"),
+                        (bucket, object),
+                        allow_early_ack,
+                        RenameDataFenceOptions::new(3, None)
+                            .with_rollback_receipt(receipt.clone())
+                            .with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
+                    ),
+                )
+                .await
+                .expect("coordinator failure must return without waiting for detached undo tasks");
+                assert!(result.is_err());
+                tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
+                    .await
+                    .expect("detached undo must reach its disk barrier");
+                assert!(
+                    receipt.is_incomplete(),
+                    "coordinator failure must preserve indeterminate recovery evidence"
+                );
+                assert!(ctx.namespace_commits_pending(), "the paused child must retain namespace ownership");
+                assert_eq!(ctx.namespace_commit_generation(), 1);
                 barrier.release();
                 tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
-                    while receipt.0.get().is_none() {
+                    while ctx.namespace_commits_pending() {
                         tokio::task::yield_now().await;
                     }
                 })
                 .await
-                .expect("cancelled caller must not cancel rollback accounting");
-            } else {
-                barrier.release();
-                assert!(rename.await.is_err());
+                .expect("completed undo children must release their namespace ownership");
+                assert_eq!(ctx.namespace_commit_generation(), 2);
+                for dir in &dirs {
+                    let reopened = reopen_local_disk(dir).await;
+                    let restored = reopened
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("old version must remain readable after rollback coordinator failure");
+                    assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
+                }
             }
-            assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt");
-            for dir in dirs.iter().skip(1) {
-                let reopened = reopen_local_disk(dir).await;
-                let restored = reopened
-                    .read_version(
-                        "",
-                        bucket,
-                        object,
-                        "",
-                        &ReadOptions {
-                            read_data: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                    .expect("old version must remain readable after caller cancellation");
-                assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice()));
-            }
-        }
+        })
+        .await;
     }
 
     #[tokio::test]
@@ -11001,9 +11147,35 @@ mod tests {
         let mut file_infos = rename_commit_fileinfos(object, DISKS, "fresh-rollback-etag");
         file_infos[3] = FileInfo::default();
 
-        SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 4)
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        ctx.set_scanner_publication_state(false);
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK);
+        let rename = SetDisks::rename_data_owned_with_fence(
+            &disks,
+            (RUSTFS_META_TMP_BUCKET, "source"),
+            file_infos,
+            (bucket, object),
+            false,
+            RenameDataFenceOptions::new(4, None).with_namespace_commit_guard(Some(ctx.begin_namespace_commit())),
+        );
+        let control = async {
+            barrier.wait_until_paused().await;
+            assert!(ctx.namespace_commits_pending(), "rollback must retain namespace publication ownership");
+            assert!(ctx.scanner_publication_state_allowed(), "rollback must not disable namespace walks");
+            assert_eq!(ctx.namespace_commit_generation(), 1);
+            barrier.release();
+        };
+        let (result, ()) = tokio::time::timeout(BARRIER_PAUSE_GUARD, async { tokio::join!(rename, control) })
             .await
-            .expect_err("three successful disks must fail a strict write quorum of four");
+            .expect("rename rollback must reach its barrier and finish after release");
+        assert_eq!(
+            result.err(),
+            Some(DiskError::ErasureWriteQuorum),
+            "three successful disks must fail a strict write quorum of four"
+        );
+        assert!(!ctx.namespace_commits_pending());
+        assert!(ctx.scanner_publication_state_allowed());
+        assert_eq!(ctx.namespace_commit_generation(), 2);
 
         for (idx, dir) in dirs.iter().enumerate() {
             let reopened = reopen_local_disk(dir).await;

crates/ecstore/src/set_disk/ops/heal.rs

@@ -2490,9 +2490,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
             return Ok((result, err.map(|e| e.into())));
         }
 
-        let disks = self.disks.read().await;
-
-        let disks = disks.clone();
+        // The inner heal and missing-object report read the registry again;
+        // release this snapshot guard before a topology writer can queue between reads.
+        let disks = self.get_disks_internal().await;
         let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
             .await
             .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
@@ -3419,6 +3419,366 @@ mod heal_result_report_tests {
         assert_eq!(unformatted, DiskError::UnformattedDisk);
     }
 
+    #[derive(Clone, Copy)]
+    enum InventoryWriterHealCase {
+        Existing,
+        Missing,
+        MissingVersion,
+    }
+
+    async fn assert_heal_object_inventory_writer(case: InventoryWriterHealCase) {
+        use crate::set_disk::core::io_primitives::disk_call_counters;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "heal-inventory-writer-bucket";
+        let object = match case {
+            InventoryWriterHealCase::Existing => "heal-inventory-writer-existing",
+            InventoryWriterHealCase::Missing => "heal-inventory-writer-missing",
+            InventoryWriterHealCase::MissingVersion => "heal-inventory-writer-missing-version",
+        };
+        set.make_bucket(
+            bucket,
+            &MakeBucketOptions {
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("heal fixture bucket should be created");
+        let body = vec![0x67; 64 * 1024];
+        let stored_version = Uuid::new_v4();
+        let stored_version_string = stored_version.to_string();
+        let published = if matches!(case, InventoryWriterHealCase::Missing) {
+            None
+        } else {
+            let mut reader = PutObjReader::from_vec(body.clone());
+            let info = set
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        no_lock: true,
+                        versioned: true,
+                        version_id: Some(stored_version_string.clone()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("full-fanout PUT should seed the heal fixture");
+            for disk in &disks {
+                let metadata = disk
+                    .read_version("", bucket, object, &stored_version_string, &ReadOptions::default())
+                    .await
+                    .expect("the seeded version must be present on every disk");
+                assert_eq!(metadata.version_id, Some(stored_version));
+                assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
+            }
+            Some(info)
+        };
+        let requested_version = match case {
+            InventoryWriterHealCase::Existing => stored_version_string.clone(),
+            InventoryWriterHealCase::Missing => String::new(),
+            InventoryWriterHealCase::MissingVersion => Uuid::new_v4().to_string(),
+        };
+        let opts = HealOpts {
+            no_lock: true,
+            ..Default::default()
+        };
+        let calls = disk_call_counters::observe(object);
+        let read_gate = set.disks.read().await;
+        // UFCS selects the trait's outer precheck, not the same-named inherent heal.
+        let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
+            set.as_ref(),
+            bucket,
+            object,
+            &requested_version,
+            &opts,
+        );
+        tokio::pin!(heal);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(heal.as_mut())),
+            std::task::Poll::Pending
+        ));
+        // These tests use the current-thread runtime: full-wait metadata tasks
+        // have been spawned, but cannot run during the single unconstrained poll.
+        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
+        let writer = set.disks.write();
+        tokio::pin!(writer);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(writer.as_mut())),
+            std::task::Poll::Pending
+        ));
+        assert!(set.disks.try_read().is_err(), "the writer must already block new inventory readers");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the suspended trait heal must have started the real metadata fanout");
+        for disk_index in 0..4 {
+            assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
+        }
+        drop(read_gate);
+
+        let (_, outcome) =
+            tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(async { drop(writer.await) }, heal) })
+                .await
+                .expect("trait heal must not deadlock its nested inventory read with the queued writer");
+        let (result, error) = outcome.expect("heal should report the object's outcome");
+        match case {
+            InventoryWriterHealCase::Existing => assert!(error.is_none(), "existing object heal failed: {error:?}"),
+            InventoryWriterHealCase::Missing => assert!(matches!(error, Some(Error::FileNotFound))),
+            InventoryWriterHealCase::MissingVersion => assert!(matches!(error, Some(Error::FileVersionNotFound))),
+        }
+        assert_eq!(result.bucket, bucket);
+        assert_eq!(result.object, object);
+        assert_eq!(result.version_id, requested_version);
+        assert_eq!(result.disk_count, 4);
+        assert_eq!(result.before.drives.len(), 4);
+        assert_eq!(result.after.drives.len(), 4);
+        for disk_index in 0..4 {
+            let endpoint = set.set_endpoints[disk_index].to_string();
+            assert_eq!(result.before.drives[disk_index].endpoint, endpoint);
+            assert_eq!(result.after.drives[disk_index].endpoint, endpoint);
+        }
+        if let Some(published) = published {
+            tokio::time::timeout(Duration::from_secs(10), async {
+                let mut reader = set
+                    .get_object_reader(
+                        bucket,
+                        object,
+                        None,
+                        Default::default(),
+                        &ObjectOptions {
+                            versioned: true,
+                            version_id: Some(stored_version_string),
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("the stored version must remain readable after heal");
+                assert_eq!(reader.object_info.etag, published.etag);
+                assert_eq!(reader.object_info.version_id, Some(stored_version));
+                let mut observed_body = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut observed_body)
+                    .await
+                    .expect("stored body should stream");
+                assert_eq!(observed_body, body);
+            })
+            .await
+            .expect("GET must finish after the inventory writer and heal");
+        }
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_existing() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::Existing).await;
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_missing() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::Missing).await;
+    }
+
+    #[tokio::test]
+    async fn heal_object_inventory_writer_missing_version() {
+        assert_heal_object_inventory_writer(InventoryWriterHealCase::MissingVersion).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn heal_object_with_queued_disk_renewal() {
+        use crate::layout::endpoints::SetupType;
+        use crate::runtime::instance::InstanceContext;
+        use crate::set_disk::core::io_primitives::disk_call_counters;
+        use std::collections::HashMap;
+        use std::future::Future;
+        use std::task::Poll;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        // renew_disk still registers local disks on the ambient context. Match
+        // the default serial group used by its other setup/registry fixtures,
+        // and restore only this temporary endpoint, including on a failed join.
+        struct RenewDiskTestState {
+            ctx: Arc<InstanceContext>,
+            was_dist_erasure: bool,
+            map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
+            endpoint: String,
+            previous_disk: Option<Option<DiskStore>>,
+        }
+
+        impl Drop for RenewDiskTestState {
+            fn drop(&mut self) {
+                let ctx = self.ctx.clone();
+                let was_dist_erasure = self.was_dist_erasure;
+                let map = self.map.clone();
+                let endpoint = self.endpoint.clone();
+                let previous_disk = self.previous_disk.take();
+                let handle = tokio::runtime::Handle::current();
+                std::thread::spawn(move || {
+                    handle.block_on(async move {
+                        let mut map = map.write().await;
+                        match previous_disk {
+                            Some(disk) => {
+                                map.insert(endpoint, disk);
+                            }
+                            None => {
+                                map.remove(&endpoint);
+                            }
+                        }
+                        drop(map);
+                        if was_dist_erasure {
+                            ctx.update_erasure_type(SetupType::DistErasure).await;
+                        }
+                    });
+                })
+                .join()
+                .expect("renew fixture state restoration should finish");
+            }
+        }
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let endpoint = set.set_endpoints[0].clone();
+        let ctx = crate::runtime::global::current_ctx();
+        let map = ctx.local_disk_map();
+        let restore = RenewDiskTestState {
+            ctx: ctx.clone(),
+            was_dist_erasure: ctx.is_dist_erasure().await,
+            map: map.clone(),
+            endpoint: endpoint.to_string(),
+            previous_disk: map.read().await.get(&endpoint.to_string()).cloned(),
+        };
+        // Only distributed erasure needs an override to avoid the ambient slot array.
+        if restore.was_dist_erasure {
+            ctx.update_erasure_type(SetupType::Erasure).await;
+        }
+
+        let bucket = "heal-disk-renewal-bucket";
+        let object = "heal-disk-renewal-object";
+        set.make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("renew fixture bucket should be created");
+        let body = vec![0x73; 64 * 1024];
+        let mut reader = PutObjReader::from_vec(body.clone());
+        let published = set
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("full-fanout PUT should seed the renewal fixture");
+        for disk in &disks {
+            let metadata = disk
+                .read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("the seeded object must be present on every disk");
+            assert_eq!(metadata.size, i64::try_from(body.len()).expect("fixture size should fit i64"));
+        }
+
+        let opts = HealOpts {
+            no_lock: true,
+            ..Default::default()
+        };
+        let calls = disk_call_counters::observe(object);
+        let read_gate = set.disks.read().await;
+        let heal = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
+            set.as_ref(),
+            bucket,
+            object,
+            "",
+            &opts,
+        );
+        tokio::pin!(heal);
+        assert!(matches!(futures::poll!(tokio::task::unconstrained(heal.as_mut())), Poll::Pending));
+        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 0);
+
+        let renew = set.renew_disk(&endpoint);
+        tokio::pin!(renew);
+        tokio::time::timeout(
+            Duration::from_secs(5),
+            futures::future::poll_fn(|cx| {
+                assert!(
+                    std::pin::pin!(tokio::task::unconstrained(renew.as_mut()))
+                        .poll(cx)
+                        .is_pending(),
+                    "renewal must reach its inventory write before returning"
+                );
+                if set.disks.try_read().is_err() {
+                    Poll::Ready(())
+                } else {
+                    Poll::Pending
+                }
+            }),
+        )
+        .await
+        .expect("real renewal must queue its topology writer behind the read gate");
+        let registered = map
+            .read()
+            .await
+            .get(&endpoint.to_string())
+            .cloned()
+            .flatten()
+            .expect("renewal must register the connected disk before its inventory write");
+        assert!(!Arc::ptr_eq(&registered, &disks[0]), "renewal must construct a new disk handle");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the suspended trait heal must have started the real metadata fanout");
+        for disk_index in 0..4 {
+            assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, disk_index), 1);
+        }
+        drop(read_gate);
+
+        let (_, outcome) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(renew, heal) })
+            .await
+            .expect("trait heal and real disk renewal must finish without a nested inventory read deadlock");
+        let (report, error) = outcome.expect("heal should report the existing object");
+        assert!(error.is_none(), "existing object heal failed after renewal: {error:?}");
+        assert_eq!(report.bucket, bucket);
+        assert_eq!(report.object, object);
+        assert_eq!(report.disk_count, 4);
+        let renewed = set.get_disks_internal().await[0]
+            .clone()
+            .expect("the renewed slot must remain online");
+        assert!(Arc::ptr_eq(&renewed, &registered), "the set must publish the newly connected handle");
+        assert_eq!(renewed.endpoint(), endpoint);
+        let format = load_format_erasure(&renewed, false)
+            .await
+            .expect("renewed disk format should remain readable");
+        assert_eq!(format.erasure.this, set.format.erasure.sets[0][0]);
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let mut reader = set
+                .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("the object must remain readable after renewal and heal");
+            assert_eq!(reader.object_info.etag, published.etag);
+            let mut observed_body = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut observed_body)
+                .await
+                .expect("stored body should stream");
+            assert_eq!(observed_body, body);
+        })
+        .await
+        .expect("GET must finish after renewal and heal");
+    }
+
     // Regression for #955: an offline disk must contribute exactly one drive
     // record. Before the fix the offline branch fell through and pushed a second
     // (Corrupt) record for the same disk, so `before/after.drives` grew to

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -2452,10 +2452,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let write_quorum = fi.write_quorum(self.default_write_quorum());
         let read_quorum = fi.read_quorum(self.default_read_quorum());
 
-        let disks = self.disks.read().await;
-
-        let disks = disks.clone();
-        // let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
+        // Release the registry guard before recovery and cleanup read it again:
+        // a queued topology writer would otherwise deadlock those nested reads.
+        let disks = self.get_disks_internal().await;
 
         let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
         self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -4051,6 +4050,7 @@ mod tests {
             let _ = drain_global_dirty_scopes();
 
             let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
             let complete_store = Arc::clone(&set_disks);
             let mut complete = tokio::spawn(async move {
                 let mut opts = ObjectOptions::default();
@@ -4062,16 +4062,6 @@ mod tests {
             tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                 .await
                 .expect("multipart completion should pause one tail disk during rename");
-            assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
-                "multipart completion must not publish success while a tail rename is still paused"
-            );
-
-            let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
-            assert!(
-                initial.is_empty(),
-                "capacity must not be marked as committed before the full multipart rename finishes"
-            );
 
             let abort_store = Arc::clone(&set_disks);
             let abort = tokio::spawn(async move {
@@ -4080,21 +4070,46 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(2).await;
-            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
 
-            let retained_staging = futures::future::join_all(
-                disk_stores
-                    .iter()
-                    .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
-            )
+            // A paused rename does not establish that the other disks reached quorum.
+            let retained_staging = tokio::time::timeout(Duration::from_secs(30), async {
+                loop {
+                    let mut retained = 0;
+                    for result in futures::future::join_all(
+                        disk_stores
+                            .iter()
+                            .map(|disk| disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &staged_part)),
+                    )
+                    .await
+                    {
+                        match result {
+                            Ok(_) => retained += 1,
+                            Err(DiskError::FileNotFound) => {}
+                            Err(error) => panic!("staged rename source lookup failed: {error}"),
+                        }
+                    }
+                    if retained <= 1 && rename_tasks.running() == 1 {
+                        break retained;
+                    }
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
             .await
-            .into_iter()
-            .filter(|result| result.is_ok())
-            .count();
+            .expect("unpaused multipart renames should finish before the tail is released");
             assert_eq!(
                 retained_staging, 1,
                 "only the paused tail disk should still retain the multipart rename source"
             );
+            assert!(
+                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
+                "multipart completion must not publish success while a tail rename is still paused"
+            );
+            let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
+            assert!(
+                initial.is_empty(),
+                "capacity must not be marked as committed before the full multipart rename finishes"
+            );
+            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
 
             signaling.set_target(rustfs_lock::ObjectKey::new(bucket, object));
             let object_attempt = signaling.attempts.load(Ordering::Acquire) + 1;
@@ -6743,6 +6758,87 @@ mod tests {
         .await;
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-topology-lock-bucket";
+        let object = "object";
+        let body = vec![0x65; 4096];
+        make_bucket_on_all(&disk_stores, bucket).await;
+        let (upload_id, parts) =
+            stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
+        let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
+        for dir in &temp_dirs {
+            assert!(
+                dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
+                "the test must create real upload staging on every disk"
+            );
+        }
+        let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
+        let complete_store = set_disks.clone();
+        let complete_upload_id = upload_id.clone();
+        let complete = tokio::spawn(async move {
+            complete_store
+                .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                .await
+        });
+        barrier.wait_until_paused().await;
+
+        // Hold a separate read gate so the real writer queues even when completion
+        // correctly releases its snapshot guard. Polling Pending proves admission
+        // to Tokio's write-preferring queue before the cleanup attempts another read.
+        let read_gate = set_disks.disks.read().await;
+        let writer = set_disks.disks.write();
+        tokio::pin!(writer);
+        assert!(matches!(
+            futures::poll!(tokio::task::unconstrained(writer.as_mut())),
+            std::task::Poll::Pending
+        ));
+        assert!(
+            set_disks.disks.try_read().is_err(),
+            "the pending writer must already block new readers before the cleanup resumes"
+        );
+        drop(read_gate);
+        barrier.release();
+
+        let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
+            .await
+            .expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
+        // A reconnect can publish the same handles; this test isolates admission
+        // order without changing the disks that contain the committed object.
+        drop(writer_guard);
+        tokio::time::timeout(Duration::from_secs(10), complete)
+            .await
+            .expect("multipart cleanup must finish after the topology writer releases")
+            .expect("completion task should not panic")
+            .expect("completion should preserve the successful object commit");
+
+        let mut reader = tokio::time::timeout(
+            Duration::from_secs(10),
+            set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
+        )
+        .await
+        .expect("GET should finish after completion")
+        .expect("the completed object should remain readable");
+        let mut observed_body = Vec::new();
+        tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
+            .await
+            .expect("the completed object body should finish streaming")
+            .expect("the completed object body should be readable");
+        assert_eq!(observed_body, body);
+        assert!(matches!(
+            set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
+            Err(StorageError::InvalidUploadID(..))
+        ));
+        for dir in &temp_dirs {
+            assert!(
+                !dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
+                "successful completion must remove its upload staging from every disk"
+            );
+        }
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -4459,7 +4459,10 @@ impl SetDisks {
                         commit_scanner_publication_lease_tokens.as_ref(),
                     )
                     .with_publication_scope(commit_scanner_publication_scope.clone())
-                    .with_rollback_receipt(commit_rollback_receipt.clone()),
+                    .with_rollback_receipt(commit_rollback_receipt.clone())
+                    .with_namespace_commit_guard(
+                        (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
+                    ),
                 )
                 .await;
                 if let Some(scope) = commit_scanner_publication_scope.as_ref() {

crates/ecstore/src/store/bucket.rs

@@ -329,11 +329,11 @@ impl ECStore {
     /// reuse its result, which is sound because bucket deletion/recreation
     /// requires the lifecycle WRITE lock and therefore cannot have run while
     /// any read guard was continuously held.
-    pub(crate) async fn acquire_bucket_incarnation_fence(
+    pub async fn acquire_bucket_incarnation_fence(
         &self,
         bucket: &str,
         expected: uuid::Uuid,
-    ) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
+    ) -> Result<super::BucketIncarnationFenceGuard> {
         let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
         let pieces = super::bucket_fence::FencePieces {
             registry: self.bucket_fence_registry.clone(),
@@ -1059,6 +1059,7 @@ mod tests {
     use crate::storage_api_contracts::{
         bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
         list::ListOperations as _,
+        namespace::NamespaceLocking as _,
         object::{ObjectIO as _, ObjectOperations as _},
     };
     use crate::store::{ECStore, init_local_disks_with_instance_ctx};
@@ -1486,10 +1487,19 @@ mod tests {
             .put_object(bucket, object, &mut reader, &ObjectOptions::default())
             .await
             .expect("object should be written");
+        let lock = ecstore.pools[0].disk_set[0]
+            .new_ns_lock(bucket, object)
+            .await
+            .expect("fixture namespace lock should be created");
+        drop(
+            lock.get_write_lock(Duration::from_secs(30))
+                .await
+                .expect("fixture rename tail should finish before checking its generation"),
+        );
         assert_eq!(
             ecstore.scanner_namespace_mutation_generation(),
-            generation_before_put.saturating_add(1),
-            "successful object creation should advance scanner namespace activity"
+            generation_before_put.saturating_add(3),
+            "successful object creation must observe the logical mutation and both fanout boundaries"
         );
         ecstore
             .get_object_info(bucket, object, &ObjectOptions::default())

crates/ecstore/src/store/bucket_fence.rs

@@ -150,7 +150,7 @@ impl BucketFenceRegistry {
 /// A held bucket lifecycle read lock plus its registration in the fence
 /// registry. Dropping the guard deregisters it; the memo is cleared when the
 /// last guard for the bucket drops (or a lost lock is observed).
-pub(crate) struct BucketIncarnationFenceGuard {
+pub struct BucketIncarnationFenceGuard {
     inner: Option<NamespaceLockGuard>,
     registry: Arc<BucketFenceRegistry>,
     bucket: String,
@@ -158,6 +158,14 @@ pub(crate) struct BucketIncarnationFenceGuard {
 }
 
 impl BucketIncarnationFenceGuard {
+    /// Propagate lifecycle lock loss into the storage commit checks.
+    /// The caller still owns this guard until the complete write tail drains.
+    pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
+        if let Some(guard) = self.namespace_lock_guard() {
+            opts.add_bucket_lifecycle_lock_guard(guard);
+        }
+    }
+
     pub(crate) fn is_lock_lost(&self) -> bool {
         self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
     }
@@ -346,6 +354,36 @@ mod tests {
         first_pieces.abandon("b", first.token);
     }
 
+    #[tokio::test]
+    async fn checkpoint_options_inherit_bucket_fence_lock_loss() {
+        let lock = NamespaceLock::new("bucket-fence-options".to_string(), Arc::new(LocalClient::new()));
+        let inner = lock
+            .acquire_guard(&lock_request("options"))
+            .await
+            .expect("acquire")
+            .expect("quorum");
+        let pieces = FencePieces {
+            registry: Arc::default(),
+            inner,
+        };
+        let registration = pieces.enter("b");
+        let fence = pieces.into_guard("b", registration.token);
+        let mut opts = crate::object_api::ObjectOptions::default();
+        fence.attach_to_object_options(&mut opts);
+        let inherited = opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .expect("checkpoint inherits lifecycle guard");
+        assert!(!inherited.is_lock_lost());
+        tokio::time::timeout(
+            Duration::from_secs(2),
+            fence.namespace_lock_guard().expect("held guard").lock_lost_notified(),
+        )
+        .await
+        .expect("distributed guard expires");
+        assert!(inherited.is_lock_lost(), "the actual pre-rename options must observe lifecycle lock loss");
+    }
+
     #[test]
     fn buckets_are_isolated() {
         let reg = BucketFenceRegistry::default();

crates/ecstore/src/store/init.rs

@@ -353,6 +353,11 @@ async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken)
 }
 
 impl ECStore {
+    /// Shutdown token owned by this store instance.
+    pub fn background_cancel_token(&self) -> Option<CancellationToken> {
+        self.ctx.background_cancel_token()
+    }
+
     /// Validate topology and process storage-class overrides before any disk is opened.
     pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
         let drive_counts = startup_pool_drive_counts(endpoint_pools);
@@ -787,6 +792,12 @@ impl ECStore {
     pub fn single_pool(&self) -> bool {
         self.pools.len() == 1
     }
+
+    /// The set-local create-only check is atomic only when every object
+    /// mutation uses that same, enabled namespace lock domain.
+    pub fn supports_atomic_create_only_write_back(&self) -> bool {
+        !self.ctx.lock_manager().is_disabled() && self.pools.len() == 1 && self.pools[0].disk_set.len() == 1
+    }
 }
 
 #[cfg(test)]
@@ -2127,7 +2138,7 @@ mod tests {
             .iter()
             .map(|&drives_per_set| (1, drives_per_set))
             .collect::<Vec<_>>();
-        build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown).await
+        build_isolated_test_store_with_layout(temp_dir, cmd_line, &pool_layouts, shutdown, None).await
     }
 
     async fn build_isolated_test_store_with_layout(
@@ -2135,6 +2146,7 @@ mod tests {
         cmd_line: &str,
         pool_layouts: &[(usize, usize)],
         shutdown: CancellationToken,
+        instance_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
     ) -> (
         Arc<crate::runtime::instance::InstanceContext>,
         Arc<crate::store::ECStore>,
@@ -2167,7 +2179,7 @@ mod tests {
         let endpoint_pools = EndpointServerPools(pools);
         crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
 
-        let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let instance_ctx = instance_ctx.unwrap_or_else(|| Arc::new(crate::runtime::instance::InstanceContext::new()));
         crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
             .await
             .expect("register local disks into the fresh context");
@@ -2535,6 +2547,348 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial_test::serial(storage_class_env)]
+    async fn early_ack_put_tails_block_scanner_publication_until_all_renames_finish() {
+        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
+
+        let temp_dir = tempfile::tempdir().expect("create scanner PUT tail store dir");
+        let (ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-put-tails", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = format!("scanner-put-tails-{}", Uuid::new_v4());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create scanner PUT tail bucket");
+        let set = &store.pools[0].disk_set[0];
+        let objects = [("scanner-tail-a", vec![0xA1; 273]), ("scanner-tail-b", vec![0xB2; 379])];
+
+        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let (active, blocked, movement_generation) = store.scanner_data_movement_activity().await;
+            assert!(!active && !blocked);
+            assert!(ctx.scanner_publication_state_allowed(), "the set admission cache should start allowed");
+            let (old_lease, _) = store
+                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
+                .await
+                .expect("publication lease should be admitted before either PUT starts");
+
+            let barriers: Vec<_> = objects
+                .iter()
+                .map(|(object, _)| {
+                    crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME)
+                })
+                .collect();
+            let trackers: Vec<_> = objects
+                .iter()
+                .map(|(object, _)| crate::set_disk::rename_fanout_barrier::observe_tasks(object))
+                .collect();
+            let puts: Vec<_> = objects
+                .iter()
+                .map(|(object, body)| {
+                    let put_store = Arc::clone(&store);
+                    let put_bucket = bucket.clone();
+                    let object = *object;
+                    let body = body.clone();
+                    tokio::spawn(async move {
+                        let mut reader = PutObjReader::from_vec(body);
+                        put_store
+                            .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
+                            .await
+                    })
+                })
+                .collect();
+            let committed = tokio::time::timeout(Duration::from_secs(30), async {
+                for barrier in &barriers {
+                    barrier.wait_until_paused().await;
+                }
+                let mut committed = Vec::with_capacity(puts.len());
+                for put in puts {
+                    committed.push(
+                        put.await
+                            .expect("early-ACK PUT task should join while its tail is paused")
+                            .expect("root PUT should return after quorum without waiting for its tail"),
+                    );
+                }
+                committed
+            })
+            .await
+            .expect("both root PUTs must quorum-ACK while their tail disks remain paused");
+
+            assert!(trackers.iter().all(|tracker| tracker.running() >= 1));
+            assert!(ctx.namespace_commits_pending());
+            assert!(
+                ctx.scanner_publication_state_allowed(),
+                "pending PUT tails must not disable scanner namespace walks"
+            );
+            let (active, blocked, observed_movement_generation) = store.scanner_data_movement_activity().await;
+            assert!(!active, "ordinary PUT tails are not decommission or rebalance work");
+            assert!(!blocked, "ordinary PUT tails must not block the movement-only scan baseline");
+            assert_eq!(observed_movement_generation, movement_generation);
+            assert!(store.scanner_data_usage_publication_blocked().await);
+            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
+            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
+            for error in [
+                store
+                    .acquire_scanner_publication_lease(
+                        movement_generation,
+                        crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL,
+                    )
+                    .await
+                    .expect_err("a new remote publication lease must reject pending PUT tails"),
+                store
+                    .validate_scanner_publication_lease(old_lease, movement_generation)
+                    .await
+                    .expect_err("an existing remote lease must not bypass pending PUT tails"),
+                store
+                    .acquire_scanner_publication_lease_guard(old_lease)
+                    .await
+                    .expect_err("target-side publication admission must reject pending PUT tails"),
+            ] {
+                assert!(
+                    error.to_string().contains("blocked"),
+                    "publication must fail because of active tails: {error}"
+                );
+            }
+            store.release_scanner_publication_lease(old_lease).await;
+
+            for (index, barrier) in barriers.iter().enumerate() {
+                let commit_generation = ctx.namespace_commit_generation();
+                let namespace_generation = store.scanner_namespace_mutation_generation();
+                barrier.release();
+                tokio::time::timeout(Duration::from_secs(30), async {
+                    while trackers[index].running() != 0 || ctx.namespace_commit_generation() <= commit_generation {
+                        tokio::task::yield_now().await;
+                    }
+                    if index + 1 == barriers.len() {
+                        while ctx.namespace_commits_pending() {
+                            tokio::task::yield_now().await;
+                        }
+                    }
+                })
+                .await
+                .expect("released tail must drain and publish its terminal namespace generation");
+                assert!(store.scanner_namespace_mutation_generation() > namespace_generation);
+                let pending = index + 1 < barriers.len();
+                assert_eq!(ctx.namespace_commits_pending(), pending);
+                assert_eq!(store.scanner_data_usage_publication_blocked().await, pending);
+                assert!(!store.scanner_data_movement_activity().await.1);
+                assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
+                assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
+            }
+
+            let (lease, generation) = store
+                .acquire_scanner_publication_lease(movement_generation, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
+                .await
+                .expect("remote publication lease should resume after both tails drain");
+            store
+                .validate_scanner_publication_lease(lease, generation)
+                .await
+                .expect("a resumed remote publication lease should validate");
+            drop(
+                store
+                    .acquire_scanner_publication_lease_guard(lease)
+                    .await
+                    .expect("target-side publication admission should resume after both tails drain"),
+            );
+            assert!(store.release_scanner_publication_lease(lease).await);
+
+            let disks = set.disk_inventory().await;
+            assert_eq!(disks.len(), 4);
+            for ((object, body), committed) in objects.iter().zip(&committed) {
+                let logical_size = i64::try_from(body.len()).expect("fixture payload size should fit i64");
+                let etag = committed.etag.as_ref().expect("root PUT should return a committed ETag");
+                for (disk_index, disk) in disks.iter().enumerate() {
+                    let file_info = disk
+                        .as_ref()
+                        .expect("every fixture disk should remain online")
+                        .read_version(
+                            "",
+                            &bucket,
+                            object,
+                            "",
+                            &crate::disk::ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .unwrap_or_else(|err| panic!("disk {disk_index} should publish {object} after its tail finishes: {err}"));
+                    assert_eq!(file_info.size, logical_size);
+                    assert_eq!(file_info.metadata.get(http::header::ETAG.as_str()), Some(etag));
+                    assert!(
+                        file_info.inline_data(),
+                        "small fixture payloads should have an inline shard on every disk"
+                    );
+                    let inline_data = file_info.data.as_ref().expect("every disk should retain its inline shard");
+                    let erasure = crate::erasure::coding::Erasure::try_new_with_options(
+                        file_info.erasure.data_blocks,
+                        file_info.erasure.parity_blocks,
+                        file_info.erasure.block_size,
+                        file_info.uses_legacy_checksum,
+                    )
+                    .expect("persisted erasure geometry should be valid");
+                    let shard_size =
+                        usize::try_from(erasure.shard_file_size(logical_size)).expect("fixture shard size should fit usize");
+                    crate::erasure::coding::bitrot_verify(
+                        Cursor::new(inline_data.clone()),
+                        inline_data.len(),
+                        shard_size,
+                        rustfs_utils::HashAlgorithm::HighwayHash256S,
+                        erasure.shard_size(),
+                    )
+                    .await
+                    .unwrap_or_else(|err| panic!("disk {disk_index} should retain a complete valid shard for {object}: {err}"));
+                }
+                let mut reader = store
+                    .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("fully drained PUT should be readable");
+                let mut actual = Vec::new();
+                reader.stream.read_to_end(&mut actual).await.expect("PUT body should drain");
+                assert_eq!(&actual, body);
+            }
+
+            let generation_before_internal_put = ctx.namespace_commit_generation();
+            let internal_object = "scanner-tail-regression/internal-metadata";
+            let internal_body = b"scanner metadata must not invalidate its own publication";
+            let mut internal_reader = PutObjReader::from_vec(internal_body.to_vec());
+            store
+                .put_object(RUSTFS_META_BUCKET, internal_object, &mut internal_reader, &ObjectOptions::default())
+                .await
+                .expect("internal metadata PUT should commit without scanner self-invalidation");
+            let internal_lock = set
+                .new_ns_lock(RUSTFS_META_BUCKET, internal_object)
+                .await
+                .expect("internal metadata tail lock should be available");
+            drop(
+                internal_lock
+                    .get_write_lock(Duration::from_secs(30))
+                    .await
+                    .expect("internal metadata tail should drain"),
+            );
+            assert_eq!(ctx.namespace_commit_generation(), generation_before_internal_put);
+            assert!(!ctx.namespace_commits_pending());
+            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
+            assert!(set.scanner_data_usage_publication_admission_guard().await.is_some());
+            let mut internal_reader = store
+                .get_object_reader(RUSTFS_META_BUCKET, internal_object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("internal metadata should remain readable");
+            let mut actual = Vec::new();
+            internal_reader
+                .stream
+                .read_to_end(&mut actual)
+                .await
+                .expect("internal metadata body should drain");
+            assert_eq!(actual, internal_body);
+        })
+        .await;
+        shutdown.cancel();
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial_test::serial(storage_class_env)]
+    async fn cancelled_early_ack_put_keeps_scanner_publication_blocked_until_tail_finishes() {
+        let temp_dir = tempfile::tempdir().expect("create cancelled scanner PUT tail store dir");
+        let (ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "scanner-cancelled-put-tail", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = format!("scanner-cancelled-put-tail-{}", Uuid::new_v4());
+        let object = "scanner-cancelled-tail";
+        let body = vec![0xC3; 273];
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create cancelled scanner PUT tail bucket");
+
+        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let tracker = crate::set_disk::rename_fanout_barrier::observe_tasks(object);
+            let tail =
+                crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME);
+            let quorum = crate::set_disk::PutObjectCommitBarrier::install(
+                &bucket,
+                object,
+                crate::set_disk::PutObjectCommitPause::AfterRenameQuorum,
+            );
+            let handoff = crate::set_disk::PutObjectCommitBarrier::install(
+                &bucket,
+                object,
+                crate::set_disk::PutObjectCommitPause::AfterRenameHandoff,
+            );
+            let put_store = Arc::clone(&store);
+            let put_bucket = bucket.clone();
+            let put_body = body.clone();
+            let put = tokio::spawn(async move {
+                let mut reader = PutObjReader::from_vec(put_body);
+                put_store
+                    .put_object(&put_bucket, object, &mut reader, &ObjectOptions::default())
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), tail.wait_until_paused())
+                .await
+                .expect("cancelled PUT should pause one disk before rename");
+            quorum.wait_until_paused().await;
+            put.abort();
+            assert!(
+                put.await
+                    .expect_err("caller should be cancelled after rename quorum")
+                    .is_cancelled()
+            );
+            quorum.release();
+            handoff.wait_until_paused().await;
+            assert!(tracker.running() >= 1);
+            assert!(ctx.namespace_commits_pending());
+            assert!(!store.scanner_data_movement_activity().await.1);
+            assert!(store.scanner_data_usage_publication_blocked().await);
+            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
+            assert!(
+                store.pools[0].disk_set[0]
+                    .scanner_data_usage_publication_admission_guard()
+                    .await
+                    .is_some()
+            );
+            let generation = store.scanner_namespace_mutation_generation();
+
+            handoff.release();
+            tail.release();
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while tracker.running() != 0 || ctx.namespace_commits_pending() {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("cancelled request's detached fanout must release scanner admission after finishing");
+            assert!(store.scanner_namespace_mutation_generation() > generation);
+            assert!(!store.scanner_data_usage_publication_blocked().await);
+            assert!(store.scanner_data_usage_publication_admission_guard().await.is_some());
+            for (disk_index, disk) in store.pools[0].disk_set[0].disk_inventory().await.iter().enumerate() {
+                let file_info = disk
+                    .as_ref()
+                    .expect("cancelled PUT fixture disk should remain online")
+                    .read_version("", &bucket, object, "", &crate::disk::ReadOptions::default())
+                    .await
+                    .unwrap_or_else(|err| panic!("cancelled PUT must still publish on disk {disk_index}: {err}"));
+                assert_eq!(file_info.size, i64::try_from(body.len()).expect("fixture body size should fit i64"));
+            }
+            let mut reader = store
+                .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("a cancelled caller must not discard its quorum-committed object");
+            let mut actual = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut actual)
+                .await
+                .expect("cancelled PUT body should drain");
+            assert_eq!(actual, body);
+        })
+        .await;
+        shutdown.cancel();
+    }
+
     #[cfg(feature = "test-util")]
     #[test]
     #[serial_test::serial(storage_class_env)]
@@ -2986,8 +3340,9 @@ mod tests {
     ) -> crate::core::pools::DecommissionTestFaultDecision {
         let target_bucket = bucket.to_string();
         let target_object = object.to_string();
-        Arc::new(move |stage, bucket, object, _attempt, succeeded| {
+        Arc::new(move |stage, bucket, object, attempt, succeeded| {
             if !succeeded
+                || attempt >= crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS
                 || stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
                 || bucket != target_bucket
                 || object != target_object
@@ -2997,6 +3352,7 @@ mod tests {
 
             // Entry retries reset the local attempt; real copy errors can skip
             // successful attempts. Only injected faults spend this global budget.
+            // A real failure may consume an attempt, so preserve the final chance.
             faults
                 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
                     (faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
@@ -5018,6 +5374,7 @@ mod tests {
             "decommission-delete-fence",
             &[(2, 4), (1, 4)],
             CancellationToken::new(),
+            None,
         ))
         .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -5149,7 +5506,15 @@ mod tests {
 
     #[test]
     fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
-        for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
+        let cases: &[&[(usize, bool, bool)]] = &[
+            &[(1, true, true), (2, true, true), (3, true, false)],
+            &[(1, true, true), (1, true, true), (2, true, false)],
+            &[(1, true, true), (3, true, false), (3, true, false)],
+            &[(1, true, true), (2, false, false), (1, true, true), (2, true, false)],
+            &[(1, true, true), (2, false, false), (3, true, false)],
+            &[(3, true, false), (4, true, false)],
+        ];
+        for case in cases {
             let faults = Arc::new(AtomicUsize::new(0));
             let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
 
@@ -5163,14 +5528,16 @@ mod tests {
             }
             assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
 
-            for (index, attempt) in attempts.into_iter().enumerate() {
+            let mut expected_faults = 0;
+            for &(attempt, succeeded, expected) in *case {
                 assert_eq!(
-                    hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
-                    index < 2,
-                    "attempts={attempts:?}, index={index}"
+                    hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, succeeded),
+                    expected,
+                    "fault plan {case:?} at attempt {attempt}"
                 );
+                expected_faults += usize::from(expected);
+                assert_eq!(faults.load(Ordering::SeqCst), expected_faults);
             }
-            assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
         }
     }
 
@@ -5306,6 +5673,15 @@ mod tests {
                     changed_result.expect("SourceChanged entry retry must converge");
                     other_result.expect("other bucket entry must continue through ordinary copy retries");
 
+                    assert_eq!(
+                        store.pool_meta.read().await.pools[0]
+                            .decommission
+                            .as_ref()
+                            .expect("decommission progress should remain available")
+                            .items_decommission_failed,
+                        0,
+                        "entry completion must not hide an exhausted copy failure"
+                    );
                     assert!(!rx.is_cancelled(), "entry-level SourceChanged must not cancel the shared worker token");
                     assert_eq!(mutation_calls.load(Ordering::SeqCst), 2, "entry must be re-listed after SourceChanged");
                     assert_eq!(ordinary_faults.load(Ordering::SeqCst), 2, "ordinary copy must consume the retry budget");
@@ -5934,6 +6310,7 @@ mod tests {
             "reverse-decommission-fixed-target",
             &[(1, 4), (1, 4)],
             CancellationToken::new(),
+            None,
         ))
         .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -6355,6 +6732,7 @@ mod tests {
             "multi-set-decommission-source-cleanup",
             &[(2, 4)],
             CancellationToken::new(),
+            None,
         ))
         .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -8870,18 +9248,17 @@ mod tests {
         const MANIFEST_COUNT: usize = 10;
 
         let temp_dir = tempfile::tempdir().expect("create fast manifest pass recovery store dir");
-        let (ctx, store, _shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-fast-manifest-pass", &[4])).await;
+        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
+        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
+        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
+            temp_dir.path(),
+            "tier-delete-fast-manifest-pass",
+            &[(1, 4)],
+            CancellationToken::new(),
+            Some(Arc::new(instance_ctx)),
+        ))
+        .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let bucket = "tier-delete-fast-manifest-pass-bucket";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("fast manifest pass bucket should be created");
-        let incarnation = store
-            .bucket_incarnation_id(bucket)
-            .await
-            .expect("fast manifest pass bucket incarnation should resolve");
         let tier_name = "FAST-MANIFEST-PASS";
         let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
         let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
@@ -8889,9 +9266,19 @@ mod tests {
             .expect("fast manifest pass tier lease should resolve")
             .backend_identity();
         for index in 0..MANIFEST_COUNT {
+            // Pagination must not depend on same-bucket lock wait deadlines.
+            let bucket = format!("tier-delete-fast-manifest-pass-{index}");
+            store
+                .make_bucket(&bucket, &MakeBucketOptions::default())
+                .await
+                .expect("fast manifest pass bucket should be created");
+            let incarnation = store
+                .bucket_incarnation_id(&bucket)
+                .await
+                .expect("fast manifest pass bucket incarnation should resolve");
             install_aborting_dispatch_fixture(
                 store.clone(),
-                bucket,
+                &bucket,
                 incarnation,
                 &format!("manifest-page-{index:06}/"),
                 tier_name,
@@ -8922,12 +9309,78 @@ mod tests {
             "one production pass must cross the default eight-manifest page limit"
         );
         assert_eq!(stats.manifests.scanned, MANIFEST_COUNT);
-        assert_eq!(stats.manifests.deleted, MANIFEST_COUNT);
-        assert_eq!(stats.manifests.failed, 0);
+        assert_eq!(stats.manifests.deleted, MANIFEST_COUNT, "full recovery result: {stats:?}");
+        assert_eq!(stats.manifests.failed, 0, "full recovery result: {stats:?}");
         assert_eq!(manifest_marker, None);
         assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
         assert_eq!(tier_delete_journal_count(store).await, 0);
         assert_eq!(backend.remove_count().await, 0, "rollback recovery must not call the remote tier");
+        shutdown.cancel();
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn tier_delete_manual_pass_retains_manifest_owned_by_startup_recovery() {
+        let temp_dir = tempfile::tempdir().expect("create automatic recovery ownership store dir");
+        let (ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-delete-auto-owner", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "tier-delete-auto-owner-bucket";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("automatic recovery bucket should be created");
+        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+        let tier_name = "AUTO-OWNER";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("automatic recovery tier lease")
+            .backend_identity();
+
+        // The automatic worker must not observe a partially installed fixture.
+        let lifecycle_guard = store
+            .acquire_bucket_lifecycle_write_lock(bucket)
+            .await
+            .expect("fixture lifecycle lock");
+        let (manifest_name, entries) =
+            install_aborting_dispatch_fixture(store.clone(), bucket, incarnation, "auto-owner/", tier_name, identity, 1).await;
+        let journal_name = tier_delete_journal_object_name(&entries[0]);
+        let hook = TierDeleteDispatchRollbackTestHook::install_slow_delete(&journal_name, &journal_name);
+        drop(lifecycle_guard);
+        ctx.wake_tier_delete_journal_recovery();
+        tokio::time::timeout(Duration::from_secs(30), hook.wait_until_delete_paused())
+            .await
+            .expect("startup recovery should own the manifest before a manual pass");
+        assert!(tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name));
+
+        let stats = recover_tier_delete_dispatch_manifests(store.clone(), 8, None)
+            .await
+            .expect("manual recovery scan");
+        assert_eq!(stats.scanned, 1, "{stats:?}");
+        assert_eq!(stats.retained, 1, "{stats:?}");
+        assert_eq!(stats.deleted, 0, "{stats:?}");
+        assert_eq!(stats.failed, 0, "{stats:?}");
+        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 1);
+        assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
+
+        hook.release_delete();
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let manifest_gone = matches!(com::read_config(store.clone(), &manifest_name).await, Err(Error::ConfigNotFound));
+                if manifest_gone && !tier_delete_dispatch_manifest_recovery_inflight_for_test(&store, &manifest_name) {
+                    break;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        .expect("automatic recovery should converge without a manual retry");
+        assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
+        assert_eq!(tier_delete_journal_count(store).await, 0);
+        assert_eq!(backend.remove_count().await, 0, "rollback must not delete from the remote tier");
+        shutdown.cancel();
     }
 
     #[cfg(feature = "test-util")]
@@ -10302,8 +10755,17 @@ mod tests {
         const JOURNAL_COUNT: usize = 40;
 
         let temp_dir = tempfile::tempdir().expect("create rollback retry store dir");
-        let (ctx, store, _shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "dispatch-rollback-retry", &[4])).await;
+        // Manual retries must own progress between fault removal and the next attempt.
+        let mut instance_ctx = crate::runtime::instance::InstanceContext::new();
+        instance_ctx.suppress_tier_delete_journal_recovery_for_test();
+        let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout(
+            temp_dir.path(),
+            "dispatch-rollback-retry",
+            &[(1, 4)],
+            CancellationToken::new(),
+            Some(Arc::new(instance_ctx)),
+        ))
+        .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
         let bucket = "dispatch-rollback-retry-bucket";
         store
@@ -10377,6 +10839,7 @@ mod tests {
 
         assert_eq!(tier_delete_dispatch_manifest_count(store.clone()).await, 0);
         assert_eq!(backend.remove_count().await, 0, "rollback retries must never call the remote tier");
+        shutdown.cancel();
     }
 
     #[cfg(feature = "test-util")]
@@ -13204,6 +13667,7 @@ mod tests {
             "partial-set-prefix-delete",
             &[(2, 4)],
             CancellationToken::new(),
+            None,
         ))
         .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -16576,6 +17040,7 @@ mod tests {
             "prepared-directory-recovery",
             &[(2, 4)],
             shutdown,
+            None,
         ))
         .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
@@ -17228,6 +17693,38 @@ mod tests {
             .expect("test thread should complete");
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn odm_write_back_requires_one_set_and_enabled_namespace_locking() {
+        for (layout, locking, supported) in [
+            (&[(1, 4)][..], true, true),
+            (&[(1, 4), (1, 4)][..], true, false),
+            (&[(2, 4)][..], true, false),
+            (&[(1, 4)][..], false, false),
+        ] {
+            temp_env::async_with_vars([("RUSTFS_LOCK_ENABLED", Some(if locking { "true" } else { "false" }))], async {
+                let dir = tempfile::tempdir().expect("isolated topology");
+                let shutdown = CancellationToken::new();
+                let (_ctx, store, _) = without_storage_class_env(build_isolated_test_store_with_layout(
+                    dir.path(),
+                    "odm-topology",
+                    layout,
+                    shutdown.clone(),
+                    None,
+                ))
+                .await;
+                assert_eq!(
+                    store.supports_atomic_create_only_write_back(),
+                    supported,
+                    "layout={layout:?}, locking={locking}"
+                );
+                shutdown.cancel();
+            })
+            .await;
+        }
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]

crates/ecstore/src/store/mod.rs

@@ -417,6 +417,7 @@ const MAX_UPLOADS_LIST: usize = 10000;
 mod bucket;
 mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
+pub use bucket_fence::BucketIncarnationFenceGuard;
 mod heal;
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
@@ -848,7 +849,7 @@ impl ECStore {
     }
 
     pub fn scanner_namespace_mutation_generation(&self) -> u64 {
-        list_objects::scanner_namespace_mutation_generation()
+        list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
     }
 
     pub async fn scanner_data_movement_active(&self) -> bool {
@@ -857,7 +858,7 @@ impl ECStore {
     }
 
     /// Return the storage-owned movement state and generation as one
-    /// authenticated activity snapshot.  The read lock is acquired before
+    /// authenticated activity snapshot. The read lock is acquired before
     /// the state locks (cancelers, pool metadata, then rebalance metadata),
     /// matching the transition writer order and preventing a terminal state
     /// from being reported with the preceding generation.
@@ -886,11 +887,12 @@ impl ECStore {
     /// Returns whether scanner metadata may still be hidden by a local
     /// data-movement state. Terminal failed/canceled decommission entries
     /// remain suspended until an operator clears or retries them, so they are
-    /// a publication barrier even after the worker has stopped.
+    /// a publication barrier even after the worker has stopped. Active PUT
+    /// rename fanouts also defer publication, including post-ACK tails.
     pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
         let operation_gate = self.ctx.data_movement_operation_gate();
         let _operation_guard = operation_gate.read_owned().await;
-        self.scanner_data_usage_publication_snapshot_blocked().await
+        self.scanner_data_usage_publication_snapshot_blocked().await || self.ctx.namespace_commits_pending()
     }
 
     pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
@@ -1070,7 +1072,7 @@ impl ECStore {
         {
             return Err(Error::other("scanner publication lease generation is stale"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
 
@@ -1109,7 +1111,7 @@ impl ECStore {
         {
             return Err(Error::other("scanner publication lease generation is stale"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
         if !self.ctx.scanner_publication_lease_is_active(token).await {
@@ -1129,7 +1131,7 @@ impl ECStore {
         if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
             return Err(Error::other("scanner publication lease generation is exhausted"));
         }
-        if self.scanner_data_movement_snapshot_locked().await.1 {
+        if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
             return Err(Error::other("scanner publication lease is blocked by data movement"));
         }
         let Some(lease_generation) = self.ctx.scanner_publication_lease_generation(token).await else {

crates/heal/src/heal/mrf_queue.rs

@@ -45,6 +45,10 @@ use uuid::Uuid;
 
 use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
 
+/// Read-only inspection of committed MRF checkpoints. The legacy consumer
+/// remains unchanged until ownership-aware replay is deployed.
+pub mod snapshot;
+
 /// Journal location inside the metadata bucket, following the resume-state
 /// layout.
 pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";

crates/heal/src/heal/mrf_queue/snapshot.rs

@@ -0,0 +1,681 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Reader-first support for owner-local MRF checkpoints.
+//!
+//! Each of two slots has a payload and a commit manifest. The manifest binds
+//! the writer identity, persistent sequence, length and whole-payload digest.
+//! Replacing the inactive slot must leave the previous committed slot intact.
+//! Production publication and reclamation are deliberately not enabled here.
+//! An unreadable commit path cannot prove that only legacy data exists. This
+//! explicit inspection API fails closed and never mutates recovery anchors.
+//! It is not wired into the legacy consumer: that transition requires the
+//! ownership-aware replay and producer handoff before writer activation.
+//! One surviving committed replica supports process restart recovery only;
+//! this reader does not establish a replication quorum or a power-loss policy.
+
+use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal};
+use crate::heal::RUSTFS_META_BUCKET;
+use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore};
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+use tokio::io::AsyncReadExt;
+use uuid::Uuid;
+
+// Root-level control files avoid requiring a new directory before the first
+// atomic commit. They remain inside the storage owner's metadata volume.
+const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
+const MANIFEST_PATHS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
+const MAGIC: &[u8; 8] = b"RFMRFC01";
+const MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
+const VERSION: u8 = 1;
+
+#[derive(Debug, thiserror::Error)]
+pub enum SnapshotError {
+    #[error("MRF checkpoint has an invalid or incomplete commit record")]
+    Corrupt,
+    #[error("MRF checkpoint format is unsupported")]
+    Unsupported,
+    #[error("MRF checkpoint exceeds the configured byte limit")]
+    TooLarge,
+    #[error("MRF checkpoint replicas disagree at the same sequence")]
+    Conflict,
+    #[error("MRF checkpoint storage is unavailable")]
+    Disk(#[source] EcstoreDiskError),
+    #[error("MRF checkpoint body could not be read")]
+    Read(#[source] std::io::Error),
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct Manifest {
+    owner: Uuid,
+    sequence: u64,
+    payload_len: usize,
+    payload_digest: [u8; 32],
+}
+
+impl Manifest {
+    fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
+        if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
+            return Err(SnapshotError::Corrupt);
+        }
+        if bytes[8] != VERSION {
+            return Err(SnapshotError::Unsupported);
+        }
+        let signed = MANIFEST_LEN - 32;
+        let checksum: [u8; 32] = Sha256::digest(&bytes[..signed]).into();
+        if checksum != bytes[signed..] {
+            return Err(SnapshotError::Corrupt);
+        }
+        let owner = Uuid::from_slice(&bytes[9..25]).map_err(|_| SnapshotError::Corrupt)?;
+        let sequence = u64::from_le_bytes(bytes[25..33].try_into().map_err(|_| SnapshotError::Corrupt)?);
+        let payload_len = u64::from_le_bytes(bytes[33..41].try_into().map_err(|_| SnapshotError::Corrupt)?);
+        let payload_len = usize::try_from(payload_len).map_err(|_| SnapshotError::TooLarge)?;
+        if owner.is_nil() || sequence == 0 || sequence == u64::MAX {
+            return Err(SnapshotError::Corrupt);
+        }
+        if payload_len > limit {
+            return Err(SnapshotError::TooLarge);
+        }
+        Ok(Self {
+            owner,
+            sequence,
+            payload_len,
+            payload_digest: bytes[41..73].try_into().map_err(|_| SnapshotError::Corrupt)?,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct CommittedSnapshot {
+    manifest: Manifest,
+    payload: Vec<u8>,
+}
+
+impl CommittedSnapshot {
+    /// Persistent single-writer sequence, not a process UUID ordering.
+    pub fn sequence(&self) -> u64 {
+        self.manifest.sequence
+    }
+
+    /// Identity recorded by the committed checkpoint's writer.
+    pub fn owner(&self) -> Uuid {
+        self.manifest.owner
+    }
+
+    /// Complete, checksum-validated record bytes. Inspection does not consume
+    /// these records or acknowledge completion to any producer.
+    pub fn payload(&self) -> &[u8] {
+        &self.payload
+    }
+
+    fn decode(manifest: &[u8], payload: Vec<u8>, limit: usize) -> Result<Self, SnapshotError> {
+        let manifest = Manifest::decode(manifest, limit)?;
+        let checksum: [u8; 32] = Sha256::digest(&payload).into();
+        if payload.len() != manifest.payload_len || checksum != manifest.payload_digest {
+            return Err(SnapshotError::Corrupt);
+        }
+        if decode_journal(&payload).1 != 0 {
+            return Err(SnapshotError::Corrupt);
+        }
+        Ok(Self { manifest, payload })
+    }
+}
+
+#[derive(Debug)]
+pub enum RecoverySnapshot {
+    /// An intact legacy snapshot, without a comparable commit sequence.
+    Legacy(Vec<u8>),
+    /// A committed checkpoint requiring ownership-aware replay before use.
+    Committed(CommittedSnapshot),
+}
+
+async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
+    let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
+        Ok(reader) => reader,
+        Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
+        Err(error) => return Err(SnapshotError::Disk(error)),
+    };
+    let maximum = limit.checked_add(1).ok_or(SnapshotError::TooLarge)?;
+    let maximum = u64::try_from(maximum).map_err(|_| SnapshotError::TooLarge)?;
+    let mut bytes = Vec::new();
+    reader
+        .take(maximum)
+        .read_to_end(&mut bytes)
+        .await
+        .map_err(SnapshotError::Read)?;
+    if bytes.len() > limit {
+        return Err(SnapshotError::TooLarge);
+    }
+    Ok(Some(bytes))
+}
+
+fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: CommittedSnapshot) -> Result<(), SnapshotError> {
+    if let Some(current) = selected {
+        if current.manifest.sequence == candidate.manifest.sequence
+            && (current.manifest != candidate.manifest || current.payload != candidate.payload)
+        {
+            return Err(SnapshotError::Conflict);
+        }
+        if current.manifest.sequence >= candidate.manifest.sequence {
+            return Ok(());
+        }
+    }
+    *selected = Some(candidate);
+    Ok(())
+}
+
+async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
+    let mut selected = None;
+    let mut damaged = None;
+    let mut identities = HashMap::new();
+    for disk in disks {
+        for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
+            let candidate = async {
+                let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
+                    return Ok(None);
+                };
+                let header = Manifest::decode(&manifest, limit)?;
+                let payload = read_bounded(disk, payload_path, header.payload_len)
+                    .await?
+                    .ok_or(SnapshotError::Corrupt)?;
+                CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
+            }
+            .await;
+            match candidate {
+                Ok(Some(candidate)) => {
+                    let identity = (
+                        candidate.manifest.owner,
+                        candidate.manifest.payload_len,
+                        candidate.manifest.payload_digest,
+                    );
+                    if identities
+                        .insert(candidate.manifest.sequence, identity)
+                        .is_some_and(|previous| previous != identity)
+                    {
+                        return Err(SnapshotError::Conflict);
+                    }
+                    select_snapshot(&mut selected, candidate)?;
+                }
+                Ok(None) => {}
+                // A future committed format may supersede all readable slots.
+                Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported),
+                Err(error) => damaged = Some(error),
+            }
+        }
+    }
+    match (selected, damaged) {
+        (Some(snapshot), _) => Ok(Some(snapshot)),
+        (None, Some(error)) => Err(error),
+        (None, None) => Ok(None),
+    }
+}
+
+async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
+    let mut selected = None;
+    let mut incomplete: Option<Vec<u8>> = None;
+    for disk in disks {
+        match read_bounded(disk, path, limit).await {
+            Ok(Some(payload)) if decode_journal(&payload).1 == 0 => {
+                if selected.as_ref().is_some_and(|current| *current != payload) {
+                    // Legacy snapshots have no sequence. There is no evidence
+                    // that the first, longest or nonempty replica is newest.
+                    return Err(SnapshotError::Conflict);
+                }
+                selected = Some(payload);
+            }
+            Ok(Some(payload)) => {
+                if let Some(previous) = &incomplete {
+                    if previous.starts_with(&payload) {
+                        continue;
+                    }
+                    if !payload.starts_with(previous) {
+                        return Err(SnapshotError::Corrupt);
+                    }
+                }
+                incomplete = Some(payload);
+            }
+            Ok(None) => {}
+            Err(error) => return Err(error),
+        }
+    }
+    if let Some(prefix) = incomplete
+        && !selected.as_ref().is_some_and(|payload| payload.starts_with(&prefix))
+    {
+        // In particular, an empty O_TRUNC replica cannot supersede another
+        // replica containing intact records followed by a torn tail.
+        return Err(SnapshotError::Corrupt);
+    }
+    Ok(selected)
+}
+
+/// Inspect local MRF checkpoints without replaying, acknowledging or deleting.
+///
+/// `max_bytes` bounds each payload read. Every local replica is examined and
+/// ambiguous identities, unavailable proof or unsupported formats return a
+/// typed error. This API must not authorize a writer without the separate
+/// ownership and mixed-version activation checks.
+pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
+    read_recovery_snapshot(&super::journal_disks().await, max_bytes).await
+}
+
+async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<RecoverySnapshot>, SnapshotError> {
+    if let Some(snapshot) = read_committed(disks, limit).await? {
+        return Ok(Some(RecoverySnapshot::Committed(snapshot)));
+    }
+    // RUSTFS_COMPAT_TODO(backlog-2263): inspect retained legacy MRF journals. Remove after all supported upgrade and rollback readers understand committed snapshots and retained journals have migrated.
+    if let Some(payload) = read_legacy(disks, MRF_SCOPED_JOURNAL_PATH, limit).await? {
+        return Ok(Some(RecoverySnapshot::Legacy(payload)));
+    }
+    Ok(read_legacy(disks, MRF_JOURNAL_PATH, limit)
+        .await?
+        .map(RecoverySnapshot::Legacy))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::heal::mrf_queue::encode_intent;
+    use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
+    use crate::heal::{DiskOption, Endpoint, new_disk};
+    use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
+    use std::sync::Arc;
+    use tempfile::TempDir;
+
+    fn payload(object: &str) -> Vec<u8> {
+        let intent = MrfIntent {
+            bucket: Arc::from("bucket"),
+            object: Arc::from(object),
+            version_id: None,
+            kind: MrfKind::PartialWrite,
+            scope: None,
+            lease: None,
+            enqueued_at_ms: 1234,
+            attempts: 0,
+        };
+        let mut bytes = Vec::new();
+        assert!(encode_intent(&intent, &mut bytes), "fixture must encode a full record");
+        bytes
+    }
+
+    fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
+        let mut bytes = Vec::with_capacity(MANIFEST_LEN);
+        bytes.extend_from_slice(MAGIC);
+        bytes.push(VERSION);
+        bytes.extend_from_slice(owner.as_bytes());
+        bytes.extend_from_slice(&sequence.to_le_bytes());
+        bytes.extend_from_slice(&u64::try_from(payload.len()).expect("fixture length fits").to_le_bytes());
+        bytes.extend_from_slice(&Sha256::digest(payload));
+        bytes.extend_from_slice(&Sha256::digest(&bytes));
+        bytes
+    }
+
+    async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore {
+        let path = root.path().join(name);
+        std::fs::create_dir_all(&path).expect("create disk directory");
+        let endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("valid disk endpoint");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("open disk");
+        let result = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await;
+        assert!(
+            matches!(result, Ok(()) | Err(EcstoreDiskError::VolumeExists)),
+            "metadata volume: {result:?}"
+        );
+        disk
+    }
+
+    // Exercise the existing storage owner's atomic CAS primitive. No production
+    // caller publishes this format until ownership-aware replay is available.
+    async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8]) {
+        let expected = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await.ok();
+        let result = EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            path,
+            expected,
+            Some(EcstoreDiskBytes::copy_from_slice(bytes)),
+        )
+        .await
+        .expect("atomic snapshot slot write");
+        assert_eq!(result, EcstoreConditionalFileUpdate::Updated);
+    }
+
+    async fn commit(disk: &EcstoreDiskStore, slot: usize, owner: Uuid, sequence: u64, bytes: &[u8]) {
+        install(disk, PAYLOAD_PATHS[slot], bytes).await;
+        install(disk, MANIFEST_PATHS[slot], &manifest(owner, sequence, bytes)).await;
+    }
+
+    #[test]
+    fn manifest_validates_identity_sequence_length_and_digest() {
+        let bytes = payload("object");
+        let owner = Uuid::new_v4();
+        assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok());
+        for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] {
+            assert!(matches!(
+                Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()),
+                Err(SnapshotError::Corrupt)
+            ));
+        }
+        assert!(matches!(
+            Manifest::decode(&manifest(owner, 1, &bytes), bytes.len() - 1),
+            Err(SnapshotError::TooLarge)
+        ));
+        let mut corrupt = manifest(owner, 1, &bytes);
+        corrupt[25] ^= 1;
+        assert!(matches!(Manifest::decode(&corrupt, bytes.len()), Err(SnapshotError::Corrupt)));
+        let mut unsupported = manifest(owner, 1, &bytes);
+        unsupported[8] = 2;
+        assert!(matches!(Manifest::decode(&unsupported, bytes.len()), Err(SnapshotError::Unsupported)));
+    }
+
+    #[test]
+    fn whole_payload_integrity_is_required_even_with_a_valid_manifest() {
+        let bytes = payload("object");
+        let owner = Uuid::new_v4();
+        let header = manifest(owner, 1, &bytes);
+        assert!(matches!(
+            CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()),
+            Err(SnapshotError::Corrupt)
+        ));
+        let invalid = b"not an MRF record".to_vec();
+        assert!(matches!(
+            CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()),
+            Err(SnapshotError::Corrupt)
+        ));
+    }
+
+    #[tokio::test]
+    async fn newest_complete_replica_wins_in_both_disk_orders() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 1, &payload("old")).await;
+        commit(&second, 1, owner, 2, &payload("new")).await;
+        for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+            let recovered = read_committed(&disks, 4096)
+                .await
+                .expect("read replicas")
+                .expect("committed snapshot");
+            assert_eq!(recovered.manifest.sequence, 2);
+            assert_eq!(recovered.payload, payload("new"));
+        }
+    }
+
+    #[tokio::test]
+    async fn divergent_commits_at_same_sequence_fail_closed() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 7, &payload("a")).await;
+        commit(&second, 1, owner, 7, &payload("b")).await;
+        assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn newer_slot_does_not_hide_a_conflicting_commit_history() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let owner = Uuid::new_v4();
+        commit(&first, 0, owner, 8, &payload("newest")).await;
+        commit(&first, 1, owner, 7, &payload("a")).await;
+        commit(&second, 1, owner, 7, &payload("b")).await;
+        assert!(matches!(read_committed(&[first, second], 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn uncommitted_or_torn_successor_preserves_previous_slot() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let old = payload("old");
+        let next = payload("next");
+        commit(&disk, 0, owner, 1, &old).await;
+        install(&disk, PAYLOAD_PATHS[1], &next).await;
+        let recovered = read_committed(std::slice::from_ref(&disk), 4096)
+            .await
+            .expect("staged payload is not a commit")
+            .expect("old snapshot");
+        assert_eq!(recovered.payload, old);
+        install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)[..20]).await;
+        let recovered = read_committed(std::slice::from_ref(&disk), 4096)
+            .await
+            .expect("torn manifest preserves old slot")
+            .expect("old snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+        install(&disk, MANIFEST_PATHS[1], &manifest(owner, 2, &next)).await;
+        install(&disk, PAYLOAD_PATHS[1], b"torn").await;
+        let recovered = read_committed(&[disk], 4096)
+            .await
+            .expect("torn payload preserves old slot")
+            .expect("old snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+    }
+
+    #[tokio::test]
+    async fn stale_manifest_cas_cannot_replace_committed_anchor() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let bytes = payload("object");
+        commit(&disk, 0, owner, 1, &bytes).await;
+        let result = EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            MANIFEST_PATHS[0],
+            None,
+            Some(manifest(owner, 2, &bytes).into()),
+        )
+        .await
+        .expect("CAS call");
+        assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch);
+        let recovered = read_committed(&[disk], 4096)
+            .await
+            .expect("read old anchor")
+            .expect("snapshot");
+        assert_eq!(recovered.manifest.sequence, 1);
+    }
+
+    #[tokio::test]
+    async fn legacy_import_requires_complete_consistent_replicas() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let bytes = payload("object");
+        for (disk, data) in [(&first, &bytes[..bytes.len() - 1]), (&second, bytes.as_slice())] {
+            EcstoreDiskAPI::write_all(
+                disk.as_ref(),
+                RUSTFS_META_BUCKET,
+                MRF_SCOPED_JOURNAL_PATH,
+                EcstoreDiskBytes::copy_from_slice(data),
+            )
+            .await
+            .expect("legacy fixture");
+        }
+        let disks = [first.clone(), second];
+        assert!(
+            matches!(read_recovery_snapshot(&disks, 4096).await.expect("intact legacy replica"), Some(RecoverySnapshot::Legacy(data)) if data == bytes)
+        );
+        EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, payload("different").into())
+            .await
+            .expect("divergent fixture");
+        assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)));
+    }
+
+    #[tokio::test]
+    async fn committed_inspection_leaves_payload_and_manifest_unchanged() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let owner = Uuid::new_v4();
+        let bytes = payload("object");
+        commit(&disk, 0, owner, 3, &bytes).await;
+        assert!(matches!(
+            read_recovery_snapshot(std::slice::from_ref(&disk), 4096)
+                .await
+                .expect("new snapshot"),
+            Some(RecoverySnapshot::Committed(_))
+        ));
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
+                .await
+                .expect("manifest retained")
+                .as_ref(),
+            manifest(owner, 3, &bytes)
+        );
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
+                .await
+                .expect("payload retained")
+                .as_ref(),
+            bytes
+        );
+    }
+
+    #[tokio::test]
+    async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
+        let scoped = |set_index| {
+            let intent = MrfIntent {
+                bucket: Arc::from("bucket"),
+                object: Arc::from("a"),
+                version_id: None,
+                kind: MrfKind::PartialWrite,
+                scope: Some(MrfScope {
+                    pool_index: 0,
+                    set_index,
+                }),
+                lease: None,
+                enqueued_at_ms: 1234,
+                attempts: 0,
+            };
+            let mut bytes = Vec::new();
+            assert!(encode_intent(&intent, &mut bytes), "scoped fixture must encode");
+            bytes
+        };
+        let mut superset = payload("a");
+        superset.extend_from_slice(&payload("b"));
+        for (case, first_bytes, second_bytes) in [
+            ("complete-subset", payload("a"), superset),
+            ("different-set", scoped(1), scoped(2)),
+            ("unknown-scope", payload("a"), scoped(1)),
+        ] {
+            let root = TempDir::new().expect("test directory");
+            let first = disk(&root, "first").await;
+            let second = disk(&root, "second").await;
+            for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
+                assert_eq!(decode_journal(bytes).1, 0, "{case}: complete fixture");
+                EcstoreDiskAPI::write_all(
+                    disk.as_ref(),
+                    RUSTFS_META_BUCKET,
+                    MRF_SCOPED_JOURNAL_PATH,
+                    EcstoreDiskBytes::copy_from_slice(bytes),
+                )
+                .await
+                .expect("write legacy replica");
+            }
+            for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+                assert!(
+                    matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Conflict)),
+                    "{case}: neither replica order proves a latest snapshot"
+                );
+            }
+            for (disk, bytes) in [(&first, &first_bytes), (&second, &second_bytes)] {
+                assert_eq!(
+                    EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
+                        .await
+                        .expect("legacy evidence retained")
+                        .as_ref(),
+                    bytes.as_slice(),
+                    "{case}: inspection must preserve both source replicas"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn oversized_or_corrupt_scoped_snapshot_never_falls_back_to_legacy() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, vec![0; 1025].into())
+            .await
+            .expect("oversized fixture");
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload("old").into())
+            .await
+            .expect("legacy fixture");
+        assert!(matches!(
+            read_recovery_snapshot(std::slice::from_ref(&disk), 1024).await,
+            Err(SnapshotError::TooLarge)
+        ));
+        assert!(matches!(read_recovery_snapshot(&[disk], 2048).await, Err(SnapshotError::Corrupt)));
+    }
+
+    #[tokio::test]
+    async fn empty_legacy_replica_cannot_erase_records_in_a_torn_replica() {
+        let root = TempDir::new().expect("test directory");
+        let first = disk(&root, "first").await;
+        let second = disk(&root, "second").await;
+        let mut incomplete = payload("durable-object");
+        incomplete.extend_from_slice(b"torn");
+        EcstoreDiskAPI::write_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, Vec::new().into())
+            .await
+            .expect("empty truncated replica");
+        EcstoreDiskAPI::write_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, incomplete.clone().into())
+            .await
+            .expect("records and torn tail");
+        for disks in [vec![first.clone(), second.clone()], vec![second.clone(), first.clone()]] {
+            assert!(matches!(read_recovery_snapshot(&disks, 4096).await, Err(SnapshotError::Corrupt)));
+        }
+        assert_eq!(
+            EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
+                .await
+                .expect("recovery anchor preserved")
+                .as_ref(),
+            incomplete
+        );
+    }
+
+    #[tokio::test]
+    async fn unreadable_commit_record_never_implies_legacy_only() {
+        let root = TempDir::new().expect("test directory");
+        let disk = disk(&root, "disk").await;
+        let legacy = payload("old");
+        EcstoreDiskAPI::write_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, legacy.clone().into())
+            .await
+            .expect("legacy fixture");
+        // Opening a directory as a record either fails at open or at read,
+        // depending on the platform. Neither outcome proves absence.
+        std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[0]))
+            .expect("unreadable manifest fixture");
+        let recovered = read_recovery_snapshot(std::slice::from_ref(&disk), 4096).await;
+        assert!(
+            matches!(recovered, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))),
+            "must preserve unavailable proof: {recovered:?}"
+        );
+        assert_eq!(
+            EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_JOURNAL_PATH)
+                .await
+                .expect("legacy remains")
+                .as_ref(),
+            legacy
+        );
+    }
+}

crates/heal/tests/heal_b5_versioned_regression_test.rs

@@ -44,7 +44,9 @@ use walkdir::WalkDir;
 
 mod storage_api;
 
-use storage_api::integration::{BucketOperations, ECStore, MakeBucketOptions, ObjectIO as _, ObjectOperations as _};
+use storage_api::integration::{
+    BucketOperations, ECStore, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _,
+};
 
 /// 256 KiB + change: large enough to be stored as non-inline erasure shards
 /// (so each data version materializes as an on-disk `part.*` file we can assert
@@ -106,6 +108,7 @@ async fn put_versioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data:
         .put_object(bucket, object, &mut reader, &opts)
         .await
         .expect("versioned put_object failed");
+    wait_for_put_tail(ecstore, bucket, object).await;
     info.version_id
         .map(|u| u.to_string())
         .expect("versioned put must return a version id")
@@ -117,6 +120,7 @@ async fn put_unversioned(ecstore: &Arc<ECStore>, bucket: &str, object: &str, dat
         .put_object(bucket, object, &mut reader, &ObjectOptions::default())
         .await
         .expect("unversioned put_object failed");
+    wait_for_put_tail(ecstore, bucket, object).await;
 }
 
 /// Create a delete-marker as the latest version (versioned:true, no version_id)
@@ -160,20 +164,16 @@ fn xl_meta_path(obj_dir: &Path) -> PathBuf {
     obj_dir.join("xl.meta")
 }
 
-async fn wait_for_two_version_copies(disks: &[PathBuf], bucket: &str, object: &str) {
-    tokio::time::timeout(Duration::from_secs(5), async {
-        loop {
-            if disks.iter().all(|disk| {
-                let object_dir = object_dir(disk, bucket, object);
-                xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 2
-            }) {
-                break;
-            }
-            tokio::time::sleep(Duration::from_millis(10)).await;
-        }
-    })
-    .await
-    .expect("PUT rename tails must converge before wiping the versioned fixture");
+async fn wait_for_put_tail(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
+    // Shards and xl.meta can exist before the detached PUT owner finishes.
+    let lock = ecstore
+        .new_ns_lock(bucket, object)
+        .await
+        .expect("fixture namespace lock should be created");
+    let _settled = lock
+        .get_write_lock(Duration::from_secs(30))
+        .await
+        .expect("PUT rename tail must finish before inspecting or wiping the fixture");
 }
 
 fn recreate_heal_opts() -> HealOpts {
@@ -305,7 +305,13 @@ mod serial_tests {
         let data_v2 = versioned_test_data(20);
         let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; // OLD, non-latest
         let v2 = put_versioned(&ecstore, bucket, object, &data_v2).await; // latest
-        wait_for_two_version_copies(&disk_paths, bucket, object).await;
+        assert!(
+            disk_paths.iter().all(|disk| {
+                let dir = object_dir(disk, bucket, object);
+                xl_meta_path(&dir).exists() && count_part_files(&dir) >= 2
+            }),
+            "both versions must exist on every disk before wiping the fixture"
+        );
 
         // ── Pre-wipe: prove the fixture actually has 2 versions on disk[0] ──
         let obj_dir0 = object_dir(&disk_paths[0], bucket, object);

crates/heal/tests/storage_api.rs

@@ -23,6 +23,7 @@ pub(crate) mod integration {
     pub(crate) use rustfs_ecstore::api::storage::ECStore;
     pub(crate) use rustfs_storage_api::BucketOperations;
     pub(crate) use rustfs_storage_api::MakeBucketOptions;
+    pub(crate) use rustfs_storage_api::NamespaceLocking;
     pub(crate) use rustfs_storage_api::ObjectIO;
     pub(crate) use rustfs_storage_api::ObjectOperations;
 }

crates/lifecycle/src/core.rs

@@ -43,6 +43,10 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
     "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
 const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
+const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
+const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
+const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
+    "Only one NoncurrentVersionTransition action per lifecycle rule is supported";
 const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
     "'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
 const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -361,6 +365,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 {
                     return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
                 }
+                if expiration.days.is_some() && expiration.date.is_some() {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidInput,
+                        ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
+                    ));
+                }
                 if let Some(expiration_date) = &expiration.date {
                     let date = OffsetDateTime::from(expiration_date.clone());
                     if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -394,11 +404,20 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 }
             }
             if let Some(transitions) = &r.transitions {
+                if transitions.len() > 1 {
+                    return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
+                }
                 for transition in transitions {
                     TransitionOps::validate(transition)?;
                 }
             }
             if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
+                if noncurrent_transitions.len() > 1 {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidInput,
+                        ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
+                    ));
+                }
                 for transition in noncurrent_transitions {
                     NoncurrentVersionTransitionOps::validate(transition)?;
                 }
@@ -473,6 +492,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
     }
 
     async fn eval(&self, obj: &ObjectOpts) -> Event {
+        // A single-object lookup cannot prove how many newer historical versions
+        // survive. Count-dependent actions wait for the complete-group evaluator.
         self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
     }
 
@@ -536,23 +557,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
             return Event::default();
         };
 
-        if let Some(restore_expires) = obj.restore_expires
-            && restore_expires.unix_timestamp() != 0
-            && now.unix_timestamp() > restore_expires.unix_timestamp()
-        {
-            let mut action = IlmAction::DeleteRestoredAction;
-            if !obj.is_latest {
-                action = IlmAction::DeleteRestoredVersionAction;
-            }
-
-            events.push(Event {
-                action,
-                due: Some(now),
-                rule_id: "".into(),
-                noncurrent_days: 0,
-                newer_noncurrent_versions: 0,
-                storage_class: "".into(),
-            });
+        if let Some(event) = obj.restored_copy_expiry(now) {
+            events.push(event);
         }
 
         if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -611,17 +617,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     continue;
                 }
 
-                if !obj.is_latest
-                    && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
-                    && let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
-                    && newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
-                {
-                    continue;
-                }
-
                 if !obj.is_latest
                     && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
                     && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
+                    && noncurrent_version_expiration
+                        .newer_noncurrent_versions
+                        .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
                 {
                     if let Some(successor_mod_time) = obj.successor_mod_time {
                         let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -651,7 +652,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     && let Some(noncurrent_version_transition) = rule
                         .noncurrent_version_transitions
                         .as_ref()
+                        .filter(|transitions| transitions.len() == 1)
                         .and_then(|transitions| transitions.first())
+                    && noncurrent_version_transition
+                        .newer_noncurrent_versions
+                        .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
                     && let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
                     && !storage_class.as_str().is_empty()
                     && !obj.delete_marker
@@ -735,7 +740,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     }
 
                     if obj.transition_status != TRANSITION_COMPLETE
-                        && let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
+                        && let Some(transition) = rule
+                            .transitions
+                            .as_ref()
+                            .filter(|transitions| transitions.len() == 1)
+                            .and_then(|transitions| transitions.first())
                         && let Some(storage_class) = transition.storage_class.as_ref()
                         && !storage_class.as_str().is_empty()
                     {
@@ -758,18 +767,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
         }
 
         if !events.is_empty() {
-            // Select the winning event using a strict total order (MinIO semantics):
-            // the earliest `due` wins, and ties break toward delete-type actions. A
-            // missing `due` is treated as UNIX_EPOCH. This replaces a hand-written
-            // `sort_by` comparator that was not a strict weak ordering (it could return
-            // `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
-            // repository toolchain and did not deterministically pick the earliest event.
+            // Eligible expiration takes precedence over transition, even when a
+            // failed transition has an earlier deadline. Within each action class,
+            // prefer the earliest deadline using a deterministic total order.
             let event = events
                 .iter()
                 .min_by_key(|event| {
                     (
-                        event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
                         ilm_action_priority_rank(&event.action),
+                        event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
                     )
                 })
                 .cloned()
@@ -1042,6 +1048,27 @@ impl ObjectOpts {
     pub fn expired_object_deletemarker(&self) -> bool {
         self.delete_marker && self.is_latest && self.num_versions == 1
     }
+
+    pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
+        let restore_expires = self.restore_expires?;
+        // Restore metadata alone does not prove that a durable remote copy exists.
+        if self.transition_status != TRANSITION_COMPLETE
+            || restore_expires.unix_timestamp() == 0
+            || now.unix_timestamp() <= restore_expires.unix_timestamp()
+        {
+            return None;
+        }
+        let action = if self.is_latest {
+            IlmAction::DeleteRestoredAction
+        } else {
+            IlmAction::DeleteRestoredVersionAction
+        };
+        expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
+            action,
+            due: Some(now),
+            ..Default::default()
+        })
+    }
 }
 
 /// Returns whether an expiry action has enough identity to target the object
@@ -1064,11 +1091,8 @@ pub fn expiration_action_has_valid_target(
     }
 }
 
-/// Total-order rank for lifecycle actions used to break `due` ties.
-///
-/// Delete-type actions rank before every other action so that, when two events
-/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
-/// values only matter relative to each other.
+/// Eligible logical expiration takes precedence over transition and restore-copy
+/// cleanup. Deadlines break ties within an action class.
 fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
     match action {
         IlmAction::DeleteAllVersionsAction
@@ -4159,6 +4183,392 @@ mod tests {
         assert_eq!(event.action, IlmAction::NoneAction);
     }
 
+    mod adversarial_regressions {
+        use super::*;
+        use s3s::dto::NoncurrentVersionExpiration;
+
+        fn run(test: impl std::future::Future<Output = ()>) {
+            with_default_ilm_process_time(|| {
+                tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .expect("lifecycle regression runtime should build")
+                    .block_on(test);
+            });
+        }
+
+        fn noncurrent_object() -> ObjectOpts {
+            ObjectOpts {
+                name: "logs/object".to_string(),
+                mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
+                successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
+                version_id: Some(Uuid::from_u128(1)),
+                size: 1024 * 1024,
+                ..Default::default()
+            }
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_retains_the_requested_newer_versions() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = Arc::new(BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                });
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid noncurrent transition policy");
+                let objects = (0..4)
+                    .map(|index| ObjectOpts {
+                        mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
+                        successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
+                        version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
+                        is_latest: index == 0,
+                        num_versions: 4,
+                        ..noncurrent_object()
+                    })
+                    .collect::<Vec<_>>();
+                let actions = crate::Evaluator::new(lc)
+                    .eval(&objects)
+                    .await
+                    .expect("complete version chain should evaluate")
+                    .into_iter()
+                    .map(|event| event.action)
+                    .collect::<Vec<_>>();
+                assert_eq!(
+                    actions,
+                    [
+                        IlmAction::NoneAction,
+                        IlmAction::NoneAction,
+                        IlmAction::NoneAction,
+                        IlmAction::TransitionVersionAction
+                    ],
+                    "the two newest noncurrent versions must remain in their current storage class"
+                );
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_checks_count_age_and_single_object_context() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("retain-two"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(3),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid counted transition");
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for (newer, expected) in [
+                    (0, IlmAction::NoneAction),
+                    (1, IlmAction::NoneAction),
+                    (2, IlmAction::TransitionVersionAction),
+                    (3, IlmAction::TransitionVersionAction),
+                ] {
+                    assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
+                }
+                assert_eq!(
+                    lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
+                    IlmAction::NoneAction,
+                    "the retention count does not replace the age condition"
+                );
+                assert_eq!(
+                    lc.eval(&object).await.action,
+                    IlmAction::NoneAction,
+                    "a single-object lookup must not assume a complete version history"
+                );
+                for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
+                    lc.rules[0]
+                        .noncurrent_version_transitions
+                        .as_mut()
+                        .expect("transition exists")[0]
+                        .newer_noncurrent_versions = retain;
+                    let expected = if matches!(retain, None | Some(0)) {
+                        IlmAction::TransitionVersionAction
+                    } else {
+                        IlmAction::NoneAction
+                    };
+                    assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
+                }
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("independent-counts"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(90),
+                    newer_noncurrent_versions: Some(4),
+                });
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(30),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid independent retention limits");
+                let object = noncurrent_object();
+                let now = datetime!(2020-05-01 00:00:00 UTC);
+                for (newer, expected) in [
+                    (1, IlmAction::NoneAction),
+                    (2, IlmAction::TransitionVersionAction),
+                    (3, IlmAction::TransitionVersionAction),
+                    (4, IlmAction::DeleteVersionAction),
+                ] {
+                    assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
+                }
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn expiration_retention_does_not_skip_an_independent_transition() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: None,
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                let transition_only = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
+
+                lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(90),
+                    newer_noncurrent_versions: Some(2),
+                });
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid combined policy");
+                let combined = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
+                assert_eq!(combined.storage_class, transition_only.storage_class);
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn current_transition_rejects_multiple_stages_in_any_order() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
+                rule.transitions = Some(vec![
+                    Transition {
+                        date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
+                        days: None,
+                        storage_class: Some(TransitionStorageClass::from_static("COLD")),
+                    },
+                    Transition {
+                        date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
+                        days: None,
+                        storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                    },
+                ]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = ObjectOpts {
+                    is_latest: true,
+                    ..noncurrent_object()
+                };
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
+                    lc.rules[0].status = ExpirationStatus::from_static(status);
+                    for _ in 0..2 {
+                        let err = lc
+                            .validate(&ObjectLockConfiguration::default())
+                            .await
+                            .expect_err("multiple transition stages must be rejected");
+                        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                        assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
+                        assert_eq!(
+                            lc.eval_inner(&object, now, 0).await.action,
+                            IlmAction::NoneAction,
+                            "legacy multi-stage configurations must not silently execute their first stage"
+                        );
+                        lc.rules[0]
+                            .transitions
+                            .as_mut()
+                            .expect("transition array is present")
+                            .reverse();
+                    }
+                }
+                lc.rules[0]
+                    .transitions
+                    .as_mut()
+                    .expect("transition array is present")
+                    .remove(0);
+                lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("one stage is supported");
+                let event = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(event.action, IlmAction::TransitionAction);
+                assert_eq!(event.storage_class, "WARM");
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
+                rule.noncurrent_version_transitions = Some(vec![
+                    NoncurrentVersionTransition {
+                        noncurrent_days: Some(30),
+                        newer_noncurrent_versions: None,
+                        storage_class: Some(TransitionStorageClass::from_static("COLD")),
+                    },
+                    NoncurrentVersionTransition {
+                        noncurrent_days: Some(1),
+                        newer_noncurrent_versions: None,
+                        storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                    },
+                ]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
+                    lc.rules[0].status = ExpirationStatus::from_static(status);
+                    for _ in 0..2 {
+                        let err = lc
+                            .validate(&ObjectLockConfiguration::default())
+                            .await
+                            .expect_err("multiple noncurrent transition stages must be rejected");
+                        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                        assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
+                        assert_eq!(
+                            lc.eval_inner(&object, now, 0).await.action,
+                            IlmAction::NoneAction,
+                            "legacy multi-stage configurations must not silently execute their first stage"
+                        );
+                        lc.rules[0]
+                            .noncurrent_version_transitions
+                            .as_mut()
+                            .expect("transition array is present")
+                            .reverse();
+                    }
+                }
+                lc.rules[0]
+                    .noncurrent_version_transitions
+                    .as_mut()
+                    .expect("transition array is present")
+                    .remove(0);
+                lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("one stage is supported");
+                let event = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(event.action, IlmAction::TransitionVersionAction);
+                assert_eq!(event.storage_class, "WARM");
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn expiration_rejects_simultaneous_days_and_date() {
+            run(async {
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![enabled_rule(
+                        Some(LifecycleExpiration {
+                            days: Some(1),
+                            ..Default::default()
+                        }),
+                        None,
+                        Some("ambiguous-expiry"),
+                    )],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("a single Days expiration is valid");
+                lc.rules[0].expiration.as_mut().expect("expiration is present").date =
+                    Some(datetime!(2099-01-01 00:00:00 UTC).into());
+                let err = lc
+                    .validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
+                assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn overdue_transition_does_not_starve_permanent_expiration() {
+            run(async {
+                let mut rule = enabled_rule(
+                    Some(LifecycleExpiration {
+                        days: Some(90),
+                        ..Default::default()
+                    }),
+                    None,
+                    Some("archive-then-delete"),
+                );
+                rule.transitions = Some(vec![Transition {
+                    days: Some(30),
+                    date: None,
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid transition and expiration policy");
+                let object = ObjectOpts {
+                    is_latest: true,
+                    version_id: None,
+                    transition_status: TRANSITION_PENDING.to_string(),
+                    ..noncurrent_object()
+                };
+                let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
+                assert_eq!(before_expiration.action, IlmAction::TransitionAction);
+                let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
+                assert_eq!(
+                    overdue.action,
+                    IlmAction::DeleteAction,
+                    "an unavailable tier must not prevent permanent expiration indefinitely"
+                );
+            });
+        }
+    }
+
     /// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
     /// follow-up to backlog#1030 / rustfs#4455).
     ///
@@ -4169,7 +4579,7 @@ mod tests {
     ///
     /// * `eval_inner` never panics and is deterministic for a fixed input;
     /// * the winning event matches an independently recomputed candidate set:
-    ///   earliest `due` wins, ties break toward delete-class actions (the
+    ///   eligible expiration wins over transition, then earliest `due` wins (the
     ///   `min_by_key` selection that replaced the rustfs#4455 comparator);
     /// * `expected_expiry_time` is monotonically non-decreasing in `days` and
     ///   always lands on the processing boundary, both at production defaults
@@ -4458,8 +4868,8 @@ mod tests {
         /// consider for a live current version under `selection`-shaped rules
         /// (expiration and first-transition only, no filters): expiration
         /// fires when `now >= due`, transition when `now > due` and the object
-        /// has not already transitioned. Selection semantics under test:
-        /// earliest due wins, ties prefer delete-class.
+        /// has not already transitioned. Eligible expiration wins over transition;
+        /// the earliest deadline wins within the selected action class.
         fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
             let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
             let mut candidates = Vec::new();
@@ -4548,8 +4958,8 @@ mod tests {
             /// Differential test of winner selection (the rustfs#4455 fix):
             /// for a live current version under randomized expiration and
             /// transition rules, `eval_inner`'s winner must carry the
-            /// minimum `(due, rank)` of the independently recomputed
-            /// candidate set — earliest due wins, ties prefer delete-class —
+            /// earliest expiration from the independently recomputed candidate
+            /// set, or the earliest transition when no expiration is eligible,
             /// and must be `NoneAction` exactly when that set is empty.
             #[test]
             #[serial]
@@ -4578,7 +4988,13 @@ mod tests {
 
                 // Oracle and evaluator must observe the same (pinned) time env.
                 let (event, expected) = with_production_time_env(|| {
-                    let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
+                    let candidates = oracle_candidates(&lc, &obj, now);
+                    let expected = candidates
+                        .iter()
+                        .filter(|(_, rank)| *rank == 0)
+                        .min()
+                        .copied()
+                        .or_else(|| candidates.into_iter().min());
                     let rt = tokio::runtime::Builder::new_current_thread()
                         .enable_all()
                         .build()

crates/lifecycle/src/evaluator.rs

@@ -116,13 +116,10 @@ impl Evaluator {
                             break 'top_loop;
                         }
                     }
-                    IlmAction::DeleteAction
-                    | IlmAction::DeleteRestoredAction
-                    | IlmAction::DeleteVersionAction
-                    | IlmAction::DeleteRestoredVersionAction
-                        if self.is_object_locked(obj) =>
-                    {
-                        event = Event::default();
+                    // Restore expiry removes only the temporary local copy; the
+                    // retained logical version and its remote data remain intact.
+                    IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => {
+                        event = obj.restored_copy_expiry(now).unwrap_or_default();
                     }
                     _ => {}
                 }
@@ -206,6 +203,95 @@ mod tests {
 
     use super::*;
     use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
+
+    #[tokio::test]
+    async fn adversarial_restore_expiry_survives_legal_hold() {
+        let mut policy = (*latest_expiration_lifecycle()).clone();
+        policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED);
+        let policy = Arc::new(policy);
+        policy
+            .validate(&lock_enabled_without_default_retention())
+            .await
+            .expect("valid disabled lifecycle rule");
+        let mut objects = [true, false].map(|is_latest| ObjectOpts {
+            is_latest,
+            num_versions: 2,
+            mod_time: Some(
+                OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 })
+                    .expect("fixed version timestamp"),
+            ),
+            successor_mod_time: (!is_latest)
+                .then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")),
+            transition_status: crate::TRANSITION_COMPLETE.to_string(),
+            restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")),
+            ..current_object_opts(ReplicationStatusType::Completed)
+        });
+        let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention()));
+        let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction];
+        let unlocked = evaluator
+            .eval(&objects)
+            .await
+            .expect("unlocked restored versions should evaluate");
+        assert_eq!(unlocked.iter().map(|event| event.action).collect::<Vec<_>>(), expected);
+
+        for object in &mut objects {
+            object
+                .user_defined
+                .insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
+        }
+        let locked = evaluator
+            .eval(&objects)
+            .await
+            .expect("locked restored versions should evaluate");
+        assert_eq!(
+            locked.iter().map(|event| event.action).collect::<Vec<_>>(),
+            expected,
+            "expiring a restored local copy preserves the retained logical version and remote object"
+        );
+
+        let mut expiring_policy = (*latest_expiration_lifecycle()).clone();
+        expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: None,
+        });
+        let expiring_evaluator =
+            Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention()));
+        let locked = expiring_evaluator
+            .eval(&objects)
+            .await
+            .expect("locked expired versions should evaluate");
+        assert_eq!(
+            locked.iter().map(|event| event.action).collect::<Vec<_>>(),
+            expected,
+            "blocked logical expiration must still allow an eligible restore-copy cleanup"
+        );
+
+        for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] {
+            for object in &mut objects {
+                object.replication_status = status.clone();
+            }
+            for evaluator in [&evaluator, &expiring_evaluator] {
+                let events = evaluator.eval(&objects).await.expect("pending replication should evaluate");
+                assert!(events.iter().all(|event| event.action == IlmAction::NoneAction));
+            }
+        }
+        for object in &mut objects {
+            object.replication_status = ReplicationStatusType::Completed;
+        }
+        for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] {
+            for object in &mut objects {
+                object.transition_status = transition_status.to_string();
+            }
+            for evaluator in [&evaluator, &expiring_evaluator] {
+                let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate");
+                assert!(
+                    events.iter().all(|event| event.action == IlmAction::NoneAction),
+                    "restore metadata cannot authorize cleanup without a completed transition"
+                );
+            }
+        }
+    }
+
     fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
         Arc::new(BucketLifecycleConfiguration {
             expiry_updated_at: None,

crates/madmin/fixtures/on_demand_migration/get_response.json

@@ -1 +1 @@
-{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
+{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}

crates/madmin/fixtures/on_demand_migration/set_request.json

@@ -1 +1 @@
-{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
+{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}

crates/madmin/fixtures/on_demand_migration/set_response.json

@@ -1 +1 @@
-{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
+{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}

crates/madmin/src/on_demand_migration.rs

@@ -17,7 +17,7 @@
 //! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}`,
 //! `GET .../status`, `POST .../backfill?op=start|cancel` and
 //! `GET .../backfill` (ODM-12), mirroring the server's config model
-//! (`crates/ecstore/src/bucket/on_demand_migration/config.rs`) and handler
+//! (`rustfs/src/on_demand_migration/config.rs`) and handler
 //! responses (`rustfs/src/admin/handlers/on_demand_migration.rs`). The SDK
 //! owns its own copies, madmin-go style; the fixtures under
 //! `fixtures/on_demand_migration/` are the contract both sides pin
@@ -78,10 +78,18 @@ pub struct OnDemandMigrationSource {
     #[serde(default)]
     pub path_style: OnDemandMigrationPathStyle,
     /// `None` means anonymous access to a public source bucket.
+    /// `None` means anonymous access to a public source bucket. The native
+    /// providers carry their credentials in `azure` / `gcs` instead.
     #[serde(default)]
     pub credentials: Option<OnDemandMigrationCredentials>,
     #[serde(default)]
     pub tls: OnDemandMigrationTls,
+    /// Required for `azure` and rejected for every other provider.
+    #[serde(default)]
+    pub azure: Option<OnDemandMigrationAzure>,
+    /// Required for `gcs_native` and rejected for every other provider.
+    #[serde(default)]
+    pub gcs: Option<OnDemandMigrationGcs>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -92,7 +100,49 @@ pub enum OnDemandMigrationProvider {
     Minio,
     Rustfs,
     R2,
+    /// GCS XML interoperability API with HMAC keys.
     Gcs,
+    /// Native Azure Blob service.
+    Azure,
+    /// Native GCS JSON API with a service-account key.
+    #[serde(rename = "gcs_native")]
+    GcsNative,
+}
+
+/// Native Azure Blob parameters. The container is `source.bucket`; exactly one
+/// of `account_key` and `sas_token` is set. Responses carry both as `REDACTED`.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct OnDemandMigrationAzure {
+    pub account: String,
+    #[serde(default)]
+    pub account_key: Option<String>,
+    #[serde(default)]
+    pub sas_token: Option<String>,
+}
+
+impl fmt::Debug for OnDemandMigrationAzure {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("OnDemandMigrationAzure")
+            .field("account", &self.account)
+            .field("account_key", &self.account_key.as_ref().map(|_| "REDACTED"))
+            .field("sas_token", &self.sas_token.as_ref().map(|_| "REDACTED"))
+            .finish()
+    }
+}
+
+/// Native GCS parameters. The bucket is `source.bucket`; the key JSON embeds a
+/// private key, so responses carry it as `REDACTED`.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct OnDemandMigrationGcs {
+    pub service_account_json: String,
+}
+
+impl fmt::Debug for OnDemandMigrationGcs {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("OnDemandMigrationGcs")
+            .field("service_account_json", &"REDACTED")
+            .finish()
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
@@ -806,6 +856,8 @@ mod tests {
                 session_token: None,
             }),
             tls: OnDemandMigrationTls::default(),
+            azure: None,
+            gcs: None,
         });
         let mut expected: OnDemandMigrationConfig = serde_json::from_str(SET_REQUEST_FIXTURE.trim()).expect("fixture");
         expected.filter.source_prefix = None;
@@ -821,6 +873,42 @@ mod tests {
         assert!(minimal.source.credentials.is_none());
     }
 
+    #[test]
+    fn native_provider_documents_round_trip_and_hide_their_secrets() {
+        for (label, json) in [
+            (
+                "azure",
+                r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#,
+            ),
+            (
+                "gcs_native",
+                r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
+            ),
+        ] {
+            let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}"));
+            assert_eq!(
+                serde_json::to_string(&source).expect("re-encodes"),
+                json,
+                "{label} must reproduce the server wire shape byte for byte"
+            );
+        }
+
+        let azure = OnDemandMigrationAzure {
+            account: "legacyaccount".to_string(),
+            account_key: Some("c2VjcmV0".to_string()),
+            sas_token: Some("sig=topsecret".to_string()),
+        };
+        let rendered = format!("{azure:?}");
+        assert!(rendered.contains("legacyaccount"));
+        assert!(!rendered.contains("c2VjcmV0"), "{rendered}");
+        assert!(!rendered.contains("topsecret"), "{rendered}");
+
+        let gcs = OnDemandMigrationGcs {
+            service_account_json: r#"{"private_key":"-----BEGIN PRIVATE KEY-----"}"#.to_string(),
+        };
+        assert!(!format!("{gcs:?}").contains("PRIVATE KEY"), "{gcs:?}");
+    }
+
     #[test]
     fn credentials_debug_never_prints_secrets() {
         let credentials = OnDemandMigrationCredentials {

crates/obs/src/metrics/mod.rs

@@ -38,3 +38,4 @@ pub(crate) use storage_api::metrics::{
     obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
     obs_transition_state_handle,
 };
+pub use storage_api::register_on_demand_migration_metrics_source;

crates/obs/src/metrics/storage_api.rs

@@ -17,13 +17,6 @@ use std::time::Duration;
 
 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
-use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{
-    BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
-};
-use rustfs_ecstore::api::bucket::on_demand_migration::{
-    BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
-    OnDemandMigrationSys as SourceOnDemandMigrationSys,
-};
 use rustfs_ecstore::api::bucket::replication::{
     BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
     MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
@@ -44,9 +37,7 @@ pub(crate) use rustfs_ecstore::api::runtime::{
 pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
 use rustfs_storage_api as storage_contracts;
 
-use crate::metrics::collectors::{
-    OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
-};
+use crate::metrics::collectors::{OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBucketStats};
 
 #[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
@@ -465,70 +456,37 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
     buckets
 }
 
-fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
-    let stats = snapshot.stats;
-    OnDemandMigrationBucketStats {
-        bucket: snapshot.bucket,
-        requests_total: stats.requests_total,
-        pulled_bytes_total: stats.pulled_bytes_total,
-        pulled_objects_total: stats.pulled_objects_total,
-        pull_failures_total: stats.pull_failures_total,
-        inflight_pulls: stats.inflight_pulls,
-        queue_depth: stats.queue_depth,
-        source_latency_buckets: stats
-            .source_latency
-            .buckets
-            .into_iter()
-            .map(|bucket| (bucket.le_ms, bucket.count))
-            .collect(),
-        source_latency_count: stats.source_latency.count,
-        source_latency_sum_ms: stats.source_latency.sum_ms,
-        breaker_state: match stats.breaker_state {
-            SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
-            SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
-            SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
-        },
-    }
+struct OnDemandMigrationMetricsSource {
+    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
+    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
 }
 
-/// Every bucket with live on-demand migration state on this node, sorted by
-/// name. Empty while the module switch is off.
-pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
-    SourceOnDemandMigrationSys::get()
-        .snapshot()
-        .into_iter()
-        .map(on_demand_migration_stats_from_snapshot)
-        .collect()
-}
+static ON_DEMAND_MIGRATION_METRICS_SOURCE: std::sync::OnceLock<OnDemandMigrationMetricsSource> = std::sync::OnceLock::new();
 
-fn on_demand_migration_backfill_stats_from_checkpoint(
-    bucket: String,
-    checkpoint: SourceBackfillCheckpoint,
-) -> OdmBackfillBucketStats {
-    OdmBackfillBucketStats {
-        bucket,
-        state: checkpoint.state.as_str().to_string(),
-        listed: checkpoint.listed,
-        enqueued: checkpoint.enqueued,
-        pulled: checkpoint.pulled,
-        skipped_existing: checkpoint.skipped_existing,
-        failed: checkpoint.failed,
-        bytes: checkpoint.bytes,
-    }
-}
-
-/// Backfill jobs running on this node, sorted by bucket. Empty until the
-/// runner is installed, and empty again once a job finishes: the series are
-/// per-node job progress, not a cluster-wide history.
-pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
-    let buckets = source_global_backfill_runner()
-        .map(|runner| {
-            runner
-                .local_job_snapshots()
-                .into_iter()
-                .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
-                .collect()
+/// Register the application-owned ODM snapshots before starting the collector.
+pub fn register_on_demand_migration_metrics_source(
+    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
+    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
+) -> bool {
+    ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .set(OnDemandMigrationMetricsSource {
+            snapshot,
+            backfill_snapshot,
         })
+        .is_ok()
+}
+
+pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
+    ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .get()
+        .map(|source| (source.snapshot)())
+        .unwrap_or_default()
+}
+
+pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
+    let buckets = ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .get()
+        .map(|source| (source.backfill_snapshot)())
         .unwrap_or_default();
     OdmBackfillRuntimeStats { server, buckets }
 }
@@ -580,6 +538,31 @@ pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_ra
 mod tests {
     use super::*;
 
+    #[test]
+    fn on_demand_migration_callbacks_supply_runtime_snapshots() {
+        assert!(register_on_demand_migration_metrics_source(
+            || vec![OnDemandMigrationBucketStats {
+                bucket: "configured".into(),
+                pulled_bytes_total: 4096,
+                ..Default::default()
+            }],
+            || vec![OdmBackfillBucketStats {
+                bucket: "backfill".into(),
+                pulled: 3,
+                ..Default::default()
+            }],
+        ));
+        let snapshot = obs_on_demand_migration_snapshot();
+        assert_eq!(snapshot.len(), 1);
+        assert_eq!(snapshot[0].bucket, "configured");
+        assert_eq!(snapshot[0].pulled_bytes_total, 4096);
+        let backfill = obs_on_demand_migration_backfill_snapshot("node-a".into());
+        assert_eq!(backfill.server, "node-a");
+        assert_eq!(backfill.buckets.len(), 1);
+        assert_eq!(backfill.buckets[0].bucket, "backfill");
+        assert_eq!(backfill.buckets[0].pulled, 3);
+    }
+
     #[test]
     fn obs_replication_numeric_conversions_floor_negative_values() {
         assert_eq!(i64_to_u64_floor_zero(-1), 0);
@@ -772,51 +755,6 @@ mod tests {
         assert_eq!(snapshot.mrf_last_flush_duration_millis, 4);
     }
 
-    #[test]
-    fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
-        // Built from JSON: the snapshot's timestamps use `time`, which obs does not depend on.
-        let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
-            "bucket": "photos",
-            "provider": "minio",
-            "endpoint_host": "source.example.com",
-            "applied_at": "2026-09-02T10:00:00Z",
-            "client_error": null,
-            "negative_cache_entries": 0,
-            "inflight_keys": 1,
-            "max_concurrent_pulls": 8,
-            "stats": {
-                "requests_total": {"get": {"source_hit": 2}},
-                "pulled_bytes_total": 4096,
-                "pulled_objects_total": {"inline": 1},
-                "pull_failures_total": {"source_timeout": 1},
-                "inflight_pulls": 1,
-                "queue_depth": 2,
-                "source_latency": {
-                    "buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
-                    "count": 3,
-                    "sum_ms": 90753
-                },
-                "last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
-                "breaker_state": "open"
-            }
-        }))
-        .expect("runtime snapshot decodes");
-
-        let stats = on_demand_migration_stats_from_snapshot(snapshot);
-
-        assert_eq!(stats.bucket, "photos");
-        assert_eq!(stats.requests_total["get"]["source_hit"], 2);
-        assert_eq!(stats.pulled_bytes_total, 4096);
-        assert_eq!(stats.pulled_objects_total["inline"], 1);
-        assert_eq!(stats.pull_failures_total["source_timeout"], 1);
-        assert_eq!(stats.inflight_pulls, 1);
-        assert_eq!(stats.queue_depth, 2);
-        assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
-        assert_eq!(stats.source_latency_count, 3);
-        assert_eq!(stats.source_latency_sum_ms, 90_753);
-        assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
-    }
-
     #[test]
     fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
         let snapshot = bucket_replication_stats_snapshot_from_parts(

crates/s3-client/src/api_get_object.rs

@@ -120,18 +120,10 @@ impl TransitionClient {
 
         let h = resp.headers().clone();
 
-        let mut body = resp.into_body();
         let body_vec = if let Some(limit) = max_response_bytes {
-            collect_response_body(body, limit).await?
+            self.collect_response_body(resp.into_body(), limit).await?
         } else {
-            let mut body_vec = Vec::new();
-            while let Some(frame) = body.frame().await {
-                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-                if let Some(data) = frame.data_ref() {
-                    body_vec.extend_from_slice(data);
-                }
-            }
-            body_vec
+            self.collect_response_body_unbounded(resp.into_body()).await?
         };
         Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
     }
@@ -143,7 +135,7 @@ mod bounded_response_tests {
     use crate::{
         api_get_options::GetObjectOptions,
         credentials::{Credentials, SignatureType, Static, Value},
-        transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
+        transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, collect_response_body},
     };
     use http_body_util::Full;
     use hyper::body::Bytes;
@@ -175,7 +167,31 @@ mod bounded_response_tests {
         assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
     }
 
-    async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
+    fn test_options() -> Options {
+        Options {
+            creds: Credentials::new(Static(Value {
+                access_key_id: "access-key".to_string(),
+                secret_access_key: "secret-key".to_string(),
+                signer_type: SignatureType::SignatureV4,
+                ..Default::default()
+            })),
+            region: "us-east-1".to_string(),
+            bucket_lookup: BucketLookupType::BucketLookupPath,
+            max_retries: 1,
+            ..Default::default()
+        }
+    }
+
+    async fn client_for_endpoint(endpoint: &str, timeouts: TransitionClientTimeouts) -> TransitionClient {
+        TransitionClient::new_with_timeouts(endpoint, test_options(), "", timeouts)
+            .await
+            .expect("fixture client should build")
+    }
+
+    async fn bounded_get_fixture_with_timeouts(
+        body: &'static [u8],
+        timeouts: TransitionClientTimeouts,
+    ) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
         let listener = match TcpListener::bind("127.0.0.1:0").await {
             Ok(listener) => listener,
             Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -209,27 +225,14 @@ mod bounded_response_tests {
             stream.write_all(body).await.expect("fixture should write response body");
             request
         });
-        let client = TransitionClient::new(
-            &endpoint,
-            Options {
-                creds: Credentials::new(Static(Value {
-                    access_key_id: "access-key".to_string(),
-                    secret_access_key: "secret-key".to_string(),
-                    signer_type: SignatureType::SignatureV4,
-                    ..Default::default()
-                })),
-                region: "us-east-1".to_string(),
-                bucket_lookup: BucketLookupType::BucketLookupPath,
-                max_retries: 1,
-                ..Default::default()
-            },
-            "",
-        )
-        .await
-        .expect("fixture client should build");
+        let client = client_for_endpoint(&endpoint, timeouts).await;
         Some((client, request))
     }
 
+    async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
+        bounded_get_fixture_with_timeouts(body, TransitionClientTimeouts::default()).await
+    }
+
     #[tokio::test]
     async fn real_transport_accepts_the_exact_closed_range_length() {
         let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
@@ -292,24 +295,7 @@ mod bounded_response_tests {
             .local_addr()
             .expect("listener local address should be available")
             .to_string();
-        let client = TransitionClient::new(
-            &endpoint,
-            Options {
-                creds: Credentials::new(Static(Value {
-                    access_key_id: "access-key".to_string(),
-                    secret_access_key: "secret-key".to_string(),
-                    signer_type: SignatureType::SignatureV4,
-                    ..Default::default()
-                })),
-                region: "us-east-1".to_string(),
-                bucket_lookup: BucketLookupType::BucketLookupPath,
-                max_retries: 1,
-                ..Default::default()
-            },
-            "",
-        )
-        .await
-        .expect("fixture client should build");
+        let client = client_for_endpoint(&endpoint, TransitionClientTimeouts::default()).await;
         let mut opts = GetObjectOptions::default();
         opts.headers
             .insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
@@ -326,6 +312,176 @@ mod bounded_response_tests {
                 .is_err()
         );
     }
+
+    #[tokio::test]
+    async fn connection_refused_returns_without_waiting_for_the_request_timeout() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        drop(listener);
+
+        let client = client_for_endpoint(
+            &endpoint,
+            TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(5), Duration::from_secs(1)),
+        )
+        .await;
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let result = tokio::time::timeout(Duration::from_secs(2), client.get_object_inner("bucket", "probe", &opts))
+            .await
+            .expect("connection refused should return before the broader request timeout");
+
+        assert!(result.is_err(), "connection refused must fail instead of hanging");
+    }
+
+    #[tokio::test]
+    async fn response_header_stall_returns_timed_out() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            tokio::time::sleep(Duration::from_millis(200)).await;
+        });
+        let client = client_for_endpoint(
+            &endpoint,
+            TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
+        )
+        .await;
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("response header stalls must be bounded");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
+        fixture.await.expect("fixture should join");
+    }
+
+    #[tokio::test]
+    async fn response_body_idle_stall_returns_timed_out() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            stream
+                .write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\nRu")
+                .await
+                .expect("fixture should write the first body chunk");
+            tokio::time::sleep(Duration::from_millis(200)).await;
+        });
+        let client = client_for_endpoint(
+            &endpoint,
+            TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
+        )
+        .await;
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("body stalls after partial progress must be bounded");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
+        fixture.await.expect("fixture should join");
+    }
+
+    #[tokio::test]
+    async fn response_body_idle_timer_resets_on_progress() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            stream
+                .write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\n")
+                .await
+                .expect("fixture should write response headers");
+            for byte in b"RustFS!" {
+                stream.write_all(&[*byte]).await.expect("fixture should write body progress");
+                tokio::time::sleep(Duration::from_millis(20)).await;
+            }
+        });
+        let client = client_for_endpoint(
+            &endpoint,
+            TransitionClientTimeouts::new(Duration::from_millis(10), Duration::from_secs(1), Duration::from_millis(100)),
+        )
+        .await;
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let (_, _, mut reader) = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect("continuous body progress must not be killed by the idle timer");
+        let mut body = Vec::new();
+        reader
+            .read_to_end(&mut body)
+            .await
+            .expect("bounded response should be readable");
+
+        assert_eq!(body, b"RustFS!");
+        fixture.await.expect("fixture should join");
+    }
 }
 
 #[derive(Default)]

crates/s3-client/src/api_list.rs

@@ -27,7 +27,6 @@ use crate::{
     transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
 };
 use http::{HeaderMap, StatusCode};
-use http_body_util::BodyExt;
 use hyper::body::Body;
 use hyper::body::Bytes;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
@@ -124,14 +123,9 @@ impl TransitionClient {
         }
 
         //let mut list_bucket_result = ListBucketV2Result::default();
-        let mut body_vec = Vec::new();
-        let mut body = resp.into_body();
-        while let Some(frame) = body.frame().await {
-            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-            if let Some(data) = frame.data_ref() {
-                body_vec.extend_from_slice(data);
-            }
-        }
+        let body_vec = self
+            .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
+            .await?;
         let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
             Ok(result) => result,
             Err(err) => {
@@ -214,7 +208,9 @@ impl TransitionClient {
 
         let resp_status = resp.status();
         let headers = resp.headers().clone();
-        let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
+        let body = self
+            .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
+            .await?;
         if resp_status != StatusCode::OK {
             return Err(std::io::Error::other(http_resp_to_error_response(
                 resp_status,
@@ -428,6 +424,30 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{BucketLookupType, Options, TransitionClientTimeouts},
+    };
+    use std::time::Duration;
+    use tokio::{
+        io::{AsyncReadExt, AsyncWriteExt},
+        net::TcpListener,
+    };
+
+    fn timeout_test_options() -> Options {
+        Options {
+            creds: Credentials::new(Static(Value {
+                access_key_id: "access-key".to_string(),
+                secret_access_key: "secret-key".to_string(),
+                signer_type: SignatureType::SignatureV4,
+                ..Default::default()
+            })),
+            region: "us-east-1".to_string(),
+            bucket_lookup: BucketLookupType::BucketLookupPath,
+            max_retries: 1,
+            ..Default::default()
+        }
+    }
 
     #[test]
     fn list_versions_xml_preserves_versions_and_delete_markers() {
@@ -525,4 +545,56 @@ mod tests {
         assert_eq!(parsed.common_prefixes.len(), 1);
         assert_eq!(parsed.common_prefixes[0].prefix, "subdir/");
     }
+
+    #[tokio::test]
+    async fn list_objects_v2_body_stall_returns_timed_out() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one list request");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            stream
+                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 512\r\nConnection: close\r\n\r\n<ListBucketResult><Name>warm")
+                .await
+                .expect("fixture should write a partial list response");
+            tokio::time::sleep(Duration::from_millis(200)).await;
+        });
+        let client = TransitionClient::new_with_timeouts(
+            &endpoint,
+            timeout_test_options(),
+            "",
+            TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
+        )
+        .await
+        .expect("fixture client should build");
+        client
+            .bucket_loc_cache
+            .lock()
+            .expect("location cache should lock")
+            .set("bucket", "us-east-1");
+
+        let err = client
+            .list_objects_v2_query("bucket", "", "", false, false, "", "", 1, HeaderMap::new())
+            .await
+            .expect_err("a stalled ListObjectsV2 body must be bounded");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
+        fixture.await.expect("fixture should join");
+    }
 }

crates/s3-client/src/api_put_object_multipart.rs

@@ -18,7 +18,6 @@
 #![allow(clippy::all)]
 
 use http::{HeaderMap, HeaderName, StatusCode};
-use http_body_util::BodyExt;
 use hyper::body::Bytes;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
@@ -247,14 +246,9 @@ impl TransitionClient {
         // Parse the CreateMultipartUpload response for the UploadId. Returning a
         // default (empty) result here made every multipart transition fail at the
         // first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
-        let mut body_vec = Vec::new();
-        let mut body = resp.into_body();
-        while let Some(frame) = body.frame().await {
-            let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
-            if let Some(data) = frame.data_ref() {
-                body_vec.extend_from_slice(data);
-            }
-        }
+        let body_vec = self
+            .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
+            .await?;
         let initiate_multipart_upload_result =
             quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
                 .map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;

crates/s3-client/src/api_remove.rs

@@ -19,7 +19,6 @@
 #![allow(clippy::all)]
 
 use http::{HeaderMap, HeaderValue, Method, StatusCode};
-use http_body_util::BodyExt;
 use hyper::body::Body;
 use hyper::body::Bytes;
 use rustfs_utils::HashAlgorithm;
@@ -351,14 +350,9 @@ impl TransitionClient {
                 )
                 .await?;
 
-            let mut body_vec = Vec::new();
-            let mut body = resp.into_body();
-            while let Some(frame) = body.frame().await {
-                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-                if let Some(data) = frame.data_ref() {
-                    body_vec.extend_from_slice(data);
-                }
-            }
+            let body_vec = self
+                .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
+                .await?;
             process_remove_multi_objects_response(
                 ReaderImpl::Body(Bytes::from(body_vec)),
                 bucket_name,

crates/s3-client/src/api_stat.rs

@@ -19,7 +19,6 @@
 #![allow(clippy::all)]
 
 use http::{HeaderMap, HeaderValue, StatusCode};
-use http_body_util::BodyExt;
 use hyper::body::Body;
 use hyper::body::Bytes;
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
@@ -119,14 +118,9 @@ impl TransitionClient {
             let resp_status = resp.status();
             let h = resp.headers().clone();
 
-            let mut body_vec = Vec::new();
-            let mut body = resp.into_body();
-            while let Some(frame) = body.frame().await {
-                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-                if let Some(data) = frame.data_ref() {
-                    body_vec.extend_from_slice(data);
-                }
-            }
+            let body_vec = self
+                .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
+                .await?;
             let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
 
             warn!("bucket exists, resperr: {:?}", resperr);
@@ -170,11 +164,13 @@ impl TransitionClient {
                 let resp_status = resp.status();
                 let h = resp.headers().clone();
 
-                let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
+                let body_vec = self
+                    .collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
+                    .await?;
                 parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
             }
 
-            Err(err) => Err(std::io::Error::other(err)),
+            Err(err) => Err(err),
         }
     }
 
@@ -274,8 +270,14 @@ impl TransitionClient {
 #[cfg(test)]
 mod tests {
     use super::parse_bucket_versioning_response;
+    use crate::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts},
+    };
     use http::{HeaderMap, StatusCode};
     use s3s::dto::BucketVersioningStatus;
+    use std::time::Duration;
+    use tokio::{io::AsyncReadExt, net::TcpListener};
 
     #[test]
     fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
@@ -338,4 +340,63 @@ mod tests {
             assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
         }
     }
+
+    #[tokio::test]
+    async fn get_bucket_versioning_preserves_request_timeout_kind() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one versioning request");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            tokio::time::sleep(Duration::from_millis(200)).await;
+        });
+        let client = TransitionClient::new_with_timeouts(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+            TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
+        )
+        .await
+        .expect("fixture client should build");
+        client
+            .bucket_loc_cache
+            .lock()
+            .expect("location cache should lock")
+            .set("bucket", "us-east-1");
+
+        let err = client
+            .get_bucket_versioning("bucket")
+            .await
+            .expect_err("a stalled versioning request must time out");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
+        fixture.await.expect("fixture should join");
+    }
 }

crates/s3-client/src/bucket_cache.rs

@@ -26,7 +26,6 @@ use crate::{
     transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
 };
 use http::Request;
-use http_body_util::BodyExt;
 use hyper::StatusCode;
 use hyper::body::Body;
 use hyper::body::Bytes;
@@ -86,7 +85,7 @@ impl TransitionClient {
         let req = self.get_bucket_location_request(bucket_name)?;
 
         let mut resp = self.doit(req).await?;
-        location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
+        location = process_bucket_location_response(self, resp, bucket_name, &self.tier_type).await?;
         {
             if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
                 bucket_loc_cache.set(bucket_name, &location);
@@ -198,6 +197,7 @@ impl TransitionClient {
 }
 
 async fn process_bucket_location_response(
+    client: &TransitionClient,
     mut resp: http::Response<Incoming>,
     bucket_name: &str,
     tier_type: &str,
@@ -237,14 +237,9 @@ async fn process_bucket_location_response(
     }
     //}
 
-    let mut body_vec = Vec::new();
-    let mut body = resp.into_body();
-    while let Some(frame) = body.frame().await {
-        let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-        if let Some(data) = frame.data_ref() {
-            body_vec.extend_from_slice(data);
-        }
-    }
+    let body_vec = client
+        .collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
+        .await?;
     let mut location = "".to_string();
     if tier_type == "huaweicloud" {
         if let Ok(body_str) = String::from_utf8(body_vec) {

crates/s3-client/src/transition_api.rs

@@ -41,7 +41,7 @@ use http::{
     request::{Builder, Request},
 };
 use http_body::Body;
-use http_body_util::{BodyExt, LengthLimitError, Limited};
+use http_body_util::BodyExt;
 use hyper::body::Bytes;
 use hyper::body::Incoming;
 use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
@@ -67,10 +67,12 @@ use s3s::dto::Owner;
 use s3s::dto::ReplicationStatus;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
+use std::error::Error as StdError;
 use std::io::Cursor;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicI32, Ordering};
 use std::task::{Context, Poll};
+use std::time::Duration as StdDuration;
 use std::{
     collections::HashMap,
     sync::{Arc, Mutex},
@@ -79,28 +81,108 @@ use time::Duration;
 use time::OffsetDateTime;
 use tokio::io::BufReader;
 use tokio::io::{AsyncRead, AsyncReadExt};
-use tracing::{debug, error, warn};
+use tracing::{debug, error, trace, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
 const C_USER_AGENT: &str = "RustFS (linux; x86)";
 pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
+const EVENT_TIER_REMOTE_TRANSPORT: &str = "tier_remote_transport";
+const LOG_COMPONENT_S3_CLIENT: &str = "s3_client";
+const LOG_SUBSYSTEM_TIER: &str = "tier";
 
 const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
 
+fn response_body_exceeds_limit_error() -> std::io::Error {
+    std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
+}
+
+fn remote_tier_timeout_error(message: &'static str) -> std::io::Error {
+    std::io::Error::new(std::io::ErrorKind::TimedOut, message)
+}
+
+fn source_chain_has_io_kind(error: &(dyn StdError + 'static), kind: std::io::ErrorKind) -> bool {
+    let mut current = Some(error);
+    while let Some(error) = current {
+        if error
+            .downcast_ref::<std::io::Error>()
+            .is_some_and(|io_error| io_error.kind() == kind)
+        {
+            return true;
+        }
+        current = error.source();
+    }
+    false
+}
+
+fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io::Error {
+    if source_chain_has_io_kind(&err, std::io::ErrorKind::TimedOut) {
+        return remote_tier_timeout_error("remote tier connection timed out");
+    }
+    std::io::Error::other(err)
+}
+
+async fn next_response_body_data<B>(
+    mut body: Pin<&mut B>,
+    idle_timeout: Option<StdDuration>,
+) -> Result<Option<Bytes>, std::io::Error>
+where
+    B: Body<Data = Bytes>,
+    B::Error: Into<Box<dyn StdError + Send + Sync>>,
+{
+    let next_nonempty_data = async {
+        loop {
+            let Some(frame) = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await else {
+                return Ok(None);
+            };
+            let frame = frame.map_err(std::io::Error::other)?;
+            let Ok(data) = frame.into_data() else {
+                continue;
+            };
+            if !data.is_empty() {
+                return Ok(Some(data));
+            }
+        }
+    };
+
+    if let Some(idle_timeout) = idle_timeout {
+        tokio::time::timeout(idle_timeout, next_nonempty_data)
+            .await
+            .map_err(|_| remote_tier_timeout_error("remote tier response body stalled"))?
+    } else {
+        next_nonempty_data.await
+    }
+}
+
+async fn collect_response_body_inner<B>(
+    body: B,
+    limit: Option<usize>,
+    idle_timeout: Option<StdDuration>,
+) -> Result<Vec<u8>, std::io::Error>
+where
+    B: Body<Data = Bytes>,
+    B::Error: Into<Box<dyn StdError + Send + Sync>>,
+{
+    let mut body_vec = Vec::new();
+    let mut body = std::pin::pin!(body);
+    while let Some(data) = next_response_body_data(body.as_mut(), idle_timeout).await? {
+        let Some(new_len) = body_vec.len().checked_add(data.len()) else {
+            return Err(response_body_exceeds_limit_error());
+        };
+        if limit.is_some_and(|limit| new_len > limit) {
+            return Err(response_body_exceeds_limit_error());
+        }
+        body_vec.extend_from_slice(&data);
+    }
+    Ok(body_vec)
+}
+
 pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
 where
     B: Body<Data = Bytes>,
-    B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+    B::Error: Into<Box<dyn StdError + Send + Sync>>,
 {
-    let body = Limited::new(body, limit).collect().await.map_err(|err| {
-        if err.is::<LengthLimitError>() {
-            std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
-        } else {
-            std::io::Error::other(err)
-        }
-    })?;
-    Ok(body.to_bytes().to_vec())
+    collect_response_body_inner(body, Some(limit), None).await
 }
 
 const C_UNKNOWN: i32 = -1;
@@ -196,6 +278,62 @@ pub struct TransitionClient {
     pub trailing_header_support: bool,
     pub max_retries: i64,
     pub tier_type: String,
+    pub timeouts: TransitionClientTimeouts,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TransitionClientTimeouts {
+    pub connect_timeout: StdDuration,
+    pub request_timeout: StdDuration,
+    pub response_body_idle_timeout: StdDuration,
+}
+
+impl TransitionClientTimeouts {
+    pub const fn new(
+        connect_timeout: StdDuration,
+        request_timeout: StdDuration,
+        response_body_idle_timeout: StdDuration,
+    ) -> Self {
+        Self {
+            connect_timeout,
+            request_timeout,
+            response_body_idle_timeout,
+        }
+    }
+
+    fn validate(self) -> Result<Self, std::io::Error> {
+        if self.connect_timeout.is_zero() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "remote tier connect timeout must be greater than zero",
+            ));
+        }
+        if self.request_timeout.is_zero() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "remote tier request timeout must be greater than zero",
+            ));
+        }
+        if self.response_body_idle_timeout.is_zero() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "remote tier response body idle timeout must be greater than zero",
+            ));
+        }
+        Ok(self)
+    }
+}
+
+impl Default for TransitionClientTimeouts {
+    fn default() -> Self {
+        Self {
+            connect_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS),
+            request_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS),
+            response_body_idle_timeout: StdDuration::from_secs(
+                rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
+            ),
+        }
+    }
 }
 
 #[derive(Debug, Default)]
@@ -288,12 +426,28 @@ async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
 
 impl TransitionClient {
     pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
-        let client = Self::private_new(endpoint, opts, tier_type).await?;
-
-        Ok(client)
+        Self::private_new(endpoint, opts, tier_type, TransitionClientTimeouts::default()).await
     }
 
-    async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
+    /// Builds a transition client with explicit transport timeout budgets.
+    ///
+    /// [`Self::new`] keeps the historical constructor surface and uses the
+    /// production defaults from [`TransitionClientTimeouts::default`].
+    pub async fn new_with_timeouts(
+        endpoint: &str,
+        opts: Options,
+        tier_type: &str,
+        timeouts: TransitionClientTimeouts,
+    ) -> Result<TransitionClient, std::io::Error> {
+        Self::private_new(endpoint, opts, tier_type, timeouts).await
+    }
+
+    async fn private_new(
+        endpoint: &str,
+        opts: Options,
+        tier_type: &str,
+        timeouts: TransitionClientTimeouts,
+    ) -> Result<TransitionClient, std::io::Error> {
         if rustls::crypto::CryptoProvider::get_default().is_none() {
             // No default provider is set yet; try to install aws-lc-rs.
             // `install_default` can only fail if another thread races us and installs a provider
@@ -306,15 +460,19 @@ impl TransitionClient {
         }
 
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
+        let timeouts = timeouts.validate()?;
 
         let tls = build_tls_config().await?;
 
+        let mut http = HttpConnector::new();
+        http.enforce_http(false);
+        http.set_connect_timeout(Some(timeouts.connect_timeout));
         let https = hyper_rustls::HttpsConnectorBuilder::new()
             .with_tls_config(tls)
             .https_or_http()
             .enable_http1()
             .enable_http2()
-            .build();
+            .wrap_connector(http);
         let http_client = Client::builder(TokioExecutor::new()).build(https);
 
         let mut client = TransitionClient {
@@ -337,6 +495,7 @@ impl TransitionClient {
             trailing_header_support: opts.trailing_headers,
             max_retries: opts.max_retries,
             tier_type: tier_type.to_string(),
+            timeouts,
         };
 
         {
@@ -501,29 +660,43 @@ impl TransitionClient {
     }
 
     pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
-        let req_method;
-        let req_uri;
-        let resp;
         let http_client = self.http_client.clone();
-        {
-            req_method = req.method().clone();
-            req_uri = req.uri().clone();
-
-            debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
-            resp = http_client.request(req);
-        }
-        let resp = resp.await;
-        debug!("http_client url: {} {}", req_method, req_uri);
-        if let Err(err) = resp {
-            error!("http_client call error: {:?}", err);
-            return Err(std::io::Error::other(err));
-        }
-
+        let req_method = req.method().clone();
+        let resp = tokio::time::timeout(self.timeouts.request_timeout, http_client.request(req)).await;
         let resp = match resp {
-            Ok(r) => r,
-            Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
+            Ok(Ok(resp)) => resp,
+            Ok(Err(err)) => {
+                let err = transition_transport_error(err);
+                error!(
+                    event = EVENT_TIER_REMOTE_TRANSPORT,
+                    component = LOG_COMPONENT_S3_CLIENT,
+                    subsystem = LOG_SUBSYSTEM_TIER,
+                    method = %req_method,
+                    error_kind = ?err.kind(),
+                    "remote tier request failed"
+                );
+                return Err(err);
+            }
+            Err(_) => {
+                warn!(
+                    event = EVENT_TIER_REMOTE_TRANSPORT,
+                    component = LOG_COMPONENT_S3_CLIENT,
+                    subsystem = LOG_SUBSYSTEM_TIER,
+                    method = %req_method,
+                    timeout_ms = self.timeouts.request_timeout.as_millis(),
+                    "remote tier request timed out before response headers"
+                );
+                return Err(remote_tier_timeout_error("remote tier request timed out before response headers"));
+            }
         };
-        debug!(status = %resp.status(), "remote tier response received");
+        trace!(
+            event = EVENT_TIER_REMOTE_TRANSPORT,
+            component = LOG_COMPONENT_S3_CLIENT,
+            subsystem = LOG_SUBSYSTEM_TIER,
+            method = %req_method,
+            status = %resp.status(),
+            "remote tier response received"
+        );
 
         //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
         //debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
@@ -537,7 +710,15 @@ impl TransitionClient {
                 .and_then(|value| value.to_str().ok())
                 .unwrap_or_default()
                 .to_string();
-            warn!(status = %status, request_id, "remote tier request rejected");
+            warn!(
+                event = EVENT_TIER_REMOTE_TRANSPORT,
+                component = LOG_COMPONENT_S3_CLIENT,
+                subsystem = LOG_SUBSYSTEM_TIER,
+                method = %req_method,
+                status = %status,
+                request_id,
+                "remote tier request rejected"
+            );
         }
         Ok(resp)
     }
@@ -581,7 +762,9 @@ impl TransitionClient {
             let resp_status = resp.status();
             let h = resp.headers().clone();
 
-            let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?;
+            let body_vec = self
+                .collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE)
+                .await?;
             let parsed_error =
                 http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
             let routing_region = parsed_error.region;
@@ -635,6 +818,22 @@ impl TransitionClient {
         Err(std::io::Error::other("remote tier request did not produce a response"))
     }
 
+    pub async fn collect_response_body<B>(&self, body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
+    where
+        B: Body<Data = Bytes>,
+        B::Error: Into<Box<dyn StdError + Send + Sync>>,
+    {
+        collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await
+    }
+
+    pub async fn collect_response_body_unbounded<B>(&self, body: B) -> Result<Vec<u8>, std::io::Error>
+    where
+        B: Body<Data = Bytes>,
+        B::Error: Into<Box<dyn StdError + Send + Sync>>,
+    {
+        collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await
+    }
+
     async fn new_request(
         &self,
         method: &http::Method,
@@ -1504,12 +1703,17 @@ pub struct CreateBucketConfiguration {
 mod tests {
     use super::{
         MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
-        signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
+        collect_response_body_inner, signer_error_to_io_error, to_object_info_for_provider, validate_header_values,
+        with_rustls_init_guard,
     };
     use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
-    use http::{HeaderMap, HeaderValue};
-    use http_body_util::Full;
+    use futures::stream;
+    use http::{HeaderMap, HeaderValue, Request};
+    use http_body::Frame;
+    use http_body_util::{Full, StreamBody};
     use hyper::body::Bytes;
+    use std::time::Duration as StdDuration;
+    use tokio::net::TcpListener;
     use uuid::Uuid;
 
     #[tokio::test]
@@ -1540,6 +1744,77 @@ mod tests {
         assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
     }
 
+    #[tokio::test]
+    async fn empty_data_frames_do_not_reset_the_body_idle_timeout() {
+        let frames = stream::unfold((), |_| async {
+            tokio::time::sleep(StdDuration::from_millis(10)).await;
+            Some((Ok::<_, std::io::Error>(Frame::data(Bytes::new())), ()))
+        });
+        let body = StreamBody::new(Box::pin(frames));
+
+        let err = tokio::time::timeout(
+            StdDuration::from_millis(200),
+            collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))),
+        )
+        .await
+        .expect("the collector should enforce its own body idle timeout")
+        .expect_err("empty frames must not count as body progress");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
+    }
+
+    #[tokio::test]
+    async fn public_body_collector_accepts_non_unpin_bodies() {
+        let body = StreamBody::new(stream::once(async { Ok::<_, std::io::Error>(Frame::data(Bytes::from_static(b"ok"))) }));
+
+        let collected = collect_response_body(body, 2)
+            .await
+            .expect("the public collector should pin non-Unpin bodies internally");
+
+        assert_eq!(collected, b"ok");
+    }
+
+    #[tokio::test]
+    async fn https_endpoints_reach_the_transport_connector() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let accepted = tokio::spawn(async move {
+            let (stream, _) = tokio::time::timeout(StdDuration::from_secs(1), listener.accept())
+                .await
+                .expect("HTTPS connector should reach the TCP listener")
+                .expect("fixture should accept the HTTPS connection");
+            drop(stream);
+        });
+        let client = super::TransitionClient::new_with_timeouts(
+            &endpoint,
+            super::Options {
+                secure: true,
+                ..Default::default()
+            },
+            "",
+            super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::from_secs(1)),
+        )
+        .await
+        .expect("fixture client should build");
+        let request = Request::builder()
+            .uri(format!("https://{endpoint}/"))
+            .body(s3s::Body::empty())
+            .expect("fixture request should build");
+
+        client
+            .doit(request)
+            .await
+            .expect_err("the fixture closes before completing the TLS handshake");
+        accepted.await.expect("fixture should join");
+    }
+
     #[test]
     fn rustls_guard_converts_panics_to_io_errors() {
         let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") })
@@ -1573,6 +1848,18 @@ mod tests {
         assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
     }
 
+    #[test]
+    fn transition_timeouts_reject_zero_budgets() {
+        for timeouts in [
+            super::TransitionClientTimeouts::new(StdDuration::ZERO, StdDuration::from_secs(1), StdDuration::from_secs(1)),
+            super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::ZERO, StdDuration::from_secs(1)),
+            super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::ZERO),
+        ] {
+            let err = timeouts.validate().expect_err("zero timeout budgets must fail closed");
+            assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        }
+    }
+
     #[test]
     fn validate_header_values_returns_header_name_for_non_utf8_values() {
         let mut headers = HeaderMap::new();

crates/scanner/src/data_usage_define.rs

@@ -196,7 +196,7 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) struct DataUsageCacheRevisions {
     main: DataUsageCacheRevision,
     backup: Option<DataUsageCacheRevision>,
@@ -503,6 +503,10 @@ pub struct DataUsageCacheInfo {
     pub lkg_leader_epoch: Option<u64>,
     #[serde(default)]
     pub lkg_scan_plan_digest: Option<DataUsageScanPlanDigest>,
+    /// Activity-sensitive identity for same-cycle set snapshot reuse. The
+    /// structural plan remains reusable across ordinary bucket writes.
+    #[serde(default)]
+    pub scan_execution_digest: Option<DataUsageScanPlanDigest>,
 }
 
 impl Serialize for DataUsageCacheInfo {
@@ -519,7 +523,8 @@ impl Serialize for DataUsageCacheInfo {
             + usize::from(self.lkg_next_cycle.is_some())
             + usize::from(self.lkg_last_update.is_some())
             + usize::from(self.lkg_leader_epoch.is_some())
-            + usize::from(self.lkg_scan_plan_digest.is_some());
+            + usize::from(self.lkg_scan_plan_digest.is_some())
+            + usize::from(self.scan_execution_digest.is_some());
         let mut state = serializer.serialize_map(Some(field_count))?;
         state.serialize_entry("name", &self.name)?;
         state.serialize_entry("next_cycle", &self.next_cycle)?;
@@ -558,6 +563,9 @@ impl Serialize for DataUsageCacheInfo {
         if let Some(scan_plan_digest) = self.lkg_scan_plan_digest {
             state.serialize_entry("lkg_scan_plan_digest", &scan_plan_digest)?;
         }
+        if let Some(scan_execution_digest) = self.scan_execution_digest {
+            state.serialize_entry("scan_execution_digest", &scan_execution_digest)?;
+        }
         state.end()
     }
 }

crates/scanner/src/data_usage_define/tests.rs

@@ -1067,6 +1067,7 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
     assert!(decoded.source.is_none());
     assert!(!decoded.snapshot_complete);
     assert!(decoded.scan_plan_digest.is_none());
+    assert!(decoded.scan_execution_digest.is_none());
     assert_eq!(decoded.cache_key_format, 0);
 }
 
@@ -1109,6 +1110,7 @@ fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after()
     assert!(decoded.source.is_none());
     assert!(!decoded.snapshot_complete);
     assert!(decoded.scan_plan_digest.is_none());
+    assert!(decoded.scan_execution_digest.is_none());
     assert_eq!(decoded.cache_key_format, 0);
 }
 
@@ -1145,6 +1147,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
             source: Some(DataUsageCacheSource::new(1, 2)),
             snapshot_complete: true,
             scan_plan_digest: Some(TEST_PLAN_DIGEST),
+            scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])),
             cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
             ..Default::default()
         },
@@ -1164,6 +1167,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
     assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
     assert!(current.info.snapshot_complete);
     assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
+    assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32])));
     assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
     assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
 

crates/scanner/src/scanner.rs

@@ -1616,7 +1616,7 @@ where
     // Refresh the storage-owned movement snapshot before reading background
     // heal state. A missing heal object yields an in-memory default; do not
     // let that default influence a cycle while publication is blocked.
-    if storeapi.scanner_data_usage_publication_blocked().await {
+    if storeapi.scanner_data_movement_pause_status().await.paused {
         mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
     }
@@ -1816,6 +1816,19 @@ where
     let publication_defer_reason = publication_defer_reason
         .or(remote_lease_defer_reason)
         .or(remote_lease_fence_defer_reason);
+    // A PUT tail can finish between the walk and lease acquisition without
+    // changing the movement epoch accepted by those leases. Re-prove the
+    // namespace baseline only after every peer has granted publication.
+    let post_lease_activity_defer_reason = if publication_defer_reason.is_none()
+        && remote_publication_leases.is_some()
+        && let Ok(result) = &scan_result
+        && result.status == ScannerCycleStatus::Complete
+    {
+        scanner_post_lease_activity_defer_reason(result.activity_digest(), probe_scanner_activity(storeapi.as_ref(), true).await)
+    } else {
+        None
+    };
+    let publication_defer_reason = publication_defer_reason.or(post_lease_activity_defer_reason);
     // Include reasons discovered while acquiring or validating remote leases.
     let publication_deferred = publication_defer_reason.is_some();
     let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
@@ -3240,6 +3253,21 @@ where
     }
 }
 
+fn scanner_post_lease_activity_defer_reason(
+    expected_digest: Option<[u8; 32]>,
+    activity: Result<ScannerActivitySnapshot, String>,
+) -> Option<ScannerCycleDeferReason> {
+    match activity {
+        Ok(snapshot)
+            if scanner_activity_allows_usage_publication(&snapshot)
+                && expected_digest == Some(scanner_activity_snapshot_digest(&snapshot)) =>
+        {
+            None
+        }
+        Ok(_) | Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum ScannerCyclePreCommitOutcome {
     RecoverCacheCycle(u64),
@@ -3428,13 +3456,11 @@ use cycle_state::*;
 use leadership::*;
 use usage_store::*;
 
-#[cfg(test)]
-pub(crate) use activity::scanner_activity_snapshot_digest;
 pub use activity::scanner_topology_digest;
 pub(crate) use activity::{
     ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
-    scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
-    scanner_dirty_usage_acknowledgements,
+    scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
+    scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
 };
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
 pub use backlog::{

crates/scanner/src/scanner/activity.rs

@@ -902,7 +902,6 @@ where
     observation
 }
 
-#[cfg(test)]
 pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
     let mut hasher = Sha256::new();
     hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());

crates/scanner/src/scanner/tests.rs

@@ -15,7 +15,8 @@
 use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
 use super::*;
 use crate::EcstoreResult;
-use crate::storage_api::scan::BucketOperations as _;
+use crate::storage_api::owner::ecstore_hold_namespace_commit;
+use crate::storage_api::scan::{BucketOperations as _, ObjectIO as _};
 use crate::{
     DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_CACHE_KEY_FORMAT, DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT,
     DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntry, DataUsageScanPlanDigest, Endpoint, EndpointServerPools,
@@ -1165,6 +1166,116 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
     global_metrics().set_cycle(None).await;
 }
 
+#[tokio::test]
+#[serial]
+async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() {
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple());
+    store
+        .make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default())
+        .await
+        .expect("fixture bucket should be created");
+    let mut reader = PutObjReader::from_vec(b"first".to_vec());
+    store.pools[0].disk_set[0]
+        .put_object(
+            &bucket,
+            "object",
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("fixture object should finish its rename fanout");
+    crate::scanner_io::record_dirty_usage_bucket(&bucket);
+    let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests();
+    let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("fixture usage baseline should be readable");
+    let pending = ecstore_hold_namespace_commit(store.as_ref());
+    let ctx = CancellationToken::new();
+    let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
+    let mut cycle_info = CurrentCycle {
+        next: 1,
+        ..Default::default()
+    };
+    let mut revision = DataUsageCacheRevision::Missing;
+    let outcome = tokio::time::timeout(
+        Duration::from_secs(30),
+        run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget)),
+    )
+    .await
+    .expect("the coordinator must finish its namespace walk while a PUT is pending");
+    assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal");
+    assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle");
+    assert_eq!(revision, DataUsageCacheRevision::Missing);
+    assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
+    assert_eq!(
+        read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+            .await
+            .expect("the prior authoritative usage must remain readable"),
+        baseline,
+        "the pending candidate must not replace the authoritative baseline"
+    );
+
+    let committed_body = b"committed-after-walk";
+    let mut reader = PutObjReader::from_vec(committed_body.to_vec());
+    store.pools[0].disk_set[0]
+        .put_object(
+            &bucket,
+            "object",
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("the pending tail must change the physical object before it drains");
+    assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
+    drop(pending);
+    let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
+    let outcome = tokio::time::timeout(
+        Duration::from_secs(30),
+        run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget)),
+    )
+    .await
+    .expect("the same cycle must converge after the pending PUT drains");
+    assert_eq!(
+        retry_budget.progress().0,
+        1,
+        "the same-cycle retry must not reuse the pre-tail bucket cache"
+    );
+    assert!(matches!(
+        outcome,
+        ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
+    ));
+    assert_eq!(cycle_info.next, 2);
+    assert!(!crate::scanner_io::dirty_usage_buckets_for_tests().contains_key(&bucket));
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("the converged usage should be persisted");
+    let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("the persisted usage should decode");
+    assert_eq!(usage.usage_snapshot_converged, Some(true));
+    assert_eq!(usage.scanner_cycle, Some(1));
+    assert_eq!(usage.objects_total_count, 1);
+    assert_eq!(
+        usage.objects_total_size,
+        u64::try_from(committed_body.len()).expect("fixture body length")
+    );
+    let bucket_usage = usage
+        .buckets_usage
+        .get(&bucket)
+        .expect("the scanned bucket should be published");
+    assert_eq!(bucket_usage.objects_count, 1);
+    assert_eq!(bucket_usage.size, u64::try_from(committed_body.len()).expect("fixture body length"));
+    global_metrics().set_cycle(None).await;
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+}
+
 #[tokio::test]
 #[serial]
 async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
@@ -8485,6 +8596,66 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
     }
 }
 
+#[test]
+fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acquisition() {
+    let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
+    let expected_digest = Some(scanner_activity_snapshot_digest(&before));
+    assert_eq!(scanner_post_lease_activity_defer_reason(expected_digest, Ok(before.clone())), None);
+
+    let mut after = before.clone();
+    after
+        .get_mut("node-2")
+        .expect("writer should be present")
+        .namespace_generation += 1;
+    assert_eq!(
+        before["node-2"].movement_generation, after["node-2"].movement_generation,
+        "the existing movement-only lease remains valid after a PUT tail drains"
+    );
+    assert!(scanner_activity_allows_usage_publication(&after));
+    let reason = scanner_post_lease_activity_defer_reason(expected_digest, Ok(after));
+    assert_eq!(reason, Some(ScannerCycleDeferReason::ActivityBaselineUnavailable));
+
+    let result = ScannerCycleResult::new(ScannerCycleStatus::Complete, None).with_remote_dirty_usage_acknowledgements(vec![
+        ScannerDirtyUsageAcknowledgement {
+            host: "node-2".to_string(),
+            instance_id: "epoch-a".to_string(),
+            generation: 5,
+        },
+    ]);
+    let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
+        result,
+        DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")),
+    );
+    assert_eq!(
+        outcome,
+        ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+    );
+    assert!(
+        acknowledgements.is_empty(),
+        "a rejected publication must not acknowledge the peer's dirty usage"
+    );
+}
+
+#[test]
+fn post_lease_activity_proof_requires_a_complete_matching_baseline() {
+    let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
+    let digest = scanner_activity_snapshot_digest(&before);
+    let mut blocked = before.clone();
+    blocked.get_mut("node-2").expect("peer should be present").publication_blocked = true;
+    let blocked_digest = scanner_activity_snapshot_digest(&blocked);
+    for (expected, observed) in [
+        (None, Ok(before)),
+        (Some(digest), Err("peer is unavailable".to_string())),
+        (Some(digest), Ok(BTreeMap::new())),
+        (Some(blocked_digest), Ok(blocked)),
+    ] {
+        assert_eq!(
+            scanner_post_lease_activity_defer_reason(expected, observed),
+            Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+        );
+    }
+}
+
 #[test]
 fn scanner_activity_snapshot_digest_fences_storage_topology() {
     let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);

crates/scanner/src/scanner_io.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
+use crate::data_usage_define::{DATA_USAGE_CACHE_KEY_FORMAT, DataUsageCacheRevisions};
 use crate::scanner_budget::ScannerCycleBudget;
 use crate::scanner_folder::{ScannerItem, scan_data_folder};
 use crate::sleeper::SCANNER_SLEEPER;
@@ -271,6 +271,8 @@ pub struct ScannerBucketScanPlan {
     all_buckets: Arc<Vec<BucketInfo>>,
     scope: ScannerBucketScanScope,
     digest: DataUsageScanPlanDigest,
+    // Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
+    execution_digest: DataUsageScanPlanDigest,
     leader_epoch: u64,
     tier_registry_generation: u64,
     /// Epoch captured once for the whole scanner cycle.  `None` is retained
@@ -456,9 +458,12 @@ async fn scanner_cycle_activity_status<S>(
 where
     S: ScannerStorage,
 {
+    // Read the pending-commit barrier before sampling its completion generation.
+    // A tail that drains during this await must invalidate the earlier baseline.
+    let publication_blocked = store.scanner_data_usage_publication_blocked().await;
     match crate::scanner::probe_scanner_activity(store, distributed).await {
         Ok(after) => {
-            let status = if after == *before {
+            let status = if !publication_blocked && after == *before {
                 ScannerCycleActivityStatus::Unchanged
             } else {
                 ScannerCycleActivityStatus::Changed
@@ -760,6 +765,7 @@ fn scanner_activity_preflight(
 pub(crate) struct ScannerCycleResult {
     pub(crate) status: ScannerCycleStatus,
     publication_epoch: Option<u64>,
+    activity_digest: Option<[u8; 32]>,
     observational_snapshot_published: bool,
     dirty_usage_clear: Option<DirtyUsageBuckets>,
     remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
@@ -774,6 +780,7 @@ impl ScannerCycleResult {
         Self {
             status,
             publication_epoch: None,
+            activity_digest: None,
             observational_snapshot_published: false,
             dirty_usage_clear,
             remote_dirty_usage_acknowledgements: Vec::new(),
@@ -793,6 +800,15 @@ impl ScannerCycleResult {
         self.publication_epoch
     }
 
+    fn with_activity_digest(mut self, activity_digest: [u8; 32]) -> Self {
+        self.activity_digest = Some(activity_digest);
+        self
+    }
+
+    pub(crate) fn activity_digest(&self) -> Option<[u8; 32]> {
+        self.activity_digest
+    }
+
     pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self {
         self.observational_snapshot_published = published;
         self

crates/scanner/src/scanner_io/cache.rs

@@ -604,10 +604,12 @@ pub(super) async fn persist_and_publish_cache_snapshot(
     store: Arc<SetDisks>,
     updates: &mpsc::Sender<DataUsageCache>,
     mut cache_snapshot: DataUsageCache,
+    initial_revisions: Option<&DataUsageCacheRevisions>,
     cache_cycle_floor: &AtomicU64,
     expected_publication_epoch: u64,
 ) -> Option<SystemTime> {
     let source = cache_snapshot.info.source?;
+    let execution_digest = cache_snapshot.info.scan_execution_digest?;
     let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
         Ok(guard) => guard,
         Err(err) => {
@@ -672,20 +674,36 @@ pub(super) async fn persist_and_publish_cache_snapshot(
         );
         return None;
     }
-    if matches!(
-        current_cache_root_entry_with_generation(
-            &persisted,
-            DATA_USAGE_ROOT,
-            source,
-            cache_snapshot.info.next_cycle,
-            cache_snapshot.info.leader_epoch,
-            scan_plan_digest,
-            cache_snapshot.info.tier_registry_generation,
-        ),
-        Ok(Some(_))
-    ) {
+    if persisted.info.scan_execution_digest == Some(execution_digest)
+        && matches!(
+            current_cache_root_entry_with_generation(
+                &persisted,
+                DATA_USAGE_ROOT,
+                source,
+                cache_snapshot.info.next_cycle,
+                cache_snapshot.info.leader_epoch,
+                scan_plan_digest,
+                cache_snapshot.info.tier_registry_generation,
+            ),
+            Ok(Some(_))
+        )
+    {
         cache_snapshot = persisted;
     } else {
+        // A later execution may have completed while this scan was walking.
+        // Only replace the cache revision from which this scan started.
+        if initial_revisions != Some(&revisions) {
+            warn!(
+                target: "rustfs::scanner::io",
+                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_IO,
+                state = "scan_baseline_revision_changed",
+                cache_name = DATA_USAGE_CACHE_NAME,
+                "Scanner skipped set snapshot without an unchanged baseline revision"
+            );
+            return None;
+        }
         if guard.is_lock_lost() {
             error!(
                 target: "rustfs::scanner::io",

crates/scanner/src/scanner_io/io_cache.rs

@@ -118,6 +118,7 @@ impl ScannerIOCache for SetDisks {
             all_buckets,
             scope,
             digest: scan_plan_digest,
+            execution_digest,
             leader_epoch,
             tier_registry_generation,
             publication_epoch,
@@ -137,20 +138,24 @@ impl ScannerIOCache for SetDisks {
                 .ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
         };
         let mut old_cache = DataUsageCache::default();
-        if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
-            warn!(
-                target: "rustfs::scanner::io",
-                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_IO,
-                pool = self.pool_index,
-                set = self.set_index,
-                cache_name = DATA_USAGE_CACHE_NAME,
-                state = "old_cache_load_failed",
-                error = %e,
-                "Scanner old data usage cache load failed; rebuilding from bucket caches"
-            );
-        }
+        let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
+            Ok(revisions) => Some(revisions),
+            Err(e) => {
+                warn!(
+                    target: "rustfs::scanner::io",
+                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                    component = LOG_COMPONENT_SCANNER,
+                    subsystem = LOG_SUBSYSTEM_IO,
+                    pool = self.pool_index,
+                    set = self.set_index,
+                    cache_name = DATA_USAGE_CACHE_NAME,
+                    state = "old_cache_load_failed",
+                    error = %e,
+                    "Scanner old data usage cache load failed; rebuilding from bucket caches"
+                );
+                None
+            }
+        };
         let scoped_scan = prepare_scoped_set_scan(
             &old_cache,
             &buckets,
@@ -195,6 +200,7 @@ impl ScannerIOCache for SetDisks {
             };
             cache.info.last_update = Some(now);
             cache.info.snapshot_complete = true;
+            cache.info.scan_execution_digest = Some(execution_digest);
             cache.info.lkg_snapshot_complete = false;
             cache.info.lkg_next_cycle = None;
             cache.info.lkg_last_update = None;
@@ -208,6 +214,7 @@ impl ScannerIOCache for SetDisks {
                 self,
                 &updates,
                 cache,
+                initial_revisions.as_ref(),
                 cache_cycle_floor.as_ref(),
                 expected_publication_epoch,
             )
@@ -637,7 +644,7 @@ impl ScannerIOCache for SetDisks {
 
                     let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
                     let bucket_scan_plan_digest =
-                        scanner_bucket_cache_digest(scan_plan_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
+                        scanner_bucket_cache_digest(execution_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
 
                     if let Some(server_epoch) = remote_server_epoch {
                         let request_sequence = remote_session_sequence;
@@ -1360,6 +1367,7 @@ impl ScannerIOCache for SetDisks {
                 cache.info.next_cycle = want_cycle;
                 cache.info.last_update.get_or_insert_with(SystemTime::now);
                 cache.info.snapshot_complete = true;
+                cache.info.scan_execution_digest = Some(execution_digest);
                 cache.info.lkg_snapshot_complete = false;
                 cache.info.lkg_next_cycle = None;
                 cache.info.lkg_last_update = None;
@@ -1371,6 +1379,7 @@ impl ScannerIOCache for SetDisks {
                 self.clone(),
                 &updates,
                 cache_snapshot,
+                initial_revisions.as_ref(),
                 cache_cycle_floor.as_ref(),
                 expected_publication_epoch,
             )

crates/scanner/src/scanner_io/io_cycle.rs

@@ -180,7 +180,7 @@ where
     // canceled decommission remains suspended after its worker exits, so
     // starting a scan in that state could build a snapshot that cannot be
     // routed to the authoritative metadata object.
-    if store.scanner_data_usage_publication_blocked().await {
+    if store.scanner_data_movement_pause_status().await.paused {
         debug!(
             target: "rustfs::scanner::io",
             event = EVENT_SCANNER_SET_STATE,
@@ -260,8 +260,13 @@ where
         }
     }
     bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
+    let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
     let scan_plan_digest =
         scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
+    let mut execution_hasher = Sha256::new();
+    execution_hasher.update(scan_plan_digest.0);
+    execution_hasher.update(activity_digest);
+    let execution_digest = DataUsageScanPlanDigest(execution_hasher.finalize().into());
     let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
     let scan_scope = resolve_scanner_bucket_scan_scope(
         store,
@@ -326,6 +331,7 @@ where
         };
         return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
             .with_publication_epoch(publication_epoch)
+            .with_activity_digest(activity_digest)
             .with_observational_snapshot_published(observational_snapshot_published)
             .with_remote_publication_lease_targets(remote_publication_lease_targets)
             .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
@@ -410,6 +416,7 @@ where
             all_buckets: Arc::clone(&all_buckets),
             scope: scan_scope.clone(),
             digest: scan_plan_digest,
+            execution_digest,
             leader_epoch,
             tier_registry_generation,
             publication_epoch,
@@ -598,6 +605,7 @@ where
     };
     Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
         .with_publication_epoch(publication_epoch)
+        .with_activity_digest(activity_digest)
         .with_observational_snapshot_published(observational_snapshot_published)
         .with_remote_publication_lease_targets(remote_publication_lease_targets)
         .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)

crates/scanner/src/scanner_io/tests.rs

@@ -20,6 +20,7 @@ use crate::scanner_folder::ScannerItem;
 use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
 use crate::storage_api::owner::{
     EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
+    ecstore_hold_namespace_commit,
 };
 use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
 use crate::{
@@ -343,6 +344,16 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
             .put_object(&bucket, object, &mut reader, &ScannerObjectOptions::default())
             .await
             .expect("object should be written to its selected pool");
+
+        // Quorum ACK can precede tail publication on the disk chosen to scan.
+        let lock = store.pools[pool_index].disk_set[0]
+            .new_ns_lock(&bucket, object)
+            .await
+            .expect("fixture namespace lock should be created");
+        let _settled = lock
+            .get_write_lock(Duration::from_secs(30))
+            .await
+            .expect("fixture rename tail should finish before the usage scan");
     }
 
     let ctx = CancellationToken::new();
@@ -362,7 +373,7 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
         .buckets_usage
         .get(&bucket)
         .expect("combined bucket usage should be present");
-    assert_eq!(bucket_usage.objects_count, 2);
+    assert_eq!(bucket_usage.objects_count, 2, "{usage:?}");
     assert_eq!(bucket_usage.size, 11);
     assert_eq!(usage.objects_total_count, 2);
     assert_eq!(usage.objects_total_size, 11);
@@ -372,6 +383,102 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
     );
 }
 
+#[tokio::test]
+#[serial]
+async fn pending_put_commit_keeps_scanner_walk_live_without_authoritative_usage() {
+    let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+    let bucket = format!("scanner-pending-put-{}", Uuid::new_v4().simple());
+    store
+        .make_bucket(&bucket, &MakeBucketOptions::default())
+        .await
+        .expect("bucket should be created across both pools");
+    for (pool_index, (object, body)) in [("pool-a", b"first".as_slice()), ("pool-b", b"second".as_slice())]
+        .into_iter()
+        .enumerate()
+    {
+        let mut reader = ScannerPutObjReader::from_vec(body.to_vec());
+        store.pools[pool_index].disk_set[0]
+            .put_object(
+                &bucket,
+                object,
+                &mut reader,
+                &ScannerObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("fixture objects must finish their rename fanouts before scanning");
+    }
+
+    let mut pending = Some(ecstore_hold_namespace_commit(store.as_ref()));
+    let mut previous_activity_digest = None;
+    let mut structural_plan_digest = None;
+    for (cycle, converged) in [(1, false), (2, true)] {
+        if converged {
+            drop(pending.take());
+        }
+        assert_eq!(store.scanner_data_usage_publication_blocked().await, !converged);
+        assert!(!store.scanner_data_movement_pause_status().await.paused);
+        let activity = crate::scanner::probe_scanner_activity(store.as_ref(), false)
+            .await
+            .expect("the fixture activity should be observable");
+        let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity);
+        if let Some(previous) = previous_activity_digest.replace(activity_digest) {
+            assert_ne!(previous, activity_digest, "draining a namespace commit must change the publication proof");
+        }
+        let ctx = CancellationToken::new();
+        let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
+        let (updates, mut receiver) = mpsc::channel(1);
+        let result = tokio::time::timeout(
+            Duration::from_secs(30),
+            ScannerIOCycle::nsscanner_with_status(
+                store.as_ref(),
+                ctx,
+                Arc::clone(&budget),
+                updates,
+                cycle,
+                1,
+                HealScanMode::Normal,
+            ),
+        )
+        .await
+        .expect("namespace scanning must finish while a PUT commit is pending")
+        .expect("namespace scanning must remain available during a pending PUT commit");
+        assert_eq!(result.activity_digest(), Some(activity_digest));
+        if !converged {
+            assert_eq!(budget.progress().0, 2, "the pending commit must not suppress actual object traversal");
+        }
+        assert_eq!(
+            result.status,
+            if converged {
+                ScannerCycleStatus::Complete
+            } else {
+                ScannerCycleStatus::Superseded
+            }
+        );
+        let usage = receiver
+            .recv()
+            .await
+            .expect("the completed walk should produce a usage candidate");
+        assert_eq!(usage.usage_snapshot_converged, Some(converged));
+        assert_eq!(usage.scanner_cycle, Some(cycle));
+        assert_eq!(usage.objects_total_count, 2);
+        assert_eq!(usage.objects_total_size, 11);
+        assert_eq!(usage.usage_snapshot_set_states.len(), 2);
+        for state in &usage.usage_snapshot_set_states {
+            let digest = state
+                .scan_plan_digest
+                .expect("each set must retain its structural cache identity");
+            assert_eq!(*structural_plan_digest.get_or_insert(digest), digest);
+        }
+        let bucket_usage = usage.buckets_usage.get(&bucket).expect("the walked bucket must be present");
+        assert_eq!(bucket_usage.objects_count, 2);
+        assert_eq!(bucket_usage.size, 11);
+        assert!(receiver.recv().await.is_none(), "each walk must emit exactly one terminal candidate");
+    }
+}
+
 #[tokio::test]
 #[serial]
 async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
@@ -387,6 +494,16 @@ async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() {
         .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default())
         .await
         .expect("object should be written only to the second pool");
+    {
+        let lock = store.pools[1].disk_set[0]
+            .new_ns_lock(&bucket, "pool-b")
+            .await
+            .expect("fixture namespace lock should be created");
+        let _settled = lock
+            .get_write_lock(Duration::from_secs(30))
+            .await
+            .expect("fixture rename tail should finish before the usage scan");
+    }
     store.pools[0]
         .delete_bucket(&bucket, &DeleteBucketOptions::default())
         .await
@@ -797,6 +914,124 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
     cache
 }
 
+#[tokio::test]
+#[serial]
+async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
+    let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+    let set = Arc::clone(&store.pools[0].disk_set[0]);
+    let epoch = scanner_publication_epoch(Arc::clone(&set)).await.expect("idle set admission");
+    let mut legacy = complete_set_usage_cache(&[("photos", 5)], DataUsageScanPlanDigest([1; 32]));
+    legacy.info.source = Some(DataUsageCacheSource::new(0, 0));
+    legacy
+        .save(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
+        .await
+        .expect("seed legacy set cache");
+    let mut persisted = DataUsageCache::default();
+    let initial = persisted
+        .load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
+        .await
+        .expect("capture the shared starting revision");
+    let mut fresh = legacy.clone();
+    fresh.info.scan_execution_digest = Some(DataUsageScanPlanDigest([2; 32]));
+    fresh.replace(
+        "photos",
+        DATA_USAGE_ROOT,
+        DataUsageEntry {
+            size: 20,
+            objects: 1,
+            ..Default::default()
+        },
+    );
+    let cycle_floor = AtomicU64::new(fresh.info.next_cycle);
+    let (tx, mut rx) = mpsc::channel(1);
+    assert!(
+        persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh.clone(), Some(&initial), &cycle_floor, epoch)
+            .await
+            .is_some(),
+        "a legacy cache without execution identity must be refreshed"
+    );
+    let published = rx.try_recv().expect("fresh snapshot should be forwarded");
+    assert_eq!(published.find("photos").expect("published bucket").size, 20);
+    assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest);
+    let current = persisted
+        .load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
+        .await
+        .expect("capture the current revision for the unidentified execution");
+
+    let mut stale = legacy.clone();
+    stale.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
+    for (candidate, revisions) in [(stale, &initial), (legacy, &current)] {
+        assert!(
+            persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(revisions), &cycle_floor, epoch)
+                .await
+                .is_none(),
+            "a stale or unidentified execution must not replace the newer snapshot"
+        );
+        assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)));
+    }
+    fresh.info.scan_execution_digest = Some(DataUsageScanPlanDigest([4; 32]));
+    assert!(
+        persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh.clone(), None, &cycle_floor, epoch)
+            .await
+            .is_none(),
+        "an unreadable starting revision must not authorize an overwrite"
+    );
+
+    fresh.info.scan_execution_digest = published.info.scan_execution_digest;
+    fresh.replace("photos", DATA_USAGE_ROOT, DataUsageEntry::default());
+    assert!(
+        persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, fresh, Some(&initial), &cycle_floor, epoch)
+            .await
+            .is_some(),
+        "an overlapping identical execution must reuse the completed snapshot"
+    );
+    assert_eq!(
+        rx.try_recv()
+            .expect("reused snapshot")
+            .find("photos")
+            .expect("reused bucket")
+            .size,
+        20
+    );
+    persisted
+        .load(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
+        .await
+        .expect("read the final durable set cache");
+    assert_eq!(persisted.find("photos").expect("durable bucket").size, 20);
+    assert_eq!(persisted.info.scan_execution_digest, published.info.scan_execution_digest);
+
+    let ctx = CancellationToken::new();
+    let empty_execution = DataUsageScanPlanDigest([5; 32]);
+    set.nsscanner_cache(
+        ctx.clone(),
+        ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()),
+        ScannerBucketScanPlan {
+            buckets: Vec::new(),
+            all_buckets: Arc::new(Vec::new()),
+            scope: ScannerBucketScanScope::default(),
+            digest: DataUsageScanPlanDigest([6; 32]),
+            execution_digest: empty_execution,
+            leader_epoch: 11,
+            tier_registry_generation: 13,
+            publication_epoch: Some(epoch),
+            dirty_usage_buckets: Arc::new(HashMap::new()),
+            bucket_failures: ScannerBucketFailureState::default(),
+            pending_maintenance_work: Arc::new(AtomicBool::new(false)),
+            cache_cycle_floor: Arc::new(AtomicU64::new(8)),
+        },
+        tx,
+        8,
+        HealScanMode::Normal,
+    )
+    .await
+    .expect("empty set scope should replace its prior nonempty cache");
+    let empty = rx.try_recv().expect("empty set snapshot should be published");
+    assert_eq!(empty.info.scan_execution_digest, Some(empty_execution));
+    assert!(empty.info.snapshot_complete);
+    let root = empty.checked_flatten(DATA_USAGE_ROOT).expect("complete empty root");
+    assert_eq!((root.size, root.objects), (0, 0));
+}
+
 fn complete_usage_baseline(
     source: DataUsageCacheSource,
     scan_plan_digest: DataUsageScanPlanDigest,

crates/scanner/src/storage_api.rs

@@ -127,6 +127,9 @@ pub(crate) use rustfs_lifecycle::{
 use rustfs_storage_api as storage_contracts;
 
 pub(crate) mod owner {
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::set_disk::test_util::hold_namespace_commit as ecstore_hold_namespace_commit;
+
     pub(crate) use super::storage_contracts::{
         HTTPPreconditions, HTTPRangeSpec, NS_SCANNER_PROTOCOL_VERSION, ObjectIO, ObjectOperations, ObjectToDelete,
     };

docs/architecture/background-services-inventory.md

@@ -13,8 +13,8 @@ Operator-facing behaviour, configuration, and troubleshooting for these services
 
 | Service | Desired source | Current-status inputs | Status surface | Side effects |
 |---|---|---|---|---|
-| Write-back pull pipeline (`crates/ecstore/src/bucket/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `crates/ecstore/src/bucket/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
-| Backfill job (module under `crates/ecstore/src/bucket/on_demand_migration/`, rustfs/backlog#2159 — not yet in the tree) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
-| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159 — not yet in the tree) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |
+| Write-back pull pipeline (`rustfs/src/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `rustfs/src/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
+| Backfill job (module under `rustfs/src/on_demand_migration/`, rustfs/backlog#2159) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
+| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |
 
 The pull pipeline has no separate loop of its own: a bucket's queue dispatcher starts lazily on the first background pull and is cancelled when the bucket's state is rebuilt or removed, and each inline pull commits in a task that outlives its request so a client disconnect cannot truncate the stored object. Neither the switch nor the config is re-read by the workers: the bucket-metadata publish hook rebuilds the state, which is the only desired-state path.

docs/architecture/compat-cleanup-register.md

@@ -11,6 +11,7 @@
 
 ## Open Items
 
+- `backlog-2263` legacy heal MRF inspection: retained per-record journals remain readable while committed-snapshot ownership and writer activation are staged. Remove legacy import only after all supported direct-upgrade and rollback readers understand committed snapshots and migration tooling confirms that no retained or restorable legacy journal requires it. This does not enable a new writer or change the automatic legacy consumer.
 - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
 - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
 - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork.

docs/architecture/crate-boundaries.md

@@ -84,3 +84,41 @@ The server-config model (`Config`, `KV`, `KVS`) and the global server-config sna
 ## Required Architecture Documents
 
 The guard requires the documents and section headings listed in its `require_source_contains` entries (`scripts/check_architecture_migration_rules.sh`); the directory index is [README.md](README.md).
+
+## On-Demand Migration Service
+
+Read-through, backfill and external pull orchestration belong in an application
+service under `rustfs/src/<service>/`. ECStore owns the storage primitives they
+need, including atomic commits, lifecycle locks and on-disk metadata. A service
+may use these primitives without moving its provider clients or scheduling
+policy into the engine.
+
+`rustfs/src/on_demand_migration/` owns source clients, pull scheduling, list
+merging, runtime state and backfill orchestration. Its `storage_api.rs` is the
+only ECStore facade boundary. Object write-back still enters the application's
+internal PUT and multipart use cases, including the atomic create-only commit,
+delete-marker protection, encryption, quota and notification rules.
+
+ECStore stores the existing ODM bytes and update timestamp without interpreting
+the JSON. Every metadata cache install or removal publishes those bytes through
+`BUCKET_CONFIG_PUBLISH_HOOK`; the application decodes them and synchronously
+withdraws corrupt configurations. Configuration writes validate structure and
+deployment constraints in the admin use case before the incarnation-fenced
+metadata update. Backfill reads metadata from its store's instance context and
+preserves the checkpoint ETag compare-and-set, lease and tail-drained writes.
+
+An ODM runtime is bound to the bucket incarnation published with its metadata,
+not just its name. Source reads and write-back reject a different incarnation.
+Checkpoint writes hold the user bucket's lifecycle fence through their complete
+commit and read-back, even if their caller stops waiting; the storage commit
+also observes lock loss. Deleting and recreating a bucket must not let work for
+its previous incarnation repopulate objects or checkpoints.
+
+Observability owns its metric DTOs and accepts application snapshot callbacks;
+it does not depend on the ODM runtime. The application registers both bucket
+and backfill snapshots during startup, before metadata and metric collection.
+
+This boundary does not change `.metadata.bin`, the ODM wire format or the
+backfill checkpoint format. An older binary may still discard unknown metadata
+fields when it rewrites a bucket; service relocation does not make mixed-version
+configuration writes or rollback preserve ODM configuration.

docs/architecture/ecstore-api-facade-inventory.md

@@ -63,3 +63,10 @@ Lifecycle, replication, and `SetDisks` split blockers, extracted contracts, and
 4. Do not replace `SetDisks` with multiple runtime structs in one change; move one operation family only after contracts and focused tests exist.
 5. Remove or narrow one facade group per change so rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility.
 6. Keep `api::bucket`, `api::config`, `api::disk`, and `api::tier` on explicit submodules and symbol lists; do not restore `pub use crate::<owner>::{...}` whole-module passthroughs for those groups.
+
+### On-Demand Migration
+
+`rustfs/src/on_demand_migration/storage_api.rs` owns the service's storage facade
+imports: opaque bucket configuration, shared remote S3 client construction,
+namespace locking, object options and metadata-object persistence. ODM types
+are owned by the application and are no longer exported through ECStore.

docs/architecture/remote-credential-sealing-adr.md

@@ -1,7 +1,7 @@
 # Remote Credential Sealing ADR
 
 **Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another.
-**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `crates/ecstore/src/bucket/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
+**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `rustfs/src/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
 
 ## Recommendation
 
@@ -37,7 +37,7 @@ Two of the three are not files at all. `bucket-targets.json` and `on-demand-migr
 | Store | Reached as | Actually persisted at | Written by | Container |
 |---|---|---|---|---|
 | Replication and ILM targets | `BUCKET_TARGETS_FILE` | `BucketMetadata::bucket_targets_config_json`, msgpack field `BucketTargetsConfigJSON` | `BucketMetadata::update_config`, then `BucketMetadata::save_with_store`; `crates/ecstore/src/bucket/metadata_sys.rs` serializes the update under a transaction lock | `{BUCKET_META_PREFIX}/{bucket}/{BUCKET_METADATA_FILE}` in `RUSTFS_META_BUCKET` (`crates/ecstore/src/disk/mod.rs`) |
-| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; `update_config` additionally refuses a blob this build cannot parse | same blob as above |
+| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; the application validates structure and deployment constraints before persistence | same blob as above |
 | Remote tiers | `TIER_CONFIG_FILE` | its own object, a four-byte `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header followed by an `rmp_serde` payload of `ExternalTierConfigMgr` | `TierConfigMgr` through `encode_external_tiering_config_blob`, under `tier_config_lock_path` | `tier_config_path` under `CONFIG_PREFIX` in `RUSTFS_META_BUCKET` |
 
 The consequence of the first two sharing a blob is that any change to how that blob parses has a blast radius covering policy, lifecycle, versioning, object lock and everything else in `BucketMetadata` — not just credentials.
@@ -48,7 +48,7 @@ Three things hold the line today, and all three keep working whether or not seal
 
 - **The reserved bucket.** `RUSTFS_META_BUCKET` is `.rustfs.sys`; `is_reserved_or_invalid_bucket` keeps it off the S3 surface, and the admin inspect archive in `rustfs/src/admin/handlers/inspect_archive.rs` runs its request through a strict bucket-name check that a dot-prefixed reserved name does not pass.
 - **Admin authorization** on every route that can read or write one of the three configurations.
-- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
+- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `rustfs/src/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
 
 So no API returns a stored secret. The bytes are reachable by reading the drives, and that is the boundary sealing is proposed to move.
 
@@ -74,7 +74,7 @@ The envelope deliberately does **not** carry its own scope. A scope read out of
 
 ## Why a hook instead of a dependency
 
-`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
+`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `BUCKET_CONFIG_PUBLISH_HOOK` in `crates/ecstore/src/bucket/metadata_sys.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
 
 ## Compatibility, per store, because the three differ
 

docs/architecture/scanner-usage-publication.md

@@ -23,6 +23,30 @@ therefore has three identities:
 If any identity changes before commit, the result is a candidate for retry or
 observation, not an authoritative baseline.
 
+Ordinary PUT rename fanouts also track instance-scoped in-flight work. A quorum
+ACK does not release it: the actual disk tasks retain ownership until their
+rename work ends, including when the request caller is cancelled. Scan admission
+remains movement-only so sustained PUTs do not stop namespace walks and
+scanner-driven lifecycle discovery. The post-walk local publication check and
+remote publication leases reject pending fanouts. Begin/end namespace generations
+invalidate scans and cached plans across the fanout; after acquiring remote
+leases, the coordinator rechecks the full activity digest before publishing an
+authoritative aggregate. This catches a tail that finishes between the scan's
+last probe and lease acquisition.
+
+This adds no namespace or movement lock. An already-verified older snapshot may
+still precede a newly started write. Sustained or stalled PUT tails can delay
+authoritative usage publication, which resumes through the existing retry
+schedule rather than a new immediate-wakeup protocol. Intermediate per-set and
+prefix cache readers retain their existing approximate-cache semantics. A
+prolonged pending tail with no generation changes can also delay cycle advancement
+and fresh rescans of already-current caches; this is not a guarantee of lifecycle
+progress under indefinitely stalled storage I/O.
+
+This PUT-tail protection requires every writer node to be upgraded. It does not
+prove that a failed tail replica has healed, and it does not extend the same
+in-flight tracking to multipart or other namespace mutation paths.
+
 ## Fences
 
 The protocol uses separate fences because they exclude different stale inputs.
@@ -33,7 +57,7 @@ They must not be collapsed unless the replacement proves the same exclusions.
 | Scanner leadership claim | scanner | competing scanner leaders and stale cycle writers |
 | Storage publication epoch | ECStore | usage computed across rebalance, decommission, or other data-movement generations |
 | Publication lease | scanner peers through ECStore-facing activity probes | remote dirty-usage or maintenance state that has not acknowledged the candidate |
-| CAS revision | backing config object store | lost updates to `.usage.v2.json`, `.usage.json`, or cycle-state objects |
+| CAS revision | backing config object store | lost updates to usage snapshots, scanner caches, or cycle-state objects |
 | Per-set freshness | scanner aggregation | a merged usage snapshot that combines stale and current set results |
 | Tier registry generation | scanner tier accounting | bytes classified against a different warm-tier registry |
 | Usage floor identity | scanner publication and ECStore quota fallback | empty or legacy values becoming plausible authoritative quota input |
@@ -42,6 +66,28 @@ A reader that cannot prove the required fence for its surface must fail closed
 or use the documented observed path below. It must not synthesize an empty usage
 snapshot for a missing or corrupt authoritative object.
 
+## Cache Execution Identity
+
+The structural scan-plan digest can remain stable across ordinary bucket writes
+so a scoped scan can retain unaffected baseline buckets. It is not sufficient
+proof for reusing a completed result within the same cycle. Bucket work uses an
+execution digest combining the structural plan and the full activity snapshot,
+with the bucket's dirty generation included in its cache identity. Completed set
+caches carry the same execution digest separately from their structural plan.
+The persisted set-root fast path requires equal execution identities as well as
+the existing source, cycle, leader, tier, and cache-structure checks.
+
+A set scan also captures its starting cache revisions. When the persisted
+execution differs, replacement requires those revisions to remain unchanged;
+otherwise a slow scan could overwrite a newer completed result. The existing
+cache lock, conditional save, and movement admission still fence the commit.
+
+The optional `scan_execution_digest` field is appended to the map-encoded cache
+metadata. Legacy caches remain readable but cannot satisfy same-cycle set-root
+reuse without this identity. Older readers can ignore the added map key, but
+older writers do not enforce its fence; readability is not a mixed-version
+publication-safety guarantee.
+
 ## Persisted Objects
 
 The persisted objects are part of the compatibility contract. Removing one

docs/operations/bucket-metadata-recovery.md

@@ -0,0 +1,22 @@
+# Bucket metadata diagnostics and recovery
+
+`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.
+
+To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:
+
+- Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`.
+- Readable entries under `_diagnostic/<bucket>/<config>`; target credentials remain redacted.
+- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details.
+
+`complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations.
+
+## Recover unreadable replication targets
+
+RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support.
+
+1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
+2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
+3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully.
+4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it.
+
+Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive.

docs/operations/on-demand-migration.md

@@ -1,12 +1,36 @@
 # On-Demand Migration
 
 **Use this when:** you are moving an existing S3-compatible bucket into RustFS without a stop-the-world copy, or you are debugging a bucket that serves reads from an external source (424 `SourceUnavailable`, an open circuit breaker, missing pulled objects, a source 403).
-**Source of truth:** `crates/ecstore/src/bucket/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.
+**Source of truth:** `rustfs/src/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.
 
 On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** to a local RustFS bucket. When a client GETs a key that does not exist locally, RustFS fetches it from the source, streams it to the client, and stores it locally in the same pass; every later read is served locally. It is a pull-style, lazy migration path — the RustFS equivalent of Cloudflare R2 Sippy, Tigris shadow buckets, and Alibaba Cloud OSS / Tencent COS mirror-back-to-origin.
 
 The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
 
+## Upgrade and rollback compatibility
+
+ODM configuration is stored in two additional keys in the existing bucket metadata map. The metadata format version remains `1` for MinIO compatibility. RustFS `1.0.0-rc.5` only re-encodes its 44 known keys: a bucket configuration write through an rc.5 node discards the ODM configuration and timestamp, even if another node originally wrote them. Restarting a newer binary cannot recover the discarded values. This also means ODM is not supported during a rolling upgrade that still allows rc.5 nodes to write bucket metadata.
+
+Upgrade every node before enabling ODM. Before any rollback to rc.5, stop new migration work, retain a secure copy of the original full configuration and credentials, and disable ODM on every bucket and node. The redacted configuration GET and metadata export are not credential backups. Objects still present only at the source cannot be read through RustFS while ODM is disabled or rc.5 is running; finish migration first, redirect those reads to the source, or plan a maintenance window. After all nodes return to a compatible release, reapply and validate the saved configuration; already stored local objects remain local. Turning the global module switch off alone does not make an old metadata writer preserve these keys.
+
+The ignored `upgrade_compatibility_test::rc5_rollback_requires_restoring_odm_configuration` test pins release commit `40a2470feb567201165a5b809b7598bb4b1f68f5`, restarts against the same data directory, writes bucket tags through rc.5, and verifies configuration recovery after returning to the current binary. Set `RUSTFS_UPGRADE_SOURCE_BINARY` to that release's executable and run `cargo test -p e2e_test rc5_rollback_requires_restoring_odm_configuration -- --ignored --test-threads=1`. The test records a known old-writer limitation; it does not certify mixed-version ODM operation.
+
+## Optional Google dependencies
+
+The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature.
+
+Both builds can read, redact and preserve GCS configuration. A build without `gcs` rejects native ODM client construction with `OnDemandMigrationBackendNotCompiled` (HTTP 501); persisted native sources report an unavailable client. GCS tier initialization returns `XRustFSAdminTierTypeUnsupported` (HTTP 501). Do not deploy that build to a cluster with GCS tiers containing transitioned objects: the configuration remains intact, but reading their remote data requires a GCS-capable binary. The `gcs` provider using HMAC credentials and the S3 interoperability API remains available in every build; only `gcs_native` and native GCS tier clients need the feature.
+
+## List continuation token rollout
+
+`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.
+
+Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled.
+
+An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
+
+For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear.
+
 ## Positioning
 
 | Capability | Direction | What it moves | Where the authoritative copy is | When to use it instead |
@@ -79,26 +103,34 @@ Setting `"enabled": false` in the config has the same read-path effect as deleti
 
 The status endpoint reports **the node that answered the request**. Counters, queue depth and breaker state are per-node runtime state, so in a distributed deployment query every node; the saved configuration and `updated_at` are cluster-wide.
 
-### Backfill (ships with ODM-12)
+### Backfill
 
-Read-through only migrates what clients touch. The background backfill job walks the source listing and pulls the remainder, with a persisted checkpoint (`.rustfs.sys/buckets/<bucket>/on-demand-migration-backfill.json`), a single-owner lease, resume after restart, and `POST .../{bucket}/backfill?op=start|cancel` plus `GET .../{bucket}/backfill` admin routes. That slice (rustfs/backlog#2159) is not part of the build this page was written against: the shape above is the agreed design, and the exact request/response bodies must be re-checked against `docs/architecture/admin-route-action-snapshot.md` once it lands.
+Backfill waits for the result of every pull, including a pull already queued by an online request. A failed or cancelled shared pull is counted as a failure, never as successful migration. The persisted continuation cursor stays at the first failed page; a takeover replays from there and skips objects already present locally. `completed_with_failures` is not a cutover-ready state.
+
+Read-through only migrates what clients touch. The background backfill job walks the source listing and pulls the remainder, with a persisted checkpoint (`.rustfs.sys/buckets/<bucket>/on-demand-migration-backfill.json`), a single-owner lease, resume after restart, and `POST .../{bucket}/backfill?op=start|cancel` plus `GET .../{bucket}/backfill` admin routes. See `docs/architecture/admin-route-action-snapshot.md` for the route contract.
 
 ## Configuration reference
 
-The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `crates/ecstore/src/bucket/on_demand_migration/config.rs`.
+The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `rustfs/src/on_demand_migration/config.rs`.
 
 | Field | Type | Default | Bounds / rules |
 |---|---|---|---|
 | `version` | integer | `1` | Must be `1` |
 | `enabled` | bool | `true` | `false` keeps the config but stops all source traffic |
-| `source.provider` | `s3` \| `aws` \| `minio` \| `rustfs` \| `r2` \| `gcs` | — (required) | Drives endpoint and addressing defaults |
-| `source.endpoint` | string \| null | — | `http(s)://host[:port]`, no path, query, fragment or userinfo. Required for every provider except `aws`, where it is derived from `region` |
-| `source.region` | string | — (required) | Non-empty. `auto` is accepted only for `r2`, `minio`, `rustfs` and is signed as `us-east-1` |
-| `source.bucket` | string | — (required) | Non-empty, no `/` and no whitespace |
+| `source.provider` | `s3` \| `aws` \| `minio` \| `rustfs` \| `r2` \| `gcs` \| `azure` \| `gcs_native` | — (required) | Drives endpoint and addressing defaults, and which backend the client builds: every value but `azure` and `gcs_native` speaks S3 |
+| `source.endpoint` | string \| null | — | `http(s)://host[:port]`, no path, query, fragment or userinfo. Required except for `aws` (derived from `region`), `azure` (derived as `https://<account>.blob.core.windows.net`) and `gcs_native` (`https://storage.googleapis.com`). Set it explicitly to point at Azurite or fake-gcs-server, subject to the same outbound policy as any other source endpoint |
+| `source.region` | string | — (required) | Non-empty. `auto` is accepted for `r2`, `minio`, `rustfs` and for the native providers, and is signed as `us-east-1`. `azure` and `gcs_native` never sign with a region, so `auto` is the honest value there |
+| `source.bucket` | string | — (required) | Non-empty, no `/` and no whitespace. For `azure` this is the container name, for `gcs_native` the bucket name; the provider block never repeats it |
 | `source.path_style` | `auto` \| `path` \| `virtual` | `auto` | `auto` resolves to path-style for IP-literal or `localhost` endpoints and for `s3`/`minio`/`rustfs`; virtual-host for `aws`/`gcs`/`r2` |
-| `source.credentials` | object \| null | `null` | `null` means anonymous, which the client builder does not support yet: the admin `PUT` refuses it with `InvalidArgument`, and a config that reached the metadata another way resolves as unavailable. `access_key` and `secret_key` must be non-empty; `session_token` is optional but must be non-empty when present |
+| `source.credentials` | object \| null | `null` | Read only by the S3 providers; `azure` and `gcs_native` must leave it `null` and carry their credentials in their own block. `null` means anonymous, which the client builder does not support yet: the admin `PUT` refuses it with `InvalidArgument`, and a config that reached the metadata another way resolves as unavailable. `access_key` and `secret_key` must be non-empty; `session_token` is optional but must be non-empty when present |
 | `source.tls.skip_verify` | bool | `false` | Disables certificate verification for the source connection |
 | `source.tls.ca_cert_pem` | string \| null | `null` | Must contain `-----BEGIN CERTIFICATE-----` |
+| `source.azure` | object \| null | `null` | Required for `provider = "azure"` and rejected for every other provider |
+| `source.azure.account` | string | — (required) | Storage account name; `[A-Za-z0-9-]` only, because it becomes the first label of the derived host |
+| `source.azure.account_key` | string \| null | `null` | Base64 storage-account key, signed per request with Shared Key. Mutually exclusive with `sas_token`; exactly one of the two is required |
+| `source.azure.sas_token` | string \| null | `null` | SAS query string without the leading `?` and without whitespace, appended to every request URL |
+| `source.gcs` | object \| null | `null` | Required for `provider = "gcs_native"` and rejected for every other provider |
+| `source.gcs.service_account_json` | string | — (required) | Service-account key JSON; must parse and carry `type: service_account`, `client_email` and `private_key`. Tokens are minted read-only (`devstorage.read_only`) |
 | `filter.prefix` | string \| null | `null` | Null or non-empty. Only local keys with this prefix consult the source |
 | `filter.source_prefix` | string \| null | `null` | Null or non-empty. Prepended to the local key to form the source key |
 | `policy.head` | `proxy` \| `local_only` | `proxy` | `local_only` answers a HEAD miss with 404 and no source traffic |
@@ -107,7 +139,7 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno
 | `policy.list_through` | bool | `false` | Merges the source listing into `ListObjectsV2` so clients see the whole namespace during the migration. Off by default: it puts the source in the path of every listing |
 | `policy.respect_local_delete_marker` | bool | `true` | A local delete marker is the final answer; only a versioned bucket can produce one |
 | `policy.preserve_etag` | bool | `true` | Keeps the source ETag on the stored object unless the bucket encrypts by default |
-| `policy.copy_tags` | bool | `false` | Copies source object tags; needs `s3:GetObjectTagging` and costs one extra source call per inline pull |
+| `policy.copy_tags` | bool | `false` | Copies source object tags; needs `s3:GetObjectTagging` and costs one extra source call per inline pull. `azure` reads blob tags instead; `gcs_native` has no tags and always finds none |
 | `policy.emit_events` | bool | `true` | Whether a write-back emits `ObjectCreated` notifications |
 | `policy.negative_cache_ttl_secs` | integer | `30` | `0..=3600`; `0` disables the negative cache |
 | `policy.inline_max_bytes` | integer | `16777216` (16 MiB) | `0..=268435456` (256 MiB). At or below this size a GET miss is teed inline; above it the response streams through and a background pull stores the object |
@@ -119,7 +151,7 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno
 | `policy.source_timeout.idle_ms` | integer | `30000` | `100..=600000`; enforced per body chunk on both the background pump and the inline tee |
 | `policy.bandwidth_limit_bytes_per_sec` | integer \| null | `null` | When set, at least `65536` |
 
-Values that are **not** configurable: the breaker opens after 5 consecutive counted failures inside a 30 s window, stays open for 30 s and then admits one probe (`breaker.rs`); the negative cache holds at most 100 000 keys per bucket with LRU eviction (`negative_cache.rs`); a background pull retries a retryable source failure at most 3 times with 1 s / 4 s / 16 s base delays plus up to 25 % jitter (`pull.rs`). The SDK's own retry policy is disabled on the source client, so one logical source call is exactly one wire request and the retry budget above is the only one.
+Values that are **not** configurable: the breaker opens after 5 consecutive counted failures inside a 30 s window, stays open for 30 s and then admits one probe (`breaker.rs`); the negative cache holds at most 100 000 keys per bucket with LRU eviction (`negative_cache.rs`); a background pull retries a retryable source failure at most 3 times with 1 s / 4 s / 16 s base delays plus up to 25 % jitter (`pull.rs`). The SDK's own retry policy is disabled on the source client. Each SDK operation makes one wire request; an ambiguous HEAD 404 additionally probes the bucket, within the same configured first-byte budget.
 
 Validation also rejects two shapes outright: a source whose endpoint and bucket name **this** bucket on this deployment (`SelfReference`), and a source that matches one of the bucket's own replication targets (`ReplicationLoop`) — that pairing would amplify a write-back into a loop.
 
@@ -133,8 +165,14 @@ Validation also rejects two shapes outright: a source whose endpoint and bucket
 | `rustfs` | Required | Path-style | `auto` allowed | A RustFS source answers the migration request locally thanks to the anti-loop marker | `real_source_test.rs` in the `e2e-nightly` lane |
 | `r2` | `https://<account-id>.r2.cloudflarestorage.com` | Virtual-host | `auto` allowed (signed as `us-east-1`) | | `cloud-source (r2)`, only while `ODM_INTEROP_R2_*` are configured; no difference recorded yet |
 | `gcs` | `https://storage.googleapis.com` | Virtual-host | Real region required | Uses the GCS XML interoperability API with an HMAC key pair, not a service-account JSON key | `cloud-source (gcs)`, only while `ODM_INTEROP_GCS_HMAC_*` are configured; no difference recorded yet |
+| `azure` | Optional; derived as `https://<account>.blob.core.windows.net` | Native Blob REST, not S3 | Unused; write `auto` | Needs `source.azure`; the container is `source.bucket`. Reads need `Read` on the blob and `List` on the container, plus `Tags` when `policy.copy_tags` is on | None yet: no interop job covers Azure |
+| `gcs_native` | Optional; derived as `https://storage.googleapis.com` | Native GCS API, not S3 | Unused; write `auto` | Needs `source.gcs`. Reads use the XML API for objects and `objects.list` for listings, both with an OAuth token minted from the service-account key; the key needs `storage.objects.get` and `storage.objects.list` | None yet: no interop job covers native GCS |
 
-Azure Blob has no preset; a native provider is deferred (rustfs/backlog#2166).
+Every backend answers the same trait contract, pinned by `backend_contract.rs` in `rustfs/src/on_demand_migration/`, and the three differences that contract allows are the ones documented here.
+
+`azure` differs in two of them. Its ETag is a concurrency token rather than a digest of the bytes, so it is stored as `odm-source-etag` provenance and never used as the expected MD5 of a pulled object — the write-back integrity check falls back to the local digest. And its listing paginates only with an opaque marker: there is no "start after this key" form, so a caller that asks for one gets `Unsupported` instead of a listing that silently starts over.
+
+`gcs_native` differs in the other two. Its listing also has no exclusive "start after" form (`startOffset` is inclusive), so it refuses one the same way. And GCS has no object tagging at all: `policy.copy_tags` finds no tags rather than failing the pull, because GCS custom metadata is already carried by the head mapping. Its ETag is normally usable: the `x-goog-hash` MD5 is converted to hex and checked against the pulled bytes, except on a composite object, which has no MD5 and whose ETag is then treated as opaque.
 
 The "Interop evidence" column names the job in `.github/workflows/on-demand-migration-interop.yml` (rustfs/backlog#2167) that last exercised the preset against a real implementation, and is where a provider difference belongs once the lane finds one. That lane is report-only and scheduled: it runs `crates/e2e_test/src/on_demand_migration/interop_test.rs` — the same case bodies as the merge-gate suite, with the source injected through `RUSTFS_ODM_INTEROP_*` — against a pinned MinIO container, and against each cloud provider whose repository secrets are configured. A provider without secrets is skipped with a note in the run summary rather than failing, so "no difference recorded yet" means exactly that and not "verified clean"; see [ci-gates.md](../testing/ci-gates.md) for the row.
 
@@ -150,6 +188,14 @@ No write, delete, ACL or versioning permission is required or used. Scope the po
 
 Behaviour a client can observe. The "Test" column names the case that pins it: `*_test.rs` files live under `crates/e2e_test/src/on_demand_migration/`, and the unit tests live next to the code in `rustfs/src/app/object/get.rs`, `head.rs` and `shared.rs`.
 
+ODM merged continuation tokens use a NUL-prefixed JSON envelope inside the existing base64 encoding. NUL is not valid in a local object key, so a legitimate JSON-shaped key can never be mistaken for a merged cursor. Upgrade every node before using list-through, and restart any in-progress ODM listing issued by an older build: its unframed JSON tokens cannot be distinguished from legitimate local keys. Ordinary local listing tokens remain unchanged. Tokens issued by this build can still resume the local side after list-through is disabled.
+
+Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket, a missing source version, or an ambiguous GET 404 is not proof that the requested key is absent. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object.
+
+Write-back currently requires namespace locking enabled and exactly one pool with one erasure set. Other topologies fail write-back explicitly as `unsupported`: source reads remain available, but backfill cannot complete successfully or certify cutover. This restriction avoids relying on a set-local condition across distinct pool or lock domains; it does not restrict ordinary S3 writes. Full cross-pool migration requires a globally fenced commit protocol.
+
+On the supported topology, write-back uses a create-only check under the local storage commit lock for both single-part PUT and multipart completion. A client write that commits while ODM is reading the source is preserved. With `respect_local_delete_marker=true`, a concurrent versioned deletion is preserved too. An explicit `respect_local_delete_marker=false` still permits revival; an unversioned deletion has no tombstone and therefore cannot be distinguished from a key that has never existed locally.
+
 | Situation | Behaviour | Test |
 |---|---|---|
 | GET miss, object at or below `inline_max_bytes` | One source GET, teed: the client streams while the same bytes are written locally. Later reads are local and carry no source marker | `get_basic_test.rs::get_miss_pulls_inline_and_serves_locally_afterwards`, `get.rs::odm_get_inline_streams_to_client_and_commits_the_same_bytes` |
@@ -219,7 +265,7 @@ Five provenance keys are written on every pulled object under both internal pref
 | Concurrency limit | Local write amplification | `max_concurrent_pulls` permits shared by inline and background pulls |
 | Bounded queue | Unbounded memory on a burst | `pull_queue_capacity` waiting jobs; overflow is counted as `queue_full` and never fails a client response |
 | Bandwidth limit | Source and network saturation | `bandwidth_limit_bytes_per_sec` (minimum 64 KiB/s) on the source client |
-| Retry budget | Transient source blips | Background pulls retry a retryable failure up to 3 times (1 s / 4 s / 16 s plus jitter). Inline pulls never retry: the bytes are already on their way to the client. The SDK retry policy on the source client is disabled (`RemoteS3RetryPolicy::Disabled`), so this is the only retry budget and one logical source call is exactly one wire request — replication targets keep the SDK's three attempts, declared on their own spec |
+| Retry budget | Transient source blips | Background pulls retry a retryable failure up to 3 times (1 s / 4 s / 16 s plus jitter). Inline pulls never retry: the bytes are already on their way to the client. The SDK retry policy is disabled (`RemoteS3RetryPolicy::Disabled`); HEAD 404 also requires one bucket probe. Replication targets keep their separately declared three SDK attempts |
 | Idle timeout | A source that answers and then goes quiet mid-body | `source_timeout.idle_ms` per body chunk on both paths. The budget measures the source read, upstream of the inline tee, so a slow client is never mistaken for an idle source; when it fires the client stream ends in an error and the write-back is discarded |
 | Anti-loop marker | Migration chains between RustFS/MinIO deployments | Every source request carries `x-rustfs-source-proxy-request` and `x-minio-source-proxy-request`; a request carrying it is always answered locally |
 | Outbound endpoint policy | SSRF | See [outbound-connection-policy.md](outbound-connection-policy.md) |

docs/operations/replication-outbound-transport.md

@@ -29,6 +29,18 @@
 
 Both knobs are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them.
 
+### Remote tier transport timeouts
+
+Remote tier S3-compatible clients use separate transport budgets. These settings do not change bucket or site replication clients.
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS` | `10` | Maximum time to establish the remote tier TCP connection. |
+| `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS` | `86400` | Maximum time for a remote tier request to reach response headers. The long default preserves large transition-upload headroom. |
+| `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS` | `60` | Maximum time without a non-empty response-body chunk. Empty HTTP/2 frames do not count as progress. |
+
+All three values must be positive integers. Zero fails tier client initialization instead of silently disabling the boundary. An invalid integer is logged and falls back to the default; very large values are accepted and provide a correspondingly long effective budget. The values are read when the tier client is built; recreate or reload the tier configuration after changing them.
+
 ## Before changing any of this
 
 Follow the SOP in `docs/postmortems/2026-09-03-replication-checksum-default-regression.md`: inventory the target-side rules the current default satisfies, run the outbound target matrix, and document any new knob here in the same PR.

docs/operations/scanner-benchmark-runbook.md

@@ -34,6 +34,142 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/
 
 ## Test Matrix
 
+### Formal Scanner/Heal ABBA
+
+The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`,
+`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least
+three A1/B1/B2/A2 groups for both baseline/candidate with background work on,
+and candidate-only background off/on. A measured leg lasts at least 900
+seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles).
+The existing `performance-ab.yml` supplies the pattern for immutable build
+provenance and failure propagation, but its short Warp workload is not this
+scanner gate. No scheduled workflow starts this matrix automatically.
+
+```bash
+scripts/run_scanner_validation_harness.sh --abba \
+  --manifest scanner-abba.json --adapter /path/to/isolated-deployment-adapter \
+  --out-dir /path/to/new-artifacts --data-root /path/to/new-test-data
+```
+
+Both roots must be new and non-overlapping. Every leg receives a unique data
+directory. The runner checks disk capacity before each leg, never removes data,
+and stops the adapter after success or failure. Retain raw artifacts and inspect
+task ownership before removing any test data. The operator must reserve the
+target machines and map the assigned directory to separate data paths on every
+node; the runner cannot prove remote isolation from local path names.
+
+The manifest has the following JSON contract (all fields are required):
+
+| Field | Value |
+|---|---|
+| `schema`, `evidence` | `1`, and `measured` or `synthetic`. |
+| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
+| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
+| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
+| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
+| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
+
+Record exact build flags and effective durability settings, not just defaults.
+Use deterministic workload seeds so every isolated leg has the same expected
+object/version/content result. Fix the foreground arrival rate (offered load),
+cache preparation procedure, configuration, and hardware across every leg.
+Do not include credentials in the manifest, adapter output, or saved commands;
+the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
+
+#### Deployment Adapter Contract
+
+The runner invokes an executable as `adapter ACTION request.json response.json`
+with no shell evaluation. Actions are separate processes: `prepare`, `measure`,
+`oracle`, and `stop`. Every action must return zero and write a JSON object of
+at most 1 MiB. Logs are kept separately and require an operator-managed disk
+quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero
+samples, and request errors fail the run. Adapters must terminate their own
+children on failure and `stop` must be idempotent even after partial preparation.
+
+The request contains the fixed manifest fields, selected build, scenario, round,
+leg, comparison (`build` or `background`), background mode (`on` or `off`),
+duration, unique `data_dir`, expected object oracle, and expected repair count.
+Adapter responsibilities:
+
+1. `prepare` deploys the selected binary into an authorized isolated topology,
+   checks actual binary/config/durability, initializes deterministic scenario
+   data and the requested cache state, and returns `{"ready": true}`. For measured
+   runs it also returns `collector` with exactly `alias`, `endpoint`, and
+   comma-separated `metrics_endpoints`; the runner starts the existing scanner
+   collector at 60-second cadence while `measure` runs.
+2. `measure` maintains the fixed offered load for the entire requested duration.
+   `cold-hot` retains cold buckets while mutating a hot bucket; `fresh-hot`
+   creates a bucket after scanner startup; `multi-hot-new` combines several hot
+   buckets with a newly created bucket; `running-heal` applies foreground load
+   during active repair; `mrf-replay` replays independently seeded durable repair
+   work. Capture same-window status for bucket-freshness issue #7108. Actual
+   fault injection and dataset generation belong to the reviewed adapter.
+3. `oracle` independently enumerates all objects and versions, reads and checks
+   their complete bytes, and verifies repairs. Return `complete: true`, integer
+   `errors: 0`, and `actual` matching the manifest's expected oracle. Never copy
+   expected values into a measured oracle or infer completion from empty queues.
+4. `stop` stops task-owned workload/server processes and returns `stopped: true`.
+   Preserve data and artifacts for diagnosis. An adapter may restore previous
+   settings but must not delete arbitrary paths or stop unrelated deployments.
+
+The `measure` response echoes the observed `evidence`, `fixed`, `build`,
+`data_dir`, and `background`, plus `sample_count` (1..3600), `elapsed_seconds`,
+and `metrics`. All metrics must be finite nonnegative numbers: `p99_ms`,
+`throughput_ops`, `rss_bytes`, `cpu_seconds`, `iops`, `rpc_count`,
+`cache_clone_bytes`, `encode_bytes`, `save_bytes`, `oldest_age_seconds`,
+`walk_objects`, `cold_walk_objects`, `healed_objects`, `errors`, and `requests`.
+Requests, throughput, and p99 must be positive; errors must be zero. Repair
+counts must match the manifest when background work is on. Keep underlying
+request samples, counter reset checks, profiler captures, and per-node telemetry
+in the cell artifact directory; aggregate values alone do not establish their
+measurement provenance. Missing production instrumentation is a pending gate,
+not permission to report a fabricated zero.
+
+For P2, `measure.convergence` contains booleans `writes_stopped`,
+`last_mutation_observed`, `first_complete_publication`; numeric
+`last_mutation_time`, `last_mutation_observed_time`, `writes_stopped_time`, `window_start`, `window_end`,
+`budget_available_seconds`, `walk_objects`, and `full_walk_objects`. Times use
+one monotonic clock. The window starts after writes stop and the final mutation
+is observed, and ends at the first complete publication. The reference is an
+independent full walk of the same static namespace. Record available budget
+seconds to interpret elapsed time. During continuing writes, omit this proof
+and report useful-work ratio and justified invalidation/re-scan work separately;
+the runner reports P2 pending and does not impose a fixed cumulative walk bound.
+
+The nightly heal workflow clones **`rustfs/auto-testing`** separately and invokes
+`auto-testing/rustfs_heal_test.sh`; that script is not a local `scripts/test`
+entry point. If an adapter uses it, record and verify the external checkout's
+owner and full commit before use. The current workflow clones the default branch,
+so its contents must not be attributed to a RustFS source SHA.
+
+#### Evidence Gates
+
+`report.json` records each group's verdict and the raw responses remain in their
+cell directories. Candidate/build p99 regression must be at most 5% and
+throughput loss at most 3%; candidate background on/off limits are 10% and 5%.
+P1 requires cold-hot walk reduction of at least the baseline cold-walk share
+times 80%, rather than a fixed 80% reduction for every workload. P2 requires
+candidate post-stop work at most 1.2 times the independent full-walk reference.
+Missing candidate convergence proof yields `inconclusive`. A2/A1 or B2/B1 p99
+or throughput drift above 5% also yields `inconclusive`, with exit code 3.
+Correctness errors and non-noisy performance regressions exit 1. Every group
+must pass; a favorable median cannot hide a failing group.
+
+Synthetic success is explicitly `synthetic_validated`, with `performance:
+pending`. It validates orchestration and gate logic only. It proves no runtime,
+distributed, crash, mixed-version, or performance behavior and cannot close the
+performance acceptance gate. Run the fake-adapter self-tests with:
+
+```bash
+scripts/test_scanner_validation_harness.sh
+```
+
+They cover the complete 120-cell schedule, data isolation, missing builds and
+oracles, zero samples/requests, swallowed request errors, offered-load drift,
+incomplete repairs, missing metrics, noise, and P1/P2/p99 regressions. A real
+deployment adapter and actual ABBA artifacts remain required before any measured
+performance or release claim.
+
 Collect at least two runs on the same RustFS commit and the same workload. Keep hardware, commit, object count, object size, bucket count, scanner-enabled state, and foreground workload constant between runs.
 
 | Run | Purpose | Example scanner settings |

docs/operations/tier-ilm-debugging.md

@@ -22,6 +22,16 @@
 | `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
 | Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` |
 
+## Lifecycle rule limits and evaluation
+
+Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible.
+
+`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class.
+
+Noncurrent expiration and transition have independent `NewerNoncurrentVersions` limits. A transition with a positive limit waits for a complete version-group evaluation to establish that enough newer noncurrent versions remain. Single-object evaluation, including the current manual transition and immediate-enqueue paths, conservatively defers these counted transitions to the lifecycle scanner. An unmet expiration retention limit does not suppress a separately eligible transition.
+
+An expired restored local copy can be cleaned up under Object Lock because the retained logical version and remote data remain intact. Cleanup requires a completed transition and still waits for pending or failed replication. The storage layer revalidates the source identity and restore metadata before removing the local copy; restore headers alone do not authorize cleanup.
+
 ## Free-version recovery controls
 
 The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism.

docs/testing/ci-gates.md

@@ -49,7 +49,7 @@ Promotion rule: never promote a report-only lane to required from one green run.
 | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
 | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
 | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
-| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from rc.2` | `e2e-upgrade.yml` `direct-upgrade` | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release |
+| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
 | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
 | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
 | PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` |
@@ -84,13 +84,19 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
-| `e2e-upgrade.yml` (weekly) | `direct-upgrade` | upgrade gate; server logs | no | see the PR row |
+| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
 | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch |
 
 Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline.
 
+## Packaged functional acceptance
+
+`rustfs-functional-chain.yml` dispatches the packaged-build suites in `rustfs-*-test.yml` on the shared lab runners. A failing suite step or job must fail its workflow. Report collection, cleanup, and dispatch of the next suite can still run with `always()`; continuing diagnostics does not make the failed suite successful.
+
+Workflow status preserves errors that the test scripts report. It does not establish complete execution or a common package identity across the chain: inspect the current run's case results, package identity, and test-script revision as well. A script that returns zero after a failed tool invocation needs its own result check.
+
 ## Release validation
 
 Post-merge and tag-driven; not a substitute for a PR gate.

rustfs/Cargo.toml

@@ -57,7 +57,8 @@ name = "swift_object_integration_test"
 required-features = ["swift"]
 
 [features]
-default = ["ftps", "webdav"]
+default = ["ftps", "webdav", "gcs"]
+gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]
 swift = ["rustfs-protocols/swift"]
@@ -66,7 +67,7 @@ sftp = ["rustfs-protocols/sftp"]
 license = []
 io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
-full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"]
+full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
 e2e-test-hooks = []
 # Shortens Connect credentials only in debug E2E builds.
 connect-e2e-short-credentials = []
@@ -275,6 +276,7 @@ rcgen = { workspace = true }
 # Async Runtime and Networking
 async-trait = { workspace = true }
 axum.workspace = true
+faster-hex.workspace = true
 futures.workspace = true
 futures-lite.workspace = true
 futures-util.workspace = true
@@ -287,6 +289,11 @@ reqwest = { workspace = true, features = ["json", "stream"] }
 socket2 = { workspace = true, features = ["all"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
 tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
+aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
+aws-smithy-types = { workspace = true }
+google-cloud-auth = { workspace = true, optional = true }
+moka = { workspace = true, features = ["sync"] }
+xxhash-rust = { workspace = true, features = ["xxh3"] }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "time"] }

rustfs/src/admin/handlers/bucket_meta.rs

@@ -64,6 +64,9 @@ use time::OffsetDateTime;
 use tracing::warn;
 use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
 
+const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic";
+const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json";
+
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
 const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
@@ -97,9 +100,198 @@ fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) ->
     checked_raw_xml(validated, raw, deserialize::<VersioningConfiguration>)
 }
 
+async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result<Option<Vec<u8>>> {
+    match conf {
+        BUCKET_POLICY_CONFIG => {
+            let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let config_json =
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
+            Ok(Some(config_json))
+        }
+        BUCKET_NOTIFICATION_CONFIG => {
+            let config: s3s::dto::NotificationConfiguration = match metadata_sys::get_notification_config(bucket).await {
+                Ok(Some(res)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+                Ok(None) => return Ok(None),
+            };
+
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .notification_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_LIFECYCLE_CONFIG => {
+            let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                .lifecycle_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_TAGGING_CONFIG => {
+            let config: Tagging = match metadata_sys::get_tagging_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                .tagging_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_QUOTA_CONFIG_FILE => {
+            let config: BucketQuota = match metadata_sys::get_quota_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let config_json =
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+            Ok(Some(config_json))
+        }
+        OBJECT_LOCK_CONFIG => {
+            let config = match metadata_sys::get_object_lock_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .object_lock_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_SSECONFIG => {
+            let config = match metadata_sys::get_sse_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .encryption_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_VERSIONING_CONFIG => {
+            let config = match metadata_sys::get_versioning_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .versioning_config_xml
+                .clone();
+            let config_xml = checked_versioning_xml(&config, raw_config)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_REPLICATION_CONFIG => {
+            let config = match metadata_sys::get_replication_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .replication_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_TARGETS_FILE => {
+            let config: BucketTargets = match metadata_sys::get_bucket_targets_config(bucket).await {
+                Ok(res) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+
+            let config_json = serde_json::to_vec(&config.redacted_credentials())
+                .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+            Ok(Some(config_json))
+        }
+        _ => Ok(None),
+    }
+}
+
 #[derive(Debug, Default, serde::Deserialize)]
 pub struct ExportBucketMetadataQuery {
+    #[serde(default)]
     pub bucket: String,
+    #[serde(default)]
+    pub diagnostic: bool,
 }
 
 pub struct ExportBucketMetadata {}
@@ -169,6 +361,7 @@ impl Operation for ExportBucketMetadata {
         };
 
         let mut zip_writer = ZipWriter::new(Cursor::new(Vec::new()));
+        let mut errors = Vec::new();
 
         let confs = [
             BUCKET_POLICY_CONFIG,
@@ -186,244 +379,49 @@ impl Operation for ExportBucketMetadata {
         for bucket in buckets {
             for &conf in confs.iter() {
                 let conf_path = path_join_buf(&[bucket.name.as_str(), conf]);
-                match conf {
-                    BUCKET_POLICY_CONFIG => {
-                        let config: BucketPolicy = match metadata_sys::get_bucket_policy(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let config_json = serde_json::to_vec(&config)
-                            .map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
+                let config = match exported_bucket_config(&bucket.name, conf).await {
+                    Ok(Some(config)) => config,
+                    Ok(None) => continue,
+                    Err(error) if !query.diagnostic => return Err(error),
+                    Err(_) => {
+                        errors.push(serde_json::json!({
+                            "bucket": bucket.name,
+                            "config": conf,
+                            "code": "configuration_unavailable",
+                        }));
+                        continue;
                     }
-                    BUCKET_NOTIFICATION_CONFIG => {
-                        let config: s3s::dto::NotificationConfiguration =
-                            match metadata_sys::get_notification_config(&bucket.name).await {
-                                Ok(Some(res)) => res,
-                                Err(e) => {
-                                    if e == StorageError::ConfigNotFound {
-                                        continue;
-                                    }
-                                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                                }
-                                Ok(None) => continue,
-                            };
-
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .notification_config_xml
-                            .clone();
-                        let config_xml =
-                            checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_LIFECYCLE_CONFIG => {
-                        let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                            .lifecycle_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-                    }
-                    BUCKET_TAGGING_CONFIG => {
-                        let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                            .tagging_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-                    }
-                    BUCKET_QUOTA_CONFIG_FILE => {
-                        let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let config_json =
-                            serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    OBJECT_LOCK_CONFIG => {
-                        let config = match metadata_sys::get_object_lock_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .object_lock_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_SSECONFIG => {
-                        let config = match metadata_sys::get_sse_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .encryption_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_VERSIONING_CONFIG => {
-                        let config = match metadata_sys::get_versioning_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .versioning_config_xml
-                            .clone();
-                        let config_xml = checked_versioning_xml(&config, raw_config)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_REPLICATION_CONFIG => {
-                        let config = match metadata_sys::get_replication_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .replication_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_TARGETS_FILE => {
-                        let config: BucketTargets = match metadata_sys::get_bucket_targets_config(&bucket.name).await {
-                            Ok(res) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-
-                        let config_json = serde_json::to_vec(&config.redacted_credentials())
-                            .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    _ => {}
-                }
+                };
+                let conf_path = if query.diagnostic {
+                    path_join_buf(&[DIAGNOSTIC_EXPORT_PREFIX, &conf_path])
+                } else {
+                    conf_path
+                };
+                zip_writer
+                    .start_file(conf_path, SimpleFileOptions::default())
+                    .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
+                zip_writer
+                    .write_all(&config)
+                    .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
             }
         }
 
+        if query.diagnostic {
+            let manifest = serde_json::to_vec(&serde_json::json!({
+                "version": 1,
+                "mode": "diagnostic",
+                "complete": errors.is_empty(),
+                "errors": errors,
+            }))
+            .map_err(|e| s3_error!(InternalError, "failed to serialize diagnostic manifest: {e}"))?;
+            zip_writer
+                .start_file(DIAGNOSTIC_EXPORT_MANIFEST, SimpleFileOptions::default())
+                .map_err(|e| s3_error!(InternalError, "failed to start diagnostic manifest: {e}"))?;
+            zip_writer
+                .write_all(&manifest)
+                .map_err(|e| s3_error!(InternalError, "failed to write diagnostic manifest: {e}"))?;
+        }
+
         let zip_bytes = zip_writer
             .finish()
             .map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
@@ -431,8 +429,17 @@ impl Operation for ExportBucketMetadata {
         header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
         header.insert(
             CONTENT_DISPOSITION,
-            "attachment; filename=bucket-meta.zip".parse().expect("valid header value"),
+            if query.diagnostic {
+                "attachment; filename=bucket-meta-diagnostic.zip"
+            } else {
+                "attachment; filename=bucket-meta.zip"
+            }
+            .parse()
+            .expect("valid header value"),
         );
+        if query.diagnostic {
+            header.insert("x-rustfs-bucket-metadata-export", "diagnostic".parse().expect("valid header value"));
+        }
         header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
         Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
     }
@@ -499,6 +506,18 @@ impl Operation for ImportBucketMetadata {
             file_contents.push((file_path, content));
         }
 
+        // Reject the whole archive before creating buckets or writing configs,
+        // even when the marker is malformed or follows ordinary config entries.
+        if file_contents.iter().any(|(path, _)| {
+            path == DIAGNOSTIC_EXPORT_MANIFEST
+                || path == DIAGNOSTIC_EXPORT_PREFIX
+                || path
+                    .strip_prefix(DIAGNOSTIC_EXPORT_PREFIX)
+                    .is_some_and(|suffix| suffix.starts_with('/'))
+        }) {
+            return Err(s3_error!(InvalidRequest, "diagnostic bucket metadata archives cannot be imported"));
+        }
+
         let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
         let quota_fleet_proof =
             if durable_quota_import {
@@ -1419,6 +1438,246 @@ mod backup_zip_compatibility_tests {
         assert_eq!(response.output.0, StatusCode::OK);
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn diagnostic_export_isolated_errors_and_import_recovers_unreadable_targets() {
+        const HEALTHY: &str = "diagnostic-healthy";
+        const UNREADABLE: &str = "diagnostic-unreadable";
+        const REPLACEMENT_TARGETS: &[u8] = br#"{"targets":[]}"#;
+        const SECRET: &str = "diagnostic-must-not-expose-this-secret";
+
+        let _ = rustfs_credentials::init_global_action_credentials(
+            Some(ROOT_ACCESS_KEY.to_string()),
+            Some(ROOT_SECRET_KEY.to_string()),
+        );
+        let temp = tempfile::tempdir().expect("create diagnostic export test root");
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(temp.path())
+            .disk_count(1)
+            .build()
+            .await;
+        env.make_bucket(HEALTHY, false).await;
+        env.make_bucket(UNREADABLE, false).await;
+        rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore))
+            .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
+            .await
+            .expect("seed IAM format");
+        let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
+            .await
+            .expect("build test IAM");
+        publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
+            Arc::clone(&env.ecstore),
+            iam,
+            Arc::new(rustfs_kms::KmsServiceManager::new()),
+        )));
+        metadata_sys::update(HEALTHY, BUCKET_VERSIONING_CONFIG, VERSIONING_XML.to_vec())
+            .await
+            .expect("seed healthy bucket config");
+
+        let minio_blob = hex_simd::decode_to_vec(
+            include_str!("../../../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex").trim(),
+        )
+        .expect("decode real MinIO metadata fixture");
+        let minio_metadata = BucketMetadata::unmarshal(&minio_blob[4..]).expect("read MinIO metadata fixture");
+        let targets_array = format!(r#"[{{"credentials":{{"secretKey":"{SECRET}"}}}}]"#).into_bytes();
+        for unreadable_targets in [targets_array, minio_metadata.bucket_targets_config_json] {
+            let mut metadata = metadata_sys::get_config_from_disk(UNREADABLE)
+                .await
+                .expect("load bucket before simulating MinIO targets");
+            let incarnation = metadata.bucket_incarnation_id;
+            metadata.bucket_targets_config_json = unreadable_targets.clone();
+            metadata
+                .save_with_store(Arc::clone(&env.ecstore))
+                .await
+                .expect("persist unreadable targets fixture");
+            crate::storage::storage_api::set_bucket_metadata(UNREADABLE.to_string(), metadata)
+                .await
+                .expect("publish unreadable targets fixture");
+            assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err());
+
+            let strict_error = ExportBucketMetadata {}
+                .call(
+                    admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
+                    Params::new(),
+                )
+                .await
+                .expect_err("a complete export must fail closed on unreadable targets");
+            assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError);
+
+            let response = ExportBucketMetadata {}
+                .call(
+                    admin_request(
+                        Method::GET,
+                        Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
+                        Vec::new(),
+                    ),
+                    Params::new(),
+                )
+                .await
+                .expect("one unreadable bucket must not abort diagnostic export");
+            assert_eq!(response.output.0, StatusCode::OK);
+            assert_eq!(response.headers["x-rustfs-bucket-metadata-export"], "diagnostic");
+            assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta-diagnostic.zip");
+            let bytes = response.output.1.collect().await.expect("read diagnostic archive").to_bytes();
+            let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open diagnostic archive");
+            let mut files = HashMap::new();
+            for index in 0..archive.len() {
+                let mut file = archive.by_index(index).expect("read diagnostic entry");
+                let mut content = Vec::new();
+                file.read_to_end(&mut content).expect("read diagnostic config");
+                assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes()));
+                assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/"));
+                files.insert(file.name().to_string(), content);
+            }
+            assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML);
+            assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}")));
+            assert!(!files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_TARGETS_FILE}")));
+            let manifest: serde_json::Value =
+                serde_json::from_slice(&files[DIAGNOSTIC_EXPORT_MANIFEST]).expect("decode diagnostic manifest");
+            assert_eq!(
+                manifest,
+                serde_json::json!({
+                    "version": 1,
+                    "mode": "diagnostic",
+                    "complete": false,
+                    "errors": [{ "bucket": UNREADABLE, "config": BUCKET_TARGETS_FILE, "code": "configuration_unavailable" }],
+                })
+            );
+
+            let error = ImportBucketMetadata {}
+                .call(
+                    admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
+                    Params::new(),
+                )
+                .await
+                .expect_err("diagnostic exports are never backups");
+            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+            assert_eq!(
+                metadata_sys::get_config_from_disk(UNREADABLE)
+                    .await
+                    .expect("load rejected import state")
+                    .bucket_targets_config_json,
+                unreadable_targets
+            );
+
+            import_archive(zip_with_entries(UNREADABLE, &[(BUCKET_TARGETS_FILE, REPLACEMENT_TARGETS)])).await;
+            let recovered = metadata_sys::get_config_from_disk(UNREADABLE)
+                .await
+                .expect("read recovered targets from disk");
+            assert_eq!(recovered.bucket_incarnation_id, incarnation);
+            assert_eq!(recovered.bucket_targets_config_json, REPLACEMENT_TARGETS);
+            assert!(
+                metadata_sys::get_bucket_targets_config(UNREADABLE)
+                    .await
+                    .expect("existing import API must recover targets readers")
+                    .is_empty()
+            );
+        }
+
+        // The marker may be malformed, come last, or be removed while the
+        // reserved directory remains. None may allow an earlier config write.
+        for marker in [
+            DIAGNOSTIC_EXPORT_MANIFEST.to_string(),
+            DIAGNOSTIC_EXPORT_PREFIX.to_string(),
+            format!("{DIAGNOSTIC_EXPORT_PREFIX}/bucket/config"),
+        ] {
+            let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
+            writer
+                .start_file(format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default())
+                .expect("start ordinary config before diagnostic marker");
+            writer
+                .write_all(b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+                .expect("write ordinary config before diagnostic marker");
+            writer
+                .start_file(
+                    format!("diagnostic-never-created/{BUCKET_VERSIONING_CONFIG}"),
+                    SimpleFileOptions::default(),
+                )
+                .expect("start a nonexistent bucket config before diagnostic marker");
+            writer.write_all(VERSIONING_XML).expect("write nonexistent bucket config");
+            writer
+                .start_file(marker, SimpleFileOptions::default())
+                .expect("start diagnostic marker");
+            writer.write_all(b"not json").expect("write malformed diagnostic marker");
+            let error = ImportBucketMetadata {}
+                .call(
+                    admin_request(
+                        Method::PUT,
+                        Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"),
+                        writer.finish().expect("finish marked archive").into_inner(),
+                    ),
+                    Params::new(),
+                )
+                .await
+                .expect_err("diagnostic preflight must reject before any config write");
+            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+            assert_eq!(
+                metadata_sys::get_config_from_disk(HEALTHY)
+                    .await
+                    .expect("read healthy config after rejected import")
+                    .versioning_config_xml,
+                VERSIONING_XML
+            );
+            assert!(
+                env.ecstore
+                    .get_bucket_info("diagnostic-never-created", &BucketOptions::default())
+                    .await
+                    .is_err(),
+                "diagnostic preflight must reject before bucket creation"
+            );
+        }
+
+        let response = ExportBucketMetadata {}
+            .call(
+                admin_request(
+                    Method::GET,
+                    Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
+                    Vec::new(),
+                ),
+                Params::new(),
+            )
+            .await
+            .expect("diagnostic export after recovery");
+        let bytes = response
+            .output
+            .1
+            .collect()
+            .await
+            .expect("read complete diagnostic archive")
+            .to_bytes();
+        let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open complete diagnostic archive");
+        let manifest: serde_json::Value = serde_json::from_reader(
+            archive
+                .by_name(DIAGNOSTIC_EXPORT_MANIFEST)
+                .expect("complete diagnostic manifest"),
+        )
+        .expect("parse complete diagnostic manifest");
+        assert_eq!(manifest["complete"], true);
+        let error = ImportBucketMetadata {}
+            .call(
+                admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
+                Params::new(),
+            )
+            .await
+            .expect_err("complete diagnostics must still reject import");
+        assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+
+        let response = ExportBucketMetadata {}
+            .call(
+                admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
+                Params::new(),
+            )
+            .await
+            .expect("ordinary cluster export must work after API recovery");
+        assert!(!response.headers.contains_key("x-rustfs-bucket-metadata-export"));
+        assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta.zip");
+        let bytes = response.output.1.collect().await.expect("read ordinary archive").to_bytes();
+        let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("open ordinary archive");
+        assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err());
+        assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok());
+        assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok());
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn g_zip_001_002_003_use_real_admin_archive_and_persistence_paths() {

rustfs/src/admin/handlers/on_demand_migration.rs

@@ -38,15 +38,6 @@ use crate::admin::runtime_sources::{
 };
 use crate::admin::storage_api::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
 use crate::admin::storage_api::bucket::metadata_sys;
-use crate::admin::storage_api::bucket::on_demand_migration::backfill::{
-    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
-};
-use crate::admin::storage_api::bucket::on_demand_migration::source_client::{
-    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
-};
-use crate::admin::storage_api::bucket::on_demand_migration::{
-    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
-};
 use crate::admin::storage_api::bucket::remote_s3_client::{
     PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
 };
@@ -57,6 +48,16 @@ use crate::admin::utils::{extract_query_params, read_compatible_admin_body};
 use crate::error::ApiError;
 use crate::license::license_check;
 use crate::module_switches::{ENV_ON_DEMAND_MIGRATION_ENABLED, on_demand_migration_enabled_from_env};
+use crate::on_demand_migration::backfill::{
+    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
+};
+use crate::on_demand_migration::source_client::{
+    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
+};
+use crate::on_demand_migration::{
+    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
+    source_backend_spec,
+};
 use crate::server::ADMIN_PREFIX;
 use hyper::{Method, StatusCode};
 use matchit::Params;
@@ -89,6 +90,8 @@ const BACKFILL_OP_CANCEL: &str = "cancel";
 pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled";
 /// Error code returned when the source bucket did not answer the probe.
 pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable";
+/// Error code returned when the configured provider was excluded at build time.
+pub(crate) const ERR_CODE_BACKEND_NOT_COMPILED: &str = "OnDemandMigrationBackendNotCompiled";
 /// Error code returned by `GET` when the bucket has no configuration.
 pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration";
 /// Error code (409) returned by `start` while a backfill job holds the lease.
@@ -577,7 +580,7 @@ async fn validate_config(bucket: &str, config: &OnDemandMigrationConfig) -> S3Re
 }
 
 fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
-    use crate::admin::storage_api::bucket::on_demand_migration::Provider;
+    use crate::on_demand_migration::Provider;
     match config.source.provider {
         Provider::S3 => SourceProvider::S3,
         Provider::Aws => SourceProvider::Aws,
@@ -585,6 +588,8 @@ fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
         Provider::Rustfs => SourceProvider::Rustfs,
         Provider::R2 => SourceProvider::R2,
         Provider::Gcs => SourceProvider::Gcs,
+        Provider::Azure => SourceProvider::Azure,
+        Provider::GcsNative => SourceProvider::GcsNative,
     }
 }
 
@@ -621,15 +626,23 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie
         // a flapping source behind a success and triple the probe's cost.
         retry: RemoteS3RetryPolicy::Disabled,
         bandwidth_limit: config.policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new),
+        // One mapping serves the probe and the runtime, so an admin probe
+        // always exercises the backend the runtime will build.
+        backend: source_backend_spec(source),
     }
 }
 
-/// Builder failures are input errors: the endpoint policy, the CA PEM or the
-/// credentials the operator supplied. Anonymous sources are not wired yet
+/// Distinguishes excluded backends from invalid endpoint, CA or credentials.
+/// Anonymous S3 sources are not wired yet
 /// (ODM-05 adds the credential-less path), so `MissingCredentials` is a 400
 /// naming the field instead of an opaque internal error.
 fn client_build_error(err: RemoteS3ClientError) -> S3Error {
     match err {
+        RemoteS3ClientError::BackendNotCompiled(provider) => custom_error(
+            ERR_CODE_BACKEND_NOT_COMPILED,
+            StatusCode::NOT_IMPLEMENTED,
+            format!("the {provider} backend is not included in this build; rebuild with the gcs feature"),
+        ),
         RemoteS3ClientError::MissingCredentials => admin_s3_error(
             S3ErrorCode::InvalidArgument,
             "source.credentials is required: anonymous sources are not supported yet",
@@ -774,7 +787,7 @@ impl Operation for GetBucketOnDemandMigrationHandler {
         let bucket = bucket_from_params(&params)?;
         let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
 
-        let Some((config, updated_at)) = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
+        let Some((config, updated_at)) = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
             admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
         })?
         else {
@@ -833,7 +846,7 @@ impl Operation for GetBucketOnDemandMigrationStatusHandler {
         let bucket = bucket_from_params(&params)?;
         let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
 
-        let config = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
+        let config = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
             admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
         })?;
         let runtime = OnDemandMigrationSys::get().bucket_snapshot(&bucket);
@@ -1282,6 +1295,14 @@ mod tests {
         assert!(err.message().unwrap_or_default().contains("source.credentials"));
     }
 
+    #[test]
+    fn backend_not_compiled_is_distinct_from_invalid_credentials() {
+        let err = client_build_error(RemoteS3ClientError::BackendNotCompiled("gcs_native"));
+        assert_eq!(err.code(), &S3ErrorCode::Custom(ERR_CODE_BACKEND_NOT_COMPILED.into()));
+        assert_eq!(err.status_code(), Some(StatusCode::NOT_IMPLEMENTED));
+        assert!(err.message().unwrap_or_default().contains("gcs feature"));
+    }
+
     #[test]
     fn module_switch_defaults_on_and_reads_the_env() {
         temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || assert!(module_enabled()));
@@ -1742,6 +1763,19 @@ mod store_tests {
                 assert_eq!(body["inflight_pulls"], Value::from(0));
                 assert_eq!(body["queue_depth"], Value::from(0));
 
+                // A malformed replacement cannot overwrite the saved source.
+                let before = metadata_sys::get(BUCKET).await.expect("saved metadata");
+                for invalid in [b"not-json".to_vec(), br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec()] {
+                    let err = SetBucketOnDemandMigrationHandler {}
+                        .call(root_request(Method::PUT, config_uri(""), invalid), bucket_params(&router))
+                        .await
+                        .expect_err("malformed replacement must be rejected");
+                    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+                    let after = metadata_sys::get(BUCKET).await.expect("saved metadata remains readable");
+                    assert_eq!(after.on_demand_migration_config_json, before.on_demand_migration_config_json);
+                    assert_eq!(after.on_demand_migration_config_updated_at, before.on_demand_migration_config_updated_at);
+                }
+
                 // The peer fan-out ran: the single unreachable peer is reported.
                 let context = crate::admin::runtime_sources::current_app_context();
                 let err = reload_peers(context.as_deref(), BUCKET)

rustfs/src/admin/storage_api.rs

@@ -20,8 +20,8 @@ use time::OffsetDateTime;
 
 mod ecstore_bucket {
     pub(crate) use crate::storage::storage_api::ecstore_bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, on_demand_migration, quota,
-        remote_s3_client, replication, target, utils, versioning, versioning_sys,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
+        replication, target, utils, versioning, versioning_sys,
     };
 }
 
@@ -284,34 +284,6 @@ pub(crate) mod durability {
     pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
 }
 
-pub(crate) mod on_demand_migration {
-    pub(crate) type OdmBucketSnapshot = super::ecstore_bucket::on_demand_migration::OdmBucketSnapshot;
-    pub(crate) type OnDemandMigrationConfig = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfig;
-    pub(crate) type OnDemandMigrationConfigError = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfigError;
-    pub(crate) type OnDemandMigrationSys = super::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
-    pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle;
-    pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider;
-    pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>;
-
-    pub(crate) mod backfill {
-        pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint;
-        pub(crate) type BackfillError = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillError;
-        pub(crate) type BackfillRequest = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillRequest;
-        pub(crate) type BackfillState = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillState;
-        pub(crate) type SkipExisting = super::super::ecstore_bucket::on_demand_migration::backfill::SkipExisting;
-        pub(crate) use super::super::ecstore_bucket::on_demand_migration::backfill::global_backfill_runner;
-    }
-
-    pub(crate) mod source_client {
-        pub(crate) type SourceClient = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClient;
-        pub(crate) type SourceClientSpec = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClientSpec;
-        pub(crate) type SourceError = super::super::ecstore_bucket::on_demand_migration::source_client::SourceError;
-        pub(crate) type SourceProbe = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProbe;
-        pub(crate) type SourceProvider = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProvider;
-        pub(crate) type SourceTimeouts = super::super::ecstore_bucket::on_demand_migration::source_client::SourceTimeouts;
-    }
-}
-
 pub(crate) mod remote_s3_client {
     pub(crate) type PathStyle = super::ecstore_bucket::remote_s3_client::PathStyle;
     pub(crate) type RemoteCredentials = super::ecstore_bucket::remote_s3_client::RemoteCredentials;
@@ -454,12 +426,6 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
     }
 
-    pub(crate) async fn get_on_demand_migration_config(
-        bucket: &str,
-    ) -> Result<Option<(super::on_demand_migration::OnDemandMigrationConfig, OffsetDateTime)>> {
-        super::ecstore_bucket::metadata_sys::get_on_demand_migration_config(bucket).await
-    }
-
     pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
     }
@@ -912,7 +878,6 @@ pub(crate) mod bucket {
     pub(crate) use super::lifecycle;
     pub(crate) use super::metadata;
     pub(crate) use super::metadata_sys;
-    pub(crate) use super::on_demand_migration;
     pub(crate) use super::quota;
     pub(crate) use super::remote_s3_client;
     pub(crate) use super::replication;

rustfs/src/app/bucket_list_through.rs

@@ -25,20 +25,19 @@
 use super::storage_api::bucket_usecase::ECStore;
 use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
 use super::storage_api::bucket_usecase::StorageObjectOptions;
-use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
-    BucketOdmState, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
-    OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
-    SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
-};
 use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
 use super::storage_api::bucket_usecase::contract::list::{ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _};
 use super::storage_api::bucket_usecase::contract::object::ObjectOperations as _;
-use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Result};
+use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Request, S3Result};
 use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params;
 use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class};
 use crate::error::ApiError;
+use crate::on_demand_migration::{
+    BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
+    MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
+    SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
+};
 use futures::StreamExt;
-use http::HeaderMap;
 use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header};
 use std::sync::Arc;
 use std::time::Instant;
@@ -51,6 +50,9 @@ type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
 /// yet, so the only class RustFS can vouch for is the default one.
 const SOURCE_STORAGE_CLASS: &str = "STANDARD";
 
+/// Enable only after every node serving continuation requests can read v2.
+const ENV_LIST_PROGRESS_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS";
+
 /// Concurrent local metadata probes when a versioned bucket has to check
 /// source-only keys for a shadowing delete marker.
 const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32;
@@ -98,16 +100,38 @@ fn invalid_continuation_token(err: &ListThroughTokenError) -> S3Error {
 /// bucket has no source, `list_through` is off, or the request carries the
 /// `source-proxy-request` anti-loop marker and therefore comes from a peer
 /// that must be answered locally.
-pub(crate) fn list_through_state(bucket: &str, headers: &HeaderMap) -> Option<Arc<BucketOdmState>> {
-    if get_header(headers, SUFFIX_SOURCE_PROXY_REQUEST).is_some() {
-        return None;
+pub(crate) async fn list_through_state<T>(
+    store: &ECStore,
+    bucket: &str,
+    req: &S3Request<T>,
+    params: &ListObjectsV2Params,
+) -> S3Result<Option<Arc<BucketOdmState>>> {
+    if get_header(&req.headers, SUFFIX_SOURCE_PROXY_REQUEST).is_some() {
+        return Ok(None);
     }
     let sys = OnDemandMigrationSys::get();
     if !sys.is_module_enabled() {
-        return None;
+        return Ok(None);
     }
-    let state = sys.state(bucket)?;
-    state.config().policy.list_through.then_some(state)
+    let Some(state) = sys.state(bucket).filter(|state| state.config().policy.list_through) else {
+        return Ok(None);
+    };
+    if params.max_keys == 0
+        || matches!(
+            source_list_plan(&params.prefix, state.config().filter.prefix.as_deref(), params.delimiter.as_deref()),
+            SourceListPlan::Skip,
+        )
+    {
+        return Ok(None);
+    }
+    let Some(expected_incarnation) = super::storage_api::bucket_usecase::access::odm_read_generation(req, bucket)? else {
+        return Ok(None);
+    };
+    let incarnation = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?;
+    if incarnation != expected_incarnation {
+        return Ok(None);
+    }
+    Ok(state.filter_incarnation(incarnation))
 }
 
 /// A merged page plus whether the source had to be left out of it.
@@ -264,7 +288,18 @@ pub(crate) async fn merged_list_objects_v2(
         buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
     }
 
-    let outcome = merger.finish();
+    let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false);
+    let outcome = match merger.finish(issue_progress_tokens) {
+        Ok(outcome) => outcome,
+        Err(ListPageError::NoProgress(MergeSide::Source)) => {
+            degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
+            merger
+                .finish(issue_progress_tokens)
+                .map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))?
+        }
+        Err(error) => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
+    };
+    drop(merger);
     let mut objects = Vec::with_capacity(outcome.picks.len());
     let mut prefixes = Vec::new();
     let mut source_only_keys = Vec::new();
@@ -429,13 +464,18 @@ mod tests {
     use super::*;
     use crate::app::bucket_usecase::DefaultBucketUsecase;
     use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
-    use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
-        FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
+    use crate::app::storage_api::bucket_usecase::s3::{
+        GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
+        XmlSerialize, XmlSerializer,
     };
-    use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
     use crate::app::storage_api::test::StoragePutObjReader;
-    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use crate::app::storage_api::test::contract::object::ObjectIO as _;
+    use crate::on_demand_migration::{
+        FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
+        SourceCredentials, TlsConfig,
+    };
+    use http::HeaderMap;
     use std::time::Duration;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
@@ -448,6 +488,7 @@ mod tests {
             source: Some("source-2".to_string()),
             source_done: false,
             last_key: Some("k".to_string()),
+            no_progress: None,
         }
     }
 
@@ -515,8 +556,29 @@ mod tests {
         assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
     }
 
+    #[test]
+    fn a_v2_token_keeps_the_local_cursor_when_list_through_is_turned_off() {
+        let mut resume = token(Some("local-2"), false);
+        resume.v = 2;
+        resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1);
+        let encoded = resume.encode();
+        let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes");
+        assert_eq!(decoded.as_ref(), Some(&resume));
+        assert!(matches!(
+            local_cursor(Some(&encoded), decoded.as_ref()),
+            LocalListCursor::Token(Some(local)) if local == "local-2"
+        ));
+        resume.local_done = true;
+        let encoded = resume.encode();
+        let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes");
+        assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
+    }
+
     #[test]
     fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() {
+        let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#;
+        assert!(decode_list_cursor(Some(json_key)).expect("valid local key").is_none());
+        assert!(matches!(local_cursor(Some(json_key), None), LocalListCursor::Token(Some(local)) if local == json_key));
         assert!(
             decode_list_cursor(Some("photos/a.jpg"))
                 .expect("plain markers decode")
@@ -551,14 +613,30 @@ mod tests {
     /// Serves exactly the scripted S3 pages and joins every connection before
     /// returning. A source retry or unexpected operation fails the test.
     async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
+        let (endpoint, server, _) = list_source(pages.into_iter()).await;
+        (endpoint, server)
+    }
+
+    async fn list_source(
+        pages: impl Iterator<Item = String> + Send + 'static,
+    ) -> (
+        String,
+        tokio_util::task::AbortOnDropHandle<Vec<String>>,
+        tokio_util::sync::CancellationToken,
+    ) {
         let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
             .await
             .expect("bind listing source");
         let address = listener.local_addr().expect("listing source address");
+        let stop = tokio_util::sync::CancellationToken::new();
+        let server_stop = stop.clone();
         let server = tokio::spawn(async move {
             let mut requests = Vec::new();
             for body in pages {
-                let (mut stream, _) = listener.accept().await.expect("accept source listing");
+                let (mut stream, _) = tokio::select! {
+                    _ = server_stop.cancelled() => break,
+                    accepted = listener.accept() => accepted.expect("accept source listing"),
+                };
                 let mut request = Vec::new();
                 let mut chunk = [0; 4096];
                 while !request.windows(4).any(|window| window == b"\r\n\r\n") {
@@ -588,7 +666,7 @@ mod tests {
             }
             requests
         });
-        (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
+        (format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server), stop)
     }
 
     fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
@@ -616,12 +694,12 @@ mod tests {
         }
     }
 
-    async fn source_policy_request(
-        pages: Vec<String>,
+    async fn source_policy_input(
+        endpoint: String,
         policy: SourceErrorPolicy,
         resume_source: Option<&str>,
         filter_prefix: Option<&str>,
-    ) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
+    ) -> (ListThroughTestState, ListObjectsV2Input) {
         let store = shared_gating_ecstore().await;
         crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
         let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
@@ -638,9 +716,8 @@ mod tests {
             )
             .await
             .expect("seed real local listing");
-        let (endpoint, server) = scripted_list_source(pages).await;
         let sys = OnDemandMigrationSys::get();
-        let _state_guard = ListThroughTestState {
+        let state_guard = ListThroughTestState {
             bucket: bucket.clone(),
             module_enabled: sys.is_module_enabled(),
         };
@@ -660,6 +737,8 @@ mod tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: filter_prefix.map(str::to_string),
@@ -671,7 +750,12 @@ mod tests {
                 ..Default::default()
             },
         };
-        sys.apply(&bucket, Some(&config)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket identity"),
+            Some(&config),
+        )
+        .await;
         assert!(
             sys.state(&bucket).expect("ODM state installed").client().is_ok(),
             "fake source client must build"
@@ -685,6 +769,7 @@ mod tests {
                 source: Some(source.into()),
                 source_done: false,
                 last_key: None,
+                no_progress: None,
             };
             base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
         });
@@ -701,6 +786,10 @@ mod tests {
             request_payer: None,
             start_after: None,
         };
+        (state_guard, input)
+    }
+
+    async fn execute_source_list(input: ListObjectsV2Input) -> S3Result<S3Response<ListObjectsV2Output>> {
         let request = S3Request {
             input,
             method: http::Method::GET,
@@ -712,12 +801,23 @@ mod tests {
             service: None,
             trailing_headers: None,
         };
-        let result = tokio::time::timeout(
+        tokio::time::timeout(
             Duration::from_secs(10),
             DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
         )
         .await
-        .expect("listing must complete within its bounded source budget");
+        .expect("listing must complete within its bounded source budget")
+    }
+
+    async fn source_policy_request(
+        pages: Vec<String>,
+        policy: SourceErrorPolicy,
+        resume_source: Option<&str>,
+        filter_prefix: Option<&str>,
+    ) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
+        let (endpoint, server) = scripted_list_source(pages).await;
+        let (_state_guard, input) = source_policy_input(endpoint, policy, resume_source, filter_prefix).await;
+        let result = execute_source_list(input).await;
         let requests = tokio::time::timeout(Duration::from_secs(5), server)
             .await
             .expect("source connections must finish")
@@ -725,6 +825,219 @@ mod tests {
         (result, requests)
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn stale_bucket_state_cannot_send_get_head_or_list_to_the_source() {
+        run_large_stack_test("list-through-incarnation", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    let (endpoint, server, stop) = list_source(std::iter::repeat(source_xml(None, false, Some("source")))).await;
+                    let (_guard, input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await;
+                    let sys = OnDemandMigrationSys::get();
+                    let old = sys.state(&input.bucket).expect("original source state");
+                    let store = shared_gating_ecstore().await;
+                    let get = S3Request {
+                        input: GetObjectInput {
+                            bucket: input.bucket.clone(),
+                            key: "missing".into(),
+                            ..Default::default()
+                        },
+                        method: http::Method::GET,
+                        uri: http::Uri::from_static("/missing"),
+                        headers: HeaderMap::new(),
+                        extensions: http::Extensions::new(),
+                        credentials: None,
+                        region: None,
+                        service: None,
+                        trailing_headers: None,
+                    };
+                    let authorized_generation =
+                        super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
+                            &store,
+                            &get,
+                            &input.bucket,
+                        )
+                        .await
+                        .expect("capture the identity before authorization");
+                    store
+                        .delete_bucket(
+                            &input.bucket,
+                            &DeleteBucketOptions {
+                                force: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("delete original bucket");
+                    store
+                        .make_bucket(&input.bucket, &MakeBucketOptions::default())
+                        .await
+                        .expect("recreate bucket");
+                    let replacement = store
+                        .bucket_incarnation_id(&input.bucket)
+                        .await
+                        .expect("replacement identity");
+                    assert_ne!(replacement, old.incarnation_id());
+                    sys.remove(&input.bucket);
+                    let mut without_source = get.clone();
+                    super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
+                        &store,
+                        &mut without_source,
+                        &input.bucket,
+                    )
+                    .await;
+                    for state_incarnation in [old.incarnation_id(), replacement] {
+                        sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
+                            .await;
+                        // Both a stale runtime and a newly published replacement must reject
+                        // requests already authorized for the deleted incarnation.
+                        for capture in 0..3 {
+                            if capture == 0 && state_incarnation == replacement {
+                                continue;
+                            }
+                            let mut get = if capture == 2 { without_source.clone() } else { get.clone() };
+                            if capture == 1 {
+                                get.extensions.insert(authorized_generation.clone());
+                            }
+                            let mut head = get.clone().map_input(|_| HeadObjectInput {
+                                bucket: input.bucket.clone(),
+                                key: "missing".into(),
+                                ..Default::default()
+                            });
+                            head.method = http::Method::HEAD;
+                            let mut list = get.clone().map_input(|_| input.clone());
+                            list.uri = http::Uri::from_static("/?list-type=2");
+                            let usecase = crate::app::object::DefaultObjectUsecase::from_global();
+                            let get_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_get_object(get))
+                                .await
+                                .expect("GET stays local")
+                                .expect_err("local object is absent");
+                            assert_eq!(*get_error.code(), S3ErrorCode::NoSuchKey);
+                            let head_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_head_object(head))
+                                .await
+                                .expect("HEAD stays local")
+                                .expect_err("local object is absent");
+                            assert_eq!(*head_error.code(), S3ErrorCode::NoSuchKey);
+                            let listing = tokio::time::timeout(
+                                Duration::from_secs(10),
+                                DefaultBucketUsecase::from_global().execute_list_objects_v2(list),
+                            )
+                            .await
+                            .expect("LIST stays local")
+                            .expect("replacement bucket lists locally");
+                            assert_eq!(listing.output.key_count, Some(0));
+                        }
+                    }
+                    stop.cancel();
+                    assert!(server.await.expect("source server must remain unused").is_empty());
+                },
+            )
+            .await;
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn list_objects_v1_stays_local_with_xml_safe_key_markers() {
+        run_large_stack_test("list-through-v1-local-markers", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    let (endpoint, server, stop) =
+                        list_source(std::iter::repeat(source_xml(None, false, Some("a-source")))).await;
+                    let (_state_guard, source_input) =
+                        source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await;
+                    let store = shared_gating_ecstore().await;
+                    store
+                        .put_object(
+                            &source_input.bucket,
+                            "a&local",
+                            &mut StoragePutObjReader::from_vec(vec![1]),
+                            &StorageObjectOptions::default(),
+                        )
+                        .await
+                        .expect("seed a second local object");
+
+                    for delimiter in [None, Some("/".to_string())] {
+                        let mut input = ListObjectsInput {
+                            bucket: source_input.bucket.clone(),
+                            max_keys: Some(1),
+                            delimiter,
+                            ..Default::default()
+                        };
+                        for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
+                            let request_marker = input.marker.clone().unwrap_or_default();
+                            let response = tokio::time::timeout(
+                                Duration::from_secs(10),
+                                DefaultBucketUsecase::from_global().execute_list_objects(S3Request {
+                                    input: input.clone(),
+                                    method: http::Method::GET,
+                                    uri: http::Uri::from_static("/"),
+                                    headers: HeaderMap::new(),
+                                    extensions: http::Extensions::new(),
+                                    credentials: None,
+                                    region: None,
+                                    service: None,
+                                    trailing_headers: None,
+                                }),
+                            )
+                            .await
+                            .expect("v1 pagination must finish")
+                            .expect("list-through must not change v1 listing");
+                            assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                            let output = response.output;
+                            let contents = output.contents.as_ref().expect("local page contents");
+                            assert_eq!(contents.len(), 1);
+                            assert_eq!(contents[0].key.as_deref(), Some(expected_key));
+                            assert_eq!(output.marker.as_deref(), Some(request_marker.as_str()));
+                            assert_eq!(output.is_truncated, Some(index == 0));
+                            assert_eq!(output.next_marker.as_deref(), (index == 0).then_some(expected_key));
+
+                            let mut xml = Vec::new();
+                            XmlSerialize::serialize(&output, &mut XmlSerializer::new(&mut xml))
+                                .expect("serialize the real v1 response");
+                            assert!(!xml.contains(&0), "XML 1.0 forbids NUL in NextMarker");
+                            let mut reader = quick_xml::Reader::from_reader(xml.as_slice());
+                            loop {
+                                if reader.read_event().expect("v1 response must be well-formed XML")
+                                    == quick_xml::events::Event::Eof
+                                {
+                                    break;
+                                }
+                            }
+                            input.marker = output.next_marker;
+                        }
+                    }
+                    stop.cancel();
+                    let requests = server.await.expect("source server must not panic");
+                    assert!(requests.is_empty(), "ListObjects v1 must issue no remote LIST requests: {requests:?}");
+                },
+            )
+            .await;
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
@@ -842,6 +1155,244 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn list_through_cross_request_empty_cursor_cycle_obeys_policy() {
+        run_large_stack_test("list-through-cross-request-cursor-cycle", || async {
+            temp_env::async_with_vars(
+                [
+                    (ENV_LIST_PROGRESS_TOKENS, Some("true")),
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
+                        let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
+                        let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
+                        let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
+                        let mut seen = std::collections::HashSet::from([input
+                            .continuation_token
+                            .clone()
+                            .expect("the first request resumes source cursor A")]);
+                        let mut client_requests = 0;
+                        let mut empty_pages = 0;
+                        let terminal = tokio::time::timeout(Duration::from_secs(30), async {
+                            loop {
+                                client_requests += 1;
+                                let response = match execute_source_list(input.clone()).await {
+                                    Ok(response) => response,
+                                    Err(error) => break Err(error),
+                                };
+                                if response.headers.contains_key("x-rustfs-on-demand-migration-list") {
+                                    break Ok(response);
+                                }
+                                let output = response.output;
+                                assert!(output.contents.as_ref().is_none_or(Vec::is_empty));
+                                assert!(output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
+                                assert_eq!(output.key_count, Some(0));
+                                assert_eq!(output.is_truncated, Some(true));
+                                let next = output
+                                    .next_continuation_token
+                                    .expect("a truncated page must carry its cursor");
+                                assert!(
+                                    seen.insert(next.clone()),
+                                    "a cross-request source cursor cycle must not return an identical empty merged token"
+                                );
+                                empty_pages += 1;
+                                input.continuation_token = Some(next);
+                            }
+                        })
+                        .await
+                        .expect("a source cursor cycle must terminate within a bounded client pagination chain");
+                        assert_eq!(empty_pages, usize::from(MAX_LIST_NO_PROGRESS_PAGES - 1));
+                        assert_eq!(client_requests, usize::from(MAX_LIST_NO_PROGRESS_PAGES));
+                        assert_source_policy_result(terminal, policy);
+                        stop.cancel();
+                        let requests = tokio::time::timeout(Duration::from_secs(5), server)
+                            .await
+                            .expect("cyclic source server must stop")
+                            .expect("cyclic source server must not panic");
+                        assert_eq!(requests.len(), 2 * client_requests, "the sixteenth empty page exhausts the budget");
+                        for (index, request) in requests.iter().enumerate() {
+                            let source_cursor = ["A", "B", "C"][index % 3];
+                            assert!(
+                                request.contains(&format!("continuation-token={source_cursor}")),
+                                "the real SDK must follow the returned source cursor: {request}"
+                            );
+                        }
+                    }
+                },
+            )
+            .await;
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn list_through_default_rollout_continues_v2_without_issuing_it_from_v1() {
+        run_large_stack_test("list-through-reader-first-rollout", || async {
+            temp_env::async_with_vars(
+                [
+                    (ENV_LIST_PROGRESS_TOKENS, None),
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
+                        let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
+                        let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
+                        let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
+                        let original = input.continuation_token.clone();
+                        for _ in 0..3 {
+                            let response = execute_source_list(input.clone()).await.expect("reader-only v1 behavior");
+                            assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                            assert_eq!(response.output.key_count, Some(0));
+                            assert_eq!(response.output.is_truncated, Some(true));
+                            let next = response.output.next_continuation_token.expect("resumable empty page");
+                            let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
+                            let decoded = std::str::from_utf8(&raw).expect("JSON token");
+                            let token = decode_list_cursor(Some(decoded)).expect("v1 reader").expect("merged token");
+                            assert_eq!(token.v, 1, "the default rollout cannot begin issuing v2");
+                            assert_eq!(token.no_progress, None);
+                            assert!(!decoded.contains("no_progress"), "ordinary v1 wire shape stays unchanged");
+                            input.continuation_token = Some(next);
+                        }
+                        assert_eq!(input.continuation_token, original, "default rollout retains the known v1 limitation");
+
+                        let raw = base64_simd::STANDARD
+                            .decode_to_vec(input.continuation_token.as_ref().expect("v1 token"))
+                            .expect("base64 continuation token");
+                        let mut token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
+                            .expect("v1 reader")
+                            .expect("merged token");
+                        token.v = 2;
+                        token.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 2);
+                        input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()));
+                        let response = execute_source_list(input.clone()).await.expect("reader-only node resumes v2");
+                        assert_eq!(response.output.key_count, Some(0));
+                        assert_eq!(response.output.is_truncated, Some(true));
+                        let next = response.output.next_continuation_token.expect("last allowed empty cursor");
+                        let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
+                        let token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
+                            .expect("v2 reader")
+                            .expect("merged token");
+                        assert_eq!(token.v, 2);
+                        assert_eq!(token.no_progress, Some(MAX_LIST_NO_PROGRESS_PAGES - 1));
+                        input.continuation_token = Some(next);
+                        assert_source_policy_result(execute_source_list(input).await, policy);
+                        stop.cancel();
+                        let requests = tokio::time::timeout(Duration::from_secs(5), server)
+                            .await
+                            .expect("cyclic source server must stop")
+                            .expect("source server must not panic");
+                        assert_eq!(requests.len(), 10, "five handler requests each fetched two source pages");
+                        for (index, request) in requests.iter().enumerate() {
+                            let cursor = ["A", "B", "C"][index % 3];
+                            assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
+                        }
+                    }
+                },
+            )
+            .await;
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn list_through_empty_advancing_pages_resume_across_handler_requests() {
+        run_large_stack_test("list-through-resumable-empty-pages", || async {
+            temp_env::async_with_vars(
+                [
+                    (ENV_LIST_PROGRESS_TOKENS, Some("true")),
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    for filter_prefix in [None, Some("photos/2024/")] {
+                        let source_key = filter_prefix.map_or("a-source", |_| "photos/2024/a-source");
+                        let (endpoint, server) = scripted_list_source(vec![
+                            source_xml(Some("A"), true, None),
+                            source_xml(Some("B"), true, None),
+                            source_xml(Some("C"), true, None),
+                            source_xml(None, false, Some(source_key)),
+                        ])
+                        .await;
+                        let (_state_guard, mut input) =
+                            source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, filter_prefix).await;
+                        let first = execute_source_list(input.clone())
+                            .await
+                            .expect("valid empty pages must remain resumable");
+                        assert!(!first.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                        assert!(first.output.contents.as_ref().is_none_or(Vec::is_empty));
+                        assert!(first.output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
+                        assert_eq!(first.output.key_count, Some(0));
+                        assert_eq!(first.output.is_truncated, Some(true));
+                        input.continuation_token = Some(first.output.next_continuation_token.expect("empty advancing cursor"));
+
+                        let second = execute_source_list(input)
+                            .await
+                            .expect("a progressing empty chain must reach its data");
+                        assert!(!second.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                        let output = second.output;
+                        let objects = output
+                            .contents
+                            .unwrap_or_default()
+                            .into_iter()
+                            .map(|object| object.key.expect("listed object key"))
+                            .collect::<Vec<_>>();
+                        let prefixes = output
+                            .common_prefixes
+                            .unwrap_or_default()
+                            .into_iter()
+                            .map(|prefix| prefix.prefix.expect("listed common prefix"))
+                            .collect::<Vec<_>>();
+                        if filter_prefix.is_some() {
+                            assert_eq!(objects, vec!["z-local"]);
+                            assert_eq!(prefixes, vec!["photos/"]);
+                        } else {
+                            assert_eq!(objects, vec!["a-source", "z-local"]);
+                            assert!(prefixes.is_empty());
+                        }
+                        assert_eq!(output.key_count, Some(2));
+                        assert_eq!(output.is_truncated, Some(false));
+                        assert!(output.next_continuation_token.is_none());
+                        let requests = tokio::time::timeout(Duration::from_secs(5), server)
+                            .await
+                            .expect("finite source connections must finish")
+                            .expect("finite source server must not panic");
+                        assert_eq!(requests.len(), 4);
+                        assert!(!requests[0].contains("continuation-token="));
+                        for (request, cursor) in requests[1..].iter().zip(["A", "B", "C"]) {
+                            assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
+                        }
+                    }
+                },
+            )
+            .await;
+        });
+    }
+
     fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
         match policy {
             SourceErrorPolicy::Propagate => {

rustfs/src/app/bucket_usecase.rs

@@ -20,7 +20,7 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
 use super::storage_api::bucket_usecase::access::ReqInfo;
 use super::storage_api::bucket_usecase::access::{
     authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
-    prepare_list_buckets_iam_authorization, req_info_ref,
+    prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
@@ -2724,7 +2724,14 @@ impl DefaultBucketUsecase {
 
     #[instrument(level = "trace", skip(self, req))]
     pub async fn execute_list_objects_v2(&self, req: S3Request<ListObjectsV2Input>) -> S3Result<S3Response<ListObjectsV2Output>> {
-        // warn!("list_objects_v2 req {:?}", &req.input);
+        self.execute_list_objects_v2_inner(req, true).await
+    }
+
+    async fn execute_list_objects_v2_inner(
+        &self,
+        mut req: S3Request<ListObjectsV2Input>,
+        allow_list_through: bool,
+    ) -> S3Result<S3Response<ListObjectsV2Output>> {
         let ListObjectsV2Input {
             bucket,
             continuation_token,
@@ -2735,7 +2742,7 @@ impl DefaultBucketUsecase {
             prefix,
             start_after,
             ..
-        } = req.input;
+        } = req.input.clone();
 
         let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
 
@@ -2750,8 +2757,18 @@ impl DefaultBucketUsecase {
         // The on-demand migration envelope is decoded whether or not this
         // bucket still merges: a token handed out under `list_through` must keep
         // paginating after the policy is turned off (rustfs/backlog#2164).
-        let merged_token = list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?;
-        let (object_infos, degraded) = match list_through::list_through_state(&bucket, &req.headers) {
+        if allow_list_through {
+            prepare_odm_read_generation(&store, &mut req, &bucket).await;
+        }
+        let (merged_token, source_state) = if allow_list_through {
+            (
+                list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?,
+                list_through::list_through_state(&store, &bucket, &req, &params).await?,
+            )
+        } else {
+            (None, None)
+        };
+        let (object_infos, degraded) = match source_state {
             Some(state) => {
                 let outcome = list_through::merged_list_objects_v2(
                     &store,
@@ -2938,7 +2955,9 @@ impl DefaultBucketUsecase {
     #[instrument(level = "debug", skip(self, req))]
     pub async fn execute_list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
         let request_marker = req.input.marker.clone();
-        let v2_resp = self.execute_list_objects_v2(req.map_input(Into::into)).await?;
+        // V1 markers are object keys, so they cannot carry the opaque merged
+        // pagination state used by V2 list-through.
+        let v2_resp = self.execute_list_objects_v2_inner(req.map_input(Into::into), false).await?;
 
         Ok(v2_resp.map_output(|v2| build_list_objects_output(v2, request_marker)))
     }

rustfs/src/app/object/get.rs

@@ -15,11 +15,11 @@
 //! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning.
 
 use super::*;
-use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+use crate::on_demand_migration::WriteBackBody;
+use crate::on_demand_migration::{
     BucketOdmState, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, PullError, PullLeader, PullOutcome, PullReason, PullSlot,
     RangeGetPolicy, SourceBody, SourceClient, SourceError, SourceGet, SourceHead, commit_inline, idle_guarded_body,
 };
-use crate::app::storage_api::object_usecase::on_demand_migration::WriteBackBody;
 use rustfs_rio::{TeeOptions, TeePrimary, tee_reader_with_options};
 use tokio_stream::wrappers::ReceiverStream;
 
@@ -3843,11 +3843,11 @@ impl DefaultObjectUsecase {
         if !odm_get_may_consult_source(opts, part_number) {
             return None;
         }
-        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
-        let (state, client) = match odm_get_verdict(lookup) {
-            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
-            OdmGetVerdict::Consult { state, client } => (state, client),
-        };
+        let sys = OnDemandMigrationSys::get();
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
         let policy = &state.config().policy;
         // The read path reports a latest delete marker as a plain 404, so the
         // marker is classified here, and only where one can exist.
@@ -3861,6 +3861,24 @@ impl DefaultObjectUsecase {
                 None => return Some(OdmGetOutcome::RetryLocal),
             }
         }
+        let expected_incarnation = match odm_read_generation(req, bucket) {
+            Ok(Some(incarnation)) => incarnation,
+            Ok(None) => return None,
+            Err(err) => return Some(OdmGetOutcome::Respond(Err(err))),
+        };
+        match store.bucket_incarnation_id(bucket).await {
+            Ok(current) if current == expected_incarnation => {}
+            Ok(_) => return None,
+            Err(err) => return Some(OdmGetOutcome::Respond(Err(ApiError::from(err).into()))),
+        }
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
+        let (state, client) = match odm_get_verdict(lookup) {
+            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
+            OdmGetVerdict::Consult { state, client } => (state, client),
+        };
         let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         let reply = odm_get_from_source(&state, client.as_ref(), &req.headers, key, range, request_context).await;
         Some(match reply {
@@ -3895,7 +3913,7 @@ impl DefaultObjectUsecase {
         result
     }
 
-    async fn execute_get_object_inner(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
+    async fn execute_get_object_inner(&self, mut req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
         let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event();
 
         if let Some(context) = &self.context {
@@ -3981,6 +3999,8 @@ impl DefaultObjectUsecase {
                 return Self::complete_get_object_error(helper, err);
             }
         };
+        let bucket = req.input.bucket.clone();
+        prepare_odm_read_generation(&store, &mut req, &bucket).await;
         if let Some(request_context_start) = request_context_start {
             rustfs_io_metrics::record_get_object_stage_duration(
                 "s3_handler",
@@ -4615,6 +4635,7 @@ fn odm_inline_client_body(primary: TeePrimary) -> StreamingBlob {
 async fn odm_get_passthrough<S: OdmGetSource>(
     state: &Arc<BucketOdmState>,
     source: &S,
+    headers: &HeaderMap,
     key: &str,
     range: Option<&HTTPRangeSpec>,
     backfill: Option<PullReason>,
@@ -4623,6 +4644,9 @@ async fn odm_get_passthrough<S: OdmGetSource>(
         Ok(get) => get,
         Err(err) => return OdmGetReply::Error(odm_get_source_failure(state, &err)),
     };
+    if let Err(err) = odm_check_source_preconditions(headers, &get.head) {
+        return OdmGetReply::Error(err);
+    }
     let content_length = match odm_content_length(get.head.size) {
         Ok(length) => length,
         Err(err) => {
@@ -4648,6 +4672,7 @@ async fn odm_get_passthrough<S: OdmGetSource>(
 async fn odm_get_inline<S: OdmGetSource>(
     state: &Arc<BucketOdmState>,
     source: &S,
+    headers: &HeaderMap,
     key: &str,
     leader: PullLeader,
     request_context: Option<request_context::RequestContext>,
@@ -4676,6 +4701,12 @@ async fn odm_get_inline<S: OdmGetSource>(
         body,
         content_range,
     } = get;
+    // HEAD and GET can observe different source versions. Validate the
+    // representation whose body will actually be returned and persisted.
+    if let Err(err) = odm_check_source_preconditions(headers, &head) {
+        leader.complete(Err(PullError::canceled("source GET did not satisfy request preconditions")));
+        return OdmGetReply::Error(err);
+    }
     // The object outgrew the inline budget between HEAD and GET: followers
     // stream through on their own and the background pull stores it.
     if head.size > policy.inline_max_bytes {
@@ -4758,19 +4789,19 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
     let policy = &state.config().policy;
     if let Some(range) = range {
         let backfill = (policy.range_get == RangeGetPolicy::ServeAndBackfill).then_some(PullReason::RangeGet);
-        return odm_get_passthrough(state, source, key, Some(range), backfill).await;
+        return odm_get_passthrough(state, source, headers, key, Some(range), backfill).await;
     }
     if head.size > policy.inline_max_bytes {
-        return odm_get_passthrough(state, source, key, None, Some(PullReason::LargeObject)).await;
+        return odm_get_passthrough(state, source, headers, key, None, Some(PullReason::LargeObject)).await;
     }
     let slot = match state.acquire_pull_slot(key).await {
         Ok(slot) => slot,
         // The bucket state was torn down under this request: serve it
         // without queueing anything on the old state.
-        Err(_) => return odm_get_passthrough(state, source, key, None, None).await,
+        Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await,
     };
     match slot {
-        PullSlot::Leader(leader) => odm_get_inline(state, source, key, leader, request_context).await,
+        PullSlot::Leader(leader) => odm_get_inline(state, source, headers, key, leader, request_context).await,
         PullSlot::Follower(follower) => {
             let first_byte = Duration::from_millis(policy.source_timeout.first_byte_ms);
             match tokio::time::timeout(first_byte, follower.wait()).await {
@@ -4778,7 +4809,7 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
                     stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
                     OdmGetReply::RetryLocal
                 }
-                Ok(Err(_)) | Err(_) => odm_get_passthrough(state, source, key, None, None).await,
+                Ok(Err(_)) | Err(_) => odm_get_passthrough(state, source, headers, key, None, None).await,
             }
         }
     }
@@ -4787,11 +4818,11 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
 #[cfg(test)]
 mod on_demand_migration_tests {
     use super::*;
-    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+    use crate::on_demand_migration::{
         BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
         Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
     };
-    use crate::app::storage_api::object_usecase::on_demand_migration::{
+    use crate::on_demand_migration::{
         LocalObject, OdmWriteBack, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
     };
     use async_trait::async_trait;
@@ -4821,6 +4852,8 @@ mod on_demand_migration_tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: None,
@@ -4906,7 +4939,7 @@ mod on_demand_migration_tests {
             Err(WriteBackError::Local("multipart is not part of the inline path".to_string()))
         }
 
-        async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, _upload_id: &str) -> Result<(), WriteBackError> {
+        async fn abort_multipart_upload(&self, _request: &WriteBackRequest, _upload_id: &str) -> Result<(), WriteBackError> {
             Ok(())
         }
     }
@@ -5296,6 +5329,73 @@ mod on_demand_migration_tests {
         assert!(rt.write_back.puts().is_empty());
     }
 
+    #[tokio::test]
+    async fn odm_get_rechecks_conditions_against_the_get_representation() {
+        for inline_max_bytes in [0, 1024] {
+            for range in [
+                None,
+                Some(HTTPRangeSpec {
+                    is_suffix_length: false,
+                    start: 0,
+                    end: 2,
+                }),
+            ] {
+                let rt = runtime(
+                    "changed-source",
+                    PolicyConfig {
+                        inline_max_bytes,
+                        ..Default::default()
+                    },
+                )
+                .await;
+                let state = rt.state("changed-source");
+                let before = source_head(b"before");
+                let after = source_head(b"after!");
+                let source = ScriptedSource::new(vec![Ok(before.clone())], vec![Ok((after, b"after!".to_vec(), None))]);
+                let mut headers = HeaderMap::new();
+                headers.insert(
+                    http::header::IF_MATCH,
+                    HeaderValue::from_str(&format!("\"{}\"", before.etag.expect("etag"))).expect("header"),
+                );
+                let error = failed(odm_get_from_source(&state, &source, &headers, KEY, range.as_ref(), None).await);
+                assert_eq!(error.code(), &S3ErrorCode::PreconditionFailed);
+                assert_eq!(source.get_calls(), 1);
+                assert_eq!(state.inflight_keys(), 0);
+                assert!(rt.write_back.puts().is_empty(), "a failed condition must not start write-back");
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn odm_get_missing_validators_cannot_bypass_a_condition() {
+        for inline_max_bytes in [0, 1024] {
+            let rt = runtime(
+                "missing-validator",
+                PolicyConfig {
+                    inline_max_bytes,
+                    ..Default::default()
+                },
+            )
+            .await;
+            let state = rt.state("missing-validator");
+            let before = source_head(b"before");
+            let after = SourceHead {
+                size: 6,
+                ..Default::default()
+            };
+            let source = ScriptedSource::new(vec![Ok(before.clone())], vec![Ok((after, b"after!".to_vec(), None))]);
+            let mut headers = HeaderMap::new();
+            headers.insert(
+                http::header::IF_MATCH,
+                HeaderValue::from_str(&format!("\"{}\"", before.etag.expect("etag"))).expect("header"),
+            );
+            let error = failed(odm_get_from_source(&state, &source, &headers, KEY, None, None).await);
+            assert_eq!(error.status_code(), Some(StatusCode::FAILED_DEPENDENCY));
+            assert_eq!(error.message(), Some("missing_source_validator"));
+            assert!(rt.write_back.puts().is_empty());
+        }
+    }
+
     #[tokio::test]
     async fn odm_get_source_not_found_is_404_and_negative_cached() {
         let rt = runtime("n", PolicyConfig::default()).await;

rustfs/src/app/object/head.rs

@@ -15,7 +15,7 @@
 //! HeadObject path.
 
 use super::*;
-use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+use crate::on_demand_migration::{
     BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead,
 };
 
@@ -146,6 +146,8 @@ impl DefaultObjectUsecase {
     /// `None` means the runtime does not intervene and the caller keeps its
     /// original 404. The source answer is never written back or queued.
     async fn on_demand_migration_head(
+        req: &S3Request<HeadObjectInput>,
+        store: &ECStore,
         bucket: &str,
         key: &str,
         opts: &ObjectOptions,
@@ -154,7 +156,33 @@ impl DefaultObjectUsecase {
         if !odm_request_may_consult_source(opts) {
             return None;
         }
-        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
+        let sys = OnDemandMigrationSys::get();
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
+        let policy = &state.config().policy;
+        if !odm_policy_admits_miss(policy, miss) {
+            return None;
+        }
+        if policy.head == HeadPolicy::LocalOnly {
+            state.stats().record_request(OdmOp::Head, OdmOutcome::Filtered);
+            return None;
+        }
+        let expected_incarnation = match odm_read_generation(req, bucket) {
+            Ok(Some(incarnation)) => incarnation,
+            Ok(None) => return None,
+            Err(err) => return Some(Err(err)),
+        };
+        match store.bucket_incarnation_id(bucket).await {
+            Ok(current) if current == expected_incarnation => {}
+            Ok(_) => return None,
+            Err(err) => return Some(Err(ApiError::from(err).into())),
+        }
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
         match odm_head_verdict(lookup, miss) {
             OdmHeadVerdict::Ignore => None,
             OdmHeadVerdict::Fail(err) => Some(Err(err)),
@@ -269,7 +297,7 @@ impl DefaultObjectUsecase {
     }
 
     #[instrument(level = "debug", skip(self, req))]
-    pub async fn execute_head_object(&self, req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
+    pub async fn execute_head_object(&self, mut req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
         if let Some(context) = &self.context {
             let _ = context.object_store();
         }
@@ -314,6 +342,8 @@ impl DefaultObjectUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        prepare_odm_read_generation(&store, &mut req, &bucket).await;
+
         // Modification Points: Explicitly handles get_object_info errors, distinguishing between object absence and other errors
         let lookup = store.get_object_info(&bucket, &key, &opts).await;
         // Single classification point for the on-demand migration gate
@@ -347,7 +377,7 @@ impl DefaultObjectUsecase {
                         return result;
                     }
                     if let Some(miss) = odm_miss
-                        && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
+                        && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
                     {
                         return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                     }
@@ -362,7 +392,7 @@ impl DefaultObjectUsecase {
                 // A latest delete marker is a local miss the source may still
                 // answer when the bucket policy says so.
                 if let Some(miss) = odm_miss
-                    && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
+                    && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
                 {
                     return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                 }
@@ -638,7 +668,7 @@ impl DefaultObjectUsecase {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+    use crate::on_demand_migration::{
         BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
         Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
     };
@@ -665,6 +695,8 @@ mod tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: None,
@@ -737,12 +769,11 @@ mod tests {
             user_metadata: HashMap::from([("owner".to_string(), "alice".to_string())]),
             version_id: Some("v1".to_string()),
             storage_class: Some("STANDARD_IA".to_string()),
-            sse: Some(
-                crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse::Kms {
-                    key_id: Some("key-1".to_string()),
-                },
-            ),
+            sse: Some(crate::on_demand_migration::source_client::SourceSse::Kms {
+                key_id: Some("key-1".to_string()),
+            }),
             is_multipart_etag: true,
+            etag_is_opaque: false,
         }
     }
 
@@ -1052,7 +1083,12 @@ mod tests {
             head: HeadPolicy::LocalOnly,
             ..Default::default()
         });
-        sys.apply(&bucket, Some(&cfg)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            Some(&cfg),
+        )
+        .await;
         let state = sys.state(&bucket).expect("bucket runtime installed");
 
         // Local hit: served locally, the runtime is never entered.
@@ -1108,7 +1144,12 @@ mod tests {
         );
 
         cfg.policy.respect_local_delete_marker = false;
-        sys.apply(&bucket, Some(&cfg)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            Some(&cfg),
+        )
+        .await;
         let err = Box::pin(usecase.execute_head_object(head_input(&bucket, "present", None)))
             .await
             .expect_err("local_only still answers 404");

rustfs/src/app/object/internal_put.rs

@@ -48,6 +48,8 @@ use http::HeaderName;
 /// other internal provenance is written verbatim.
 pub(crate) struct InternalPutContext {
     pub(crate) bucket: String,
+    /// Pins background work to its original bucket across deletion and recreation.
+    pub(crate) expected_bucket_incarnation_id: Option<Uuid>,
     pub(crate) key: String,
     /// Plaintext object length. The single-object path requires it, exactly
     /// like S3 PutObject rejects an unknown `Content-Length`.
@@ -57,6 +59,9 @@ pub(crate) struct InternalPutContext {
     pub(crate) expected_md5_hex: Option<String>,
     /// ETag to store instead of the computed one.
     pub(crate) preserve_etag: Option<String>,
+    /// Reject an existing current object under the storage commit lock.
+    pub(crate) if_absent: bool,
+    pub(crate) preserve_delete_marker: bool,
     pub(crate) content_headers: HashMap<String, String>,
     pub(crate) user_metadata: HashMap<String, String>,
     pub(crate) tags: Option<String>,
@@ -236,10 +241,13 @@ impl DefaultObjectUsecase {
         let start_time = Instant::now();
         let InternalPutContext {
             bucket,
+            expected_bucket_incarnation_id,
             key,
             size,
             expected_md5_hex,
             preserve_etag,
+            if_absent,
+            preserve_delete_marker,
             content_headers,
             user_metadata,
             tags,
@@ -252,7 +260,10 @@ impl DefaultObjectUsecase {
         };
         let size = i64::try_from(size).map_err(|_| ApiError::invalid_request("internal put size exceeds the supported range"))?;
 
-        let headers = internal_put_headers(&content_headers)?;
+        let mut headers = internal_put_headers(&content_headers)?;
+        if if_absent {
+            headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
+        }
         validate_internal_write_target(&key, &bucket, &headers).await?;
         remove_source_replication_bookkeeping(&mut internal_metadata);
 
@@ -287,6 +298,8 @@ impl DefaultObjectUsecase {
             origin: PutObjectOrigin::Internal {
                 principal_id,
                 emit_events,
+                preserve_delete_marker,
+                expected_bucket_incarnation_id,
             },
         };
         let committed = self
@@ -358,6 +371,8 @@ impl DefaultObjectUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
+
         let dsc = must_replicate_object(
             &ctx.bucket,
             &ctx.key,
@@ -419,7 +434,10 @@ impl DefaultObjectUsecase {
         let bucket = ctx.bucket.as_str();
         let key = ctx.key.as_str();
         let store = self.object_store().ok_or_else(not_initialized)?;
-        let mut opts = ObjectOptions::default();
+        let mut opts = ObjectOptions {
+            expected_bucket_incarnation_id: ctx.expected_bucket_incarnation_id,
+            ..Default::default()
+        };
         let session = store
             .get_multipart_info(bucket, key, upload_id, &opts)
             .await
@@ -527,10 +545,15 @@ impl DefaultObjectUsecase {
             .map_err(api_error_from_s3)?;
         let store = self.object_store().ok_or_else(not_initialized)?;
 
-        let headers = HeaderMap::new();
+        let mut headers = HeaderMap::new();
+        if ctx.if_absent {
+            headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
+        }
         let mut opts =
             get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?;
+        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
         opts.preserve_etag = ctx.preserve_etag.clone();
+        opts.preserve_delete_marker = ctx.preserve_delete_marker;
         let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
         opts.versioned = versioned;
         opts.version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &key).await;
@@ -686,10 +709,24 @@ impl DefaultObjectUsecase {
     }
 
     /// Discard an internal multipart upload and its staged parts.
-    pub(crate) async fn internal_abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), ApiError> {
+    pub(crate) async fn internal_abort_multipart_upload(
+        &self,
+        bucket: &str,
+        key: &str,
+        upload_id: &str,
+        expected_bucket_incarnation_id: Option<Uuid>,
+    ) -> Result<(), ApiError> {
         let store = self.object_store().ok_or_else(not_initialized)?;
         store
-            .abort_multipart_upload(bucket, key, upload_id, &ObjectOptions::default())
+            .abort_multipart_upload(
+                bucket,
+                key,
+                upload_id,
+                &ObjectOptions {
+                    expected_bucket_incarnation_id,
+                    ..Default::default()
+                },
+            )
             .await
             .map_err(ApiError::from)?;
         rustfs_scanner::record_dirty_usage_bucket(bucket);
@@ -743,10 +780,13 @@ mod tests {
     fn internal_context(bucket: &str, key: &str, body: &[u8]) -> InternalPutContext {
         InternalPutContext {
             bucket: bucket.to_string(),
+            expected_bucket_incarnation_id: None,
             key: key.to_string(),
             size: Some(body.len() as u64),
             expected_md5_hex: Some(md5_hex(body)),
             preserve_etag: None,
+            if_absent: false,
+            preserve_delete_marker: false,
             content_headers: HashMap::from([
                 ("Content-Type".to_string(), "text/plain".to_string()),
                 ("Cache-Control".to_string(), "max-age=60".to_string()),
@@ -1123,9 +1163,14 @@ mod tests {
         ))
         .await
         .expect("part of the aborted upload must stage");
-        Box::pin(usecase.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id))
-            .await
-            .expect("internal abort must succeed");
+        Box::pin(usecase.internal_abort_multipart_upload(
+            &bucket,
+            &ctx.key,
+            &aborted_upload_id,
+            ctx.expected_bucket_incarnation_id,
+        ))
+        .await
+        .expect("internal abort must succeed");
         let uploads = Box::pin(store.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100))
             .await
             .expect("list multipart uploads after abort");

rustfs/src/app/object/mod.rs

@@ -21,8 +21,8 @@ use crate::storage_api::table::get_bucket_metadata;
 
 use super::storage_api::object_usecase::access::{
     PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
-    has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized,
-    replication_request_authorized, req_info_mut, req_info_ref,
+    has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
+    recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::object_usecase::bucket::quota::BucketQuota;

rustfs/src/app/object/on_demand_migration_put.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 //! On-demand migration write-back (rustfs/backlog#2153): the app-layer
-//! [`OdmWriteBack`] the ecstore pull pipeline stores source objects with.
+//! [`OdmWriteBack`] the migration service stores source objects with.
 //!
 //! Every write goes through the internal put entry points, so a pulled
 //! object is indistinguishable from a client PUT: bucket default SSE, quota,
@@ -34,7 +34,7 @@
 use super::*;
 
 use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
-use crate::app::storage_api::object_usecase::on_demand_migration::{
+use crate::on_demand_migration::{
     LocalObject, OdmWriteBack, SourceHead, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
     is_multipart_etag,
 };
@@ -66,6 +66,15 @@ impl OnDemandMigrationWriteBack {
             .object_store()
             .ok_or_else(|| WriteBackError::Local("object store is not initialized".to_string()))
     }
+
+    fn require_atomic_write_back(&self) -> Result<(), WriteBackError> {
+        if !self.store()?.supports_atomic_create_only_write_back() {
+            return Err(WriteBackError::Unsupported(
+                "write-back requires namespace locking and exactly one pool with one erasure set".to_string(),
+            ));
+        }
+        Ok(())
+    }
 }
 
 fn rfc3339(time: OffsetDateTime) -> String {
@@ -115,6 +124,12 @@ pub(super) fn expected_md5_hex(head: &SourceHead) -> Option<String> {
     if head.sse.is_some() {
         return None;
     }
+    // Azure stamps an opaque concurrency token in the ETag slot. It is
+    // recorded as provenance, but reading it as a digest would compare the
+    // pulled bytes against a value that never described them.
+    if head.etag_is_opaque {
+        return None;
+    }
     let etag = head.etag.as_deref()?;
     if etag.len() != 32 || is_multipart_etag(etag) || !etag.bytes().all(|byte| byte.is_ascii_hexdigit()) {
         return None;
@@ -157,10 +172,13 @@ pub(super) async fn write_back_context(request: &WriteBackRequest, single_part:
     };
     InternalPutContext {
         bucket: request.bucket.clone(),
+        expected_bucket_incarnation_id: Some(request.bucket_incarnation_id),
         key: request.key.clone(),
         size: Some(head.size),
         expected_md5_hex: single_part.then(|| expected_md5_hex(head)).flatten(),
         preserve_etag,
+        if_absent: true,
+        preserve_delete_marker: request.respect_delete_marker,
         content_headers: content_headers(head),
         user_metadata: head.user_metadata.clone(),
         tags: request.tags.as_ref().and_then(encode_tags),
@@ -207,6 +225,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
     }
 
     async fn put_object(&self, request: &WriteBackRequest, body: WriteBackBody) -> Result<WriteBackOutcome, WriteBackError> {
+        self.require_atomic_write_back()?;
         let ctx = write_back_context(request, true).await;
         self.usecase()
             .internal_put_object(ctx, body)
@@ -216,6 +235,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
     }
 
     async fn create_multipart_upload(&self, request: &WriteBackRequest) -> Result<String, WriteBackError> {
+        self.require_atomic_write_back()?;
         let ctx = write_back_context(request, false).await;
         self.usecase()
             .internal_create_multipart_upload(&ctx)
@@ -249,6 +269,7 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
         upload_id: &str,
         parts: Vec<WriteBackPart>,
     ) -> Result<WriteBackOutcome, WriteBackError> {
+        self.require_atomic_write_back()?;
         let ctx = write_back_context(request, false).await;
         let parts = parts
             .into_iter()
@@ -265,9 +286,9 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
             .map_err(write_back_error)
     }
 
-    async fn abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), WriteBackError> {
+    async fn abort_multipart_upload(&self, request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError> {
         self.usecase()
-            .internal_abort_multipart_upload(bucket, key, upload_id)
+            .internal_abort_multipart_upload(&request.bucket, &request.key, upload_id, Some(request.bucket_incarnation_id))
             .await
             .map_err(write_back_error)
     }
@@ -277,15 +298,15 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
 mod tests {
     use super::*;
     use crate::app::storage_api::multipart_usecase::contract::multipart::MultipartOperations as _;
-    use crate::app::storage_api::object_usecase::on_demand_migration::{PullFailureReason, SourceSse};
     use crate::app::storage_api::s3::{
         BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
         ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
         ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
     };
     use crate::app::storage_api::test::bucket::utils::serialize;
-    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
+    use crate::on_demand_migration::{PullFailureReason, SourceSse};
     use http::Method;
     use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, contains_key_str, get_str};
     use sha2::{Digest as Sha256Digest, Sha256};
@@ -325,15 +346,17 @@ mod tests {
         }
     }
 
-    fn request(bucket: &str, key: &str, head: SourceHead) -> WriteBackRequest {
+    fn request(bucket: &str, bucket_incarnation_id: Uuid, key: &str, head: SourceHead) -> WriteBackRequest {
         WriteBackRequest {
             bucket: bucket.to_string(),
+            bucket_incarnation_id,
             key: key.to_string(),
             head,
             source_label: SOURCE_LABEL.to_string(),
             pulled_at: OffsetDateTime::from_unix_timestamp(1_756_800_000).expect("valid timestamp"),
             preserve_etag: true,
             emit_events: true,
+            respect_delete_marker: true,
             tags: Some(HashMap::from([
                 ("team".to_string(), "storage".to_string()),
                 ("env".to_string(), "prod".to_string()),
@@ -454,7 +477,15 @@ mod tests {
         let body = b"pulled from the legacy bucket".to_vec();
         let head = source_head(&body);
         let outcome = write_back
-            .put_object(&request(&bucket, "dir/obj.txt", head.clone()), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "dir/obj.txt",
+                    head.clone(),
+                ),
+                body_stream(&body),
+            )
             .await
             .expect("write-back must commit");
         assert_eq!(outcome.etag, head.etag, "single-part source ETag is preserved");
@@ -486,6 +517,107 @@ mod tests {
         assert!(!local.delete_marker);
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn write_back_rejects_unsupported_topology_before_any_mutation() {
+        let (_dir, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
+        crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
+        let bucket = "odm-unsupported";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket");
+        let write_back = OnDemandMigrationWriteBack::new();
+        let req = request(
+            bucket,
+            store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"),
+            "key",
+            source_head(b"source"),
+        );
+        assert!(matches!(
+            write_back.put_object(&req, body_stream(b"source")).await,
+            Err(WriteBackError::Unsupported(_))
+        ));
+        assert!(matches!(
+            write_back.create_multipart_upload(&req).await,
+            Err(WriteBackError::Unsupported(_))
+        ));
+        assert!(matches!(
+            write_back.complete_multipart_upload(&req, "no-session", Vec::new()).await,
+            Err(WriteBackError::Unsupported(_))
+        ));
+        assert_nothing_left(&store, bucket, "key").await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn stale_write_back_cannot_mutate_a_recreated_bucket() {
+        let (store, bucket) = write_back_test_bucket("odm-wb-incarnation", false).await;
+        let old_id = store.bucket_incarnation_id(&bucket).await.expect("old bucket incarnation");
+        let stale = request(&bucket, old_id, "object", source_head(b"source"));
+        let (resume, wait) = tokio::sync::oneshot::channel();
+        let delayed = {
+            let stale = stale.clone();
+            tokio::spawn(async move {
+                wait.await.expect("resume old source pull");
+                OnDemandMigrationWriteBack::new()
+                    .put_object(&stale, body_stream(b"source"))
+                    .await
+            })
+        };
+        store
+            .delete_bucket(
+                &bucket,
+                &DeleteBucketOptions {
+                    force: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("delete original bucket");
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let new_id = store.bucket_incarnation_id(&bucket).await.expect("replacement incarnation");
+        assert_ne!(old_id, new_id);
+        resume.send(()).expect("release delayed pull");
+        assert!(delayed.await.expect("delayed pull task").is_err());
+        assert_nothing_left(&store, &bucket, "object").await;
+
+        let write_back = OnDemandMigrationWriteBack::new();
+        assert!(write_back.create_multipart_upload(&stale).await.is_err());
+        let current = request(&bucket, new_id, "object", source_head(b"current"));
+        let upload = write_back
+            .create_multipart_upload(&current)
+            .await
+            .expect("create replacement upload");
+        // A stale capability must fail independently of whether its upload ID
+        // happens to name a valid session in the replacement bucket.
+        assert!(
+            write_back
+                .upload_part(&stale, &upload, 1, 6, body_stream(b"source"))
+                .await
+                .is_err()
+        );
+        let part = write_back
+            .upload_part(&current, &upload, 1, 7, body_stream(b"current"))
+            .await
+            .expect("stage current part");
+        assert!(
+            write_back
+                .complete_multipart_upload(&stale, &upload, vec![part.clone()])
+                .await
+                .is_err()
+        );
+        assert!(write_back.abort_multipart_upload(&stale, &upload).await.is_err());
+        write_back
+            .complete_multipart_upload(&current, &upload, vec![part])
+            .await
+            .expect("stale cleanup preserves replacement upload");
+        assert_eq!(raw_object_bytes(&store, &bucket, "object").await, b"current");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn write_back_integrity_failure_leaves_nothing_behind() {
@@ -495,7 +627,15 @@ mod tests {
         head.etag = Some(md5_hex(b"a different body"));
 
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "wrong.bin", head), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "wrong.bin",
+                    head,
+                ),
+                body_stream(&body),
+            )
             .await
             .expect_err("an ETag mismatch must fail the write-back");
         assert_eq!(err, WriteBackError::Integrity);
@@ -503,6 +643,159 @@ mod tests {
         assert_nothing_left(&store, &bucket, "wrong.bin").await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn write_back_commit_does_not_overwrite_a_concurrent_client_put() {
+        use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
+        for versioned in [false, true] {
+            let (store, bucket) = write_back_test_bucket("odm-wb-race", versioned).await;
+            let source = b"old source bytes";
+            let client = b"new client bytes";
+            let req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(source),
+            );
+            let client_req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(client),
+            );
+            let mut client_ctx = write_back_context(&client_req, true).await;
+            client_ctx.if_absent = false;
+            let client_after = PutObjectCommitBarrier::install(&bucket, "race", PutObjectCommitPause::AfterNamespace);
+            let client_put = tokio::spawn(async move {
+                DefaultObjectUsecase::from_global()
+                    .internal_put_object(client_ctx, body_stream(client))
+                    .await
+            });
+            client_after.wait_until_paused().await;
+            let source_before = PutObjectCommitBarrier::install(&bucket, "race", PutObjectCommitPause::BeforeNamespace);
+            let write_back = OnDemandMigrationWriteBack::new();
+            let (result, ()) = tokio::join!(write_back.put_object(&req, body_stream(source)), async {
+                source_before.wait_until_paused().await;
+                drop(source_before);
+                drop(client_after);
+            });
+            let committed = client_put.await.expect("client task").expect("ordinary client write wins");
+            assert!(
+                matches!(result, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")),
+                "{result:?}"
+            );
+            let stored = stored_object(&store, &bucket, "race").await;
+            assert_eq!(stored.etag, committed.etag);
+            assert_eq!(stored.version_id, committed.version_id);
+            assert_eq!(committed.version_id.is_some(), versioned);
+            assert_eq!(raw_object_bytes(&store, &bucket, "race").await, client);
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn write_back_multipart_completion_preserves_a_client_put_after_staging() {
+        let (store, bucket) = write_back_test_bucket("odm-mpu-race", false).await;
+        let write_back = OnDemandMigrationWriteBack::new();
+        let req = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "race",
+            source_head(b"source"),
+        );
+        let upload_id = write_back.create_multipart_upload(&req).await.expect("create");
+        let part = write_back
+            .upload_part(&req, &upload_id, 1, 6, body_stream(b"source"))
+            .await
+            .expect("stage");
+        let mut client_ctx = write_back_context(
+            &request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(b"client"),
+            ),
+            true,
+        )
+        .await;
+        client_ctx.if_absent = false;
+        let committed = DefaultObjectUsecase::from_global()
+            .internal_put_object(client_ctx, body_stream(b"client"))
+            .await
+            .expect("client put after staging");
+        let result = write_back.complete_multipart_upload(&req, &upload_id, vec![part]).await;
+        assert!(
+            matches!(result, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")),
+            "{result:?}"
+        );
+        write_back
+            .abort_multipart_upload(&req, &upload_id)
+            .await
+            .expect("abort rejected upload");
+        let stored = stored_object(&store, &bucket, "race").await;
+        assert_eq!(stored.etag, committed.etag);
+        assert_eq!(stored.version_id, committed.version_id);
+        assert_eq!(raw_object_bytes(&store, &bucket, "race").await, b"client");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn write_back_preserves_delete_markers_unless_policy_allows_revival() {
+        for multipart in [false, true] {
+            let (store, bucket) = write_back_test_bucket("odm-wb-tombstone", true).await;
+            let write_back = OnDemandMigrationWriteBack::new();
+            let mut req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "deleted",
+                source_head(b"source"),
+            );
+            let staged = if multipart {
+                let id = write_back.create_multipart_upload(&req).await.expect("create");
+                let part = write_back
+                    .upload_part(&req, &id, 1, 6, body_stream(b"source"))
+                    .await
+                    .expect("part");
+                Some((id, part))
+            } else {
+                None
+            };
+            store
+                .delete_object(
+                    &bucket,
+                    "deleted",
+                    ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("delete marker");
+            let marker = stored_object(&store, &bucket, "deleted").await;
+            assert!(marker.delete_marker);
+            let rejected = if let Some((id, part)) = staged {
+                let result = write_back.complete_multipart_upload(&req, &id, vec![part]).await;
+                write_back.abort_multipart_upload(&req, &id).await.expect("abort");
+                result
+            } else {
+                write_back.put_object(&req, body_stream(b"source")).await
+            };
+            assert!(
+                matches!(rejected, Err(WriteBackError::Local(ref error)) if error.contains("PreconditionFailed")),
+                "{rejected:?}"
+            );
+            let retained = stored_object(&store, &bucket, "deleted").await;
+            assert!(retained.delete_marker);
+            assert_eq!(retained.version_id, marker.version_id);
+            req.respect_delete_marker = false;
+            write_back
+                .put_object(&req, body_stream(b"source"))
+                .await
+                .expect("explicit revival policy");
+            assert!(!stored_object(&store, &bucket, "deleted").await.delete_marker);
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn write_back_truncated_stream_leaves_nothing_behind() {
@@ -516,7 +809,15 @@ mod tests {
             Err(io::Error::new(io::ErrorKind::BrokenPipe, "tee primary dropped before EOF")),
         ]);
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "torn.bin", head.clone()), torn)
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "torn.bin",
+                    head.clone(),
+                ),
+                torn,
+            )
             .await
             .expect_err("a broken stream must fail the write-back");
         assert_ne!(err, WriteBackError::Integrity, "{err}");
@@ -525,7 +826,15 @@ mod tests {
         // A clean EOF short of the advertised size is just as fatal.
         let short = stream(vec![Ok(Bytes::copy_from_slice(&body[..64 * 1024]))]);
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "short.bin", head), short)
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "short.bin",
+                    head,
+                ),
+                short,
+            )
             .await
             .expect_err("a short body must fail the write-back");
         assert!(matches!(err, WriteBackError::Local(_) | WriteBackError::Integrity), "{err}");
@@ -541,7 +850,12 @@ mod tests {
         let mut head = source_head(&body);
         head.etag = Some(format!("{}-2", md5_hex(&body)));
         head.is_multipart_etag = true;
-        let request = request(&bucket, "big/object.bin", head.clone());
+        let request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "big/object.bin",
+            head.clone(),
+        );
         let write_back = OnDemandMigrationWriteBack::new();
 
         let upload_id = write_back.create_multipart_upload(&request).await.expect("create");
@@ -579,10 +893,7 @@ mod tests {
             .upload_part(&request, &aborted, 1, 4096, body_stream(&body[..4096]))
             .await
             .expect("stage part");
-        write_back
-            .abort_multipart_upload(&bucket, "big/object.bin", &aborted)
-            .await
-            .expect("abort");
+        write_back.abort_multipart_upload(&request, &aborted).await.expect("abort");
         let uploads = store
             .list_multipart_uploads(&bucket, "big/object.bin", None, None, None, 100)
             .await
@@ -635,7 +946,12 @@ mod tests {
 
             let body = b"plaintext that must be encrypted at rest".to_vec();
             let head = source_head(&body);
-            let request = request(&bucket, "secret.txt", head.clone());
+            let request = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "secret.txt",
+                head.clone(),
+            );
             // The source ETag is not forced onto an encrypted object; the
             // local ETag is whatever the SSE write path computes.
             assert_eq!(write_back_context(&request, true).await.preserve_etag, None);
@@ -673,7 +989,15 @@ mod tests {
         let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("odm-wb-quota", 64).await;
         let body = vec![0x71; 4096];
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "over.bin", source_head(&body)), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "over.bin",
+                    source_head(&body),
+                ),
+                body_stream(&body),
+            )
             .await
             .expect_err("a full quota must reject the write-back");
         assert!(matches!(err, WriteBackError::Quota(_)), "{err}");
@@ -738,7 +1062,12 @@ mod tests {
 
         let body = b"replicate me".to_vec();
         let head = source_head(&body);
-        let request = request(&bucket, "replicated.txt", head);
+        let request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "replicated.txt",
+            head,
+        );
         let ctx = write_back_context(&request, true).await;
         assert!(ctx.emit_events, "policy.emit_events reaches the creation event");
         assert_eq!(ctx.principal_id, ON_DEMAND_MIGRATION_PRINCIPAL_ID);
@@ -786,7 +1115,12 @@ mod tests {
         head.user_metadata
             .insert(forged_replica_key.to_string(), ReplicationStatusType::Replica.as_str().to_string());
 
-        let mut write_request = request(&bucket, "unadmitted.txt", head);
+        let mut write_request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "unadmitted.txt",
+            head,
+        );
         write_request.tags = Some(HashMap::from([("replicate".to_string(), "no".to_string())]));
         OnDemandMigrationWriteBack::new()
             .put_object(&write_request, body_stream(&body))
@@ -874,11 +1208,17 @@ mod tests {
         head.sse = None;
         head.etag = None;
         assert_eq!(expected_md5_hex(&head), None);
+
+        // An Azure ETag can be any string the service chooses; even one that
+        // happens to look like an MD5 must not be checked against the bytes.
+        let mut head = source_head(b"abc");
+        head.etag_is_opaque = true;
+        assert_eq!(expected_md5_hex(&head), None, "opaque provider ETag");
     }
 
     #[test]
     fn provenance_and_tags_are_stable() {
-        let mut request = request("b", "k", source_head(b"x"));
+        let mut request = request("b", Uuid::nil(), "k", source_head(b"x"));
         let metadata = provenance_metadata(&request);
         assert_eq!(metadata.len(), 10, "five keys under two prefixes");
         assert_provenance(&metadata, &request.head);
@@ -899,7 +1239,7 @@ mod tests {
     #[tokio::test]
     async fn write_back_context_applies_the_etag_and_event_policy() {
         let body = b"context".to_vec();
-        let mut request = request("no-such-bucket", "k", source_head(&body));
+        let mut request = request("no-such-bucket", Uuid::nil(), "k", source_head(&body));
         let ctx = write_back_context(&request, true).await;
         assert_eq!(ctx.expected_md5_hex, Some(md5_hex(&body)));
         assert_eq!(ctx.preserve_etag, Some(md5_hex(&body)));

rustfs/src/app/object/put.rs

@@ -949,7 +949,12 @@ pub(super) enum PutObjectOrigin<'a> {
     /// request and no credential: managed-SSE authorization treats the write
     /// as internal, and the creation event, when requested, names
     /// `principal_id` instead of an access key.
-    Internal { principal_id: &'static str, emit_events: bool },
+    Internal {
+        principal_id: &'static str,
+        emit_events: bool,
+        preserve_delete_marker: bool,
+        expected_bucket_incarnation_id: Option<Uuid>,
+    },
 }
 
 impl PutObjectOrigin<'_> {
@@ -963,7 +968,13 @@ impl PutObjectOrigin<'_> {
     fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> {
         match self {
             Self::S3 { req, .. } => apply_bucket_generation_guard(req, bucket, opts),
-            Self::Internal { .. } => Ok(()),
+            Self::Internal {
+                expected_bucket_incarnation_id,
+                ..
+            } => {
+                opts.expected_bucket_incarnation_id = *expected_bucket_incarnation_id;
+                Ok(())
+            }
         }
     }
 
@@ -1603,6 +1614,12 @@ impl DefaultObjectUsecase {
         if let Some(etag) = preserve_etag {
             opts.preserve_etag = Some(etag);
         }
+        if let PutObjectOrigin::Internal {
+            preserve_delete_marker, ..
+        } = &origin
+        {
+            opts.preserve_delete_marker = *preserve_delete_marker;
+        }
         if let Some(quota_check) = quota_check.as_ref() {
             apply_quota_admission(&mut opts, quota_check)?;
         }
@@ -1769,6 +1786,7 @@ impl DefaultObjectUsecase {
             PutObjectOrigin::Internal {
                 principal_id,
                 emit_events,
+                ..
             } => {
                 let principal_id = *principal_id;
                 let request_context = request_context::RequestContext::fallback();