From d5426f59ec9a1c639854f8cae4ee8b888c31535f Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:14:22 +0800
Subject: [PATCH 01/20] fix(ci): isolate functional evidence and preserve every
 result (#7201)

* fix(ci): preserve reported functional suite failures

* fix(ci): isolate functional evidence and preserve every result

* fix(ci): exclude sensitive scratch files from suite artifacts
---
 .github/workflows/rustfs-heal-test.yml        |  89 ++-
 .github/workflows/rustfs-kms-test.yml         | 128 ++--
 .github/workflows/rustfs-performance-test.yml |  68 ++-
 .github/workflows/rustfs-replication-test.yml | 128 ++--
 .github/workflows/rustfs-s3-compat-test.yml   | 131 ++--
 .github/workflows/rustfs-security-test.yml    |  11 +-
 .github/workflows/rustfs-storage-test.yml     | 131 ++--
 .github/workflows/rustfs-upgrade-test.yml     | 154 ++---
 scripts/functional_case_report.py             |  77 +++
 scripts/test/oidc_keycloak_live.sh            |   4 +-
 scripts/test_security_workflow.py             | 565 ++++++++++++++++--
 11 files changed, 968 insertions(+), 518 deletions(-)
 create mode 100644 scripts/functional_case_report.py

diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index efbe2a09c..4e6b4e190 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -59,6 +59,21 @@ jobs:
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -114,7 +129,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -124,7 +139,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -134,13 +149,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -149,8 +161,9 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
           import re
           import sys
 
@@ -162,6 +175,7 @@ jobs:
 
           steps = {}
           order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
           version = None
           version_node = None
           verdict = None
@@ -175,14 +189,15 @@ jobs:
                           n, desc, status = m.group(1), m.group(2), m.group(3)
                           if n not in steps:
                               order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
                           continue
                       m = ver_re.match(line)
                       if m:
                           version, version_node = m.group(1), m.group(2)
                           continue
                       m = result_re.match(line)
-                      if m:
+                      if m and verdict != 'FAIL':
                           verdict, verdict_detail = m.group(1), m.group(2)
           except FileNotFoundError:
               pass
@@ -202,30 +217,43 @@ jobs:
                   out.write(f'| {n} | {desc} | {status} |\n')
               if not order:
                   out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
           PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${STEPS_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -257,11 +285,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -289,14 +316,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -312,14 +341,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-heal-test*.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index 5c3a2b6b1..c9eb02d00 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -52,6 +52,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -108,8 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -139,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -154,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -258,11 +236,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -290,14 +267,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -313,14 +292,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-kms.log
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index ff3e2978d..148e5ccf3 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -76,8 +76,6 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
@@ -89,6 +87,22 @@ jobs:
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -120,7 +134,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -130,7 +144,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -140,7 +154,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -153,17 +167,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -183,7 +195,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -191,7 +202,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -199,6 +210,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -208,8 +221,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          } > "${REPORT_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -228,11 +241,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -260,14 +272,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -283,20 +297,26 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-perf-test*.log
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+          if-no-files-found: error
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 839d1de7f..ae8f9dc50 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -65,6 +65,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -113,8 +132,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -137,10 +154,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -162,80 +176,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -267,11 +245,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -299,14 +276,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -322,14 +301,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index 875db70bb..99828537b 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -40,6 +40,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -87,8 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -105,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -130,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -238,11 +213,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -270,14 +244,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -293,14 +269,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-s3-compat.log
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index ee37e7d5d..02fe7d662 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -92,7 +92,7 @@ jobs:
           set -euo pipefail
           umask 077
           SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${SECURITY_ARTIFACTS_DIR}"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
       # auto-testing is private: clone it with the dedicated PF token (not
@@ -148,7 +148,7 @@ jobs:
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
-          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
           RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
         run: |
           set -euo pipefail
@@ -172,7 +172,7 @@ jobs:
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
-          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
 
       - name: Generate report
         id: report
@@ -305,7 +305,10 @@ jobs:
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
+          path: |
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
           if-no-files-found: error
           retention-days: 3
 
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index 1e99dce4e..e16fc0058 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -49,6 +49,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -96,8 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -120,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -145,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -253,11 +228,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -285,14 +259,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -308,14 +284,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 612765874..5a91793a0 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -82,6 +82,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,7 +161,6 @@ jobs:
       - name: Run upgrade compatibility suite
         id: test
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
@@ -200,10 +218,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -224,103 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
-          rows = []
-          index = {}
-          topo_rows = []
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${MATRIX_TABLE}"
+              echo ""
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -352,11 +311,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -384,14 +342,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -407,14 +367,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)
diff --git a/scripts/functional_case_report.py b/scripts/functional_case_report.py
new file mode 100644
index 000000000..7036eaad5
--- /dev/null
+++ b/scripts/functional_case_report.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Preserve every functional case execution and its suite context in reports."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import re
+
+
+def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
+    ansi = re.compile(r"\x1b\[[0-9;]*m")
+    start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
+    done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
+    context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
+    topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
+    rows = []
+    pending = {}
+    topo_rows = []
+    context = "context not recorded"
+    complete = True
+    try:
+        with log_file.open(encoding="utf-8", errors="replace") as log:
+            for raw in log:
+                line = ansi.sub("", raw).strip()
+                if match := context_re.match(line):
+                    context = match[1]
+                    pending.clear()
+                elif match := topo_re.match(line):
+                    topo_rows.append(match.groups())
+                elif match := start_re.match(line):
+                    case_id, name = match.groups()
+                    pending[case_id] = len(rows)
+                    rows.append([case_id, f"{name} ({context})", "RUNNING"])
+                elif match := done_re.match(line):
+                    status, case_id = match.groups()
+                    index = pending.pop(case_id, None)
+                    if index is None:
+                        complete = False
+                        rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
+                    else:
+                        rows[index][2] = status
+    except FileNotFoundError:
+        pass
+
+    counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
+    with case_file.open("w", encoding="utf-8") as out:
+        out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
+        for status, count in counts.items():
+            out.write(f"- {status}: {count}\n")
+        out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
+        for row in rows:
+            out.write("| " + " | ".join(value.replace("|", "&#124;") for value in row) + " |\n")
+        if not rows:
+            out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
+
+    valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
+    if matrix_file is not None:
+        with matrix_file.open("w", encoding="utf-8") as out:
+            out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
+            out.write("| --- | --- | --- | --- | --- |\n")
+            for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                result = "PASS" if nfail == "0" else "FAIL"
+                out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
+            if not topo_rows:
+                out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
+        valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
+    return valid
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("log_file", type=Path)
+    parser.add_argument("case_file", type=Path)
+    parser.add_argument("matrix_file", type=Path, nargs="?")
+    args = parser.parse_args()
+    raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)
diff --git a/scripts/test/oidc_keycloak_live.sh b/scripts/test/oidc_keycloak_live.sh
index c464f6dcb..73b8eb6da 100755
--- a/scripts/test/oidc_keycloak_live.sh
+++ b/scripts/test/oidc_keycloak_live.sh
@@ -187,7 +187,7 @@ values = {}
 for element in root.iter():
     values[element.tag.rsplit("}", 1)[-1]] = element.text or ""
 for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"):
-    assert values.get(field), values
+    assert values.get(field), f"missing required STS field: {field}"
 print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken")))
 PY
 )
@@ -218,7 +218,6 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \
   --data-urlencode DurationSeconds=900 \
   --data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")"
 [[ "${TAMPERED_STATUS}" == 403 ]] || {
-  cat "${WORK_DIR}/sts-tampered.xml" >&2
   echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2
   exit 1
 }
@@ -235,7 +234,6 @@ BAD_STATUS="$(curl --noproxy '*' -sS \
   --data-urlencode DurationSeconds=900 \
   --data-urlencode "WebIdentityToken=${BAD_TOKEN}")"
 [[ "${BAD_STATUS}" == 403 ]] || {
-  cat "${WORK_DIR}/sts-bad.xml" >&2
   echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2
   exit 1
 }
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index ea2d75487..4b4b063dd 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -3,14 +3,18 @@
 
 from __future__ import annotations
 
+import glob
+import json
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import unittest
 from pathlib import Path
 
 from check_test_wiring import yaml_block
+from functional_case_report import generate_report
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -38,7 +42,46 @@ def shell_body(lines: list[str]) -> str:
     return "\n".join(shell_lines)
 
 
-class SecurityWorkflowTests(unittest.TestCase):
+class WorkflowSteps:
+    def uploaded_files(self) -> set[Path]:
+        upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
+        start = upload.index("          path: |") + 1
+        paths = []
+        for line in upload[start:]:
+            if not line.startswith("            "):
+                break
+            paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
+        return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
+
+    def render(self, value: str) -> str:
+        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+        result = {}
+        for line in yaml_block(lines, "env", indent) or []:
+            if line.strip() and not line.lstrip().startswith("#"):
+                key, value = line.strip().split(": ", 1)
+                result[key] = self.render(value.strip("'\""))
+        return result
+
+    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+        lines = self.steps[name]
+        result = subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+        )
+        for line in lines:
+            if line.startswith("        id: "):
+                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+        if Path(self.env["GITHUB_ENV"]).exists():
+            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+                key, value = line.split("=", 1)
+                self.env[key] = value
+                self.context[f"env.{key}"] = value
+        return result
+
+
+class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -78,6 +121,7 @@ class SecurityWorkflowTests(unittest.TestCase):
             '#!/usr/bin/env bash\nset -euo pipefail\n'
             'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
             'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
+            'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
             'case "$FAKE_REPORT" in\n'
             f'  present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
             '  empty) : > "$REPORT_FILE" ;;\n'
@@ -86,33 +130,6 @@ class SecurityWorkflowTests(unittest.TestCase):
             'exit "$FAKE_EXIT"\n'
         )
 
-    def render(self, value: str) -> str:
-        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
-    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
-        result = {}
-        for line in yaml_block(lines, "env", indent) or []:
-            if line.strip() and not line.lstrip().startswith("#"):
-                key, value = line.strip().split(": ", 1)
-                result[key] = self.render(value.strip("'\""))
-        return result
-
-    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
-        lines = self.steps[name]
-        result = subprocess.run(
-            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
-            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
-        )
-        for line in lines:
-            if line.startswith("        id: "):
-                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
-        if Path(self.env["GITHUB_ENV"]).exists():
-            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
-                key, value = line.split("=", 1)
-                self.env[key] = value
-                self.context[f"env.{key}"] = value
-        return result
-
     def test_workflow_wiring(self) -> None:
         names = list(self.steps)
         self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -128,7 +145,7 @@ class SecurityWorkflowTests(unittest.TestCase):
         for name in ("Upload functional report to dashboard", "Upload report and logs"):
             self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
         artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
-        self.assertIn("          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
+        self.assertIn("          path: |", artifact_settings)
         self.assertIn("          if-no-files-found: error", artifact_settings)
 
     def test_suite_report_and_result_matrix(self) -> None:
@@ -147,7 +164,7 @@ class SecurityWorkflowTests(unittest.TestCase):
                 if outcome != "skipped" or mode == "present":
                     suite = self.run_step("Run security suite")
                     self.assertEqual(suite.returncode, exit_code, suite.stderr)
-                    logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
+                    logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
                     self.assertEqual(len(logs), 1)
                     self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
                 self.context["steps.test.outcome"] = outcome
@@ -169,37 +186,91 @@ class SecurityWorkflowTests(unittest.TestCase):
                 summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
                 self.assertEqual(summary, contents)
                 self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
+                expected = {self.artifacts / "report.md"}
+                if outcome != "skipped" or mode == "present":
+                    expected.add(self.artifacts / "suite.log")
+                    self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n")
+                if mode in ("present", "empty"):
+                    expected.add(self.artifacts / "suite-report.md")
+                (self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
+                scratch = Path(str(self.artifacts) + "-scratch")
+                (scratch / "case.out").write_text("FAKE-SECRET-CANARY")
+                self.assertEqual(self.uploaded_files(), expected)
+
+
+    def test_oidc_negative_responses_never_print_issued_credentials(self):
+        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+        for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
+            start = source.index('[[ "${' + variable + '}" == 403 ]]')
+            end = source.index("\n", source.index("grep -q '<Code>AccessDenied</Code>'", start))
+            guard = source[start:end]
+            credential_xml = "<Credentials><AccessKeyId>FAKE-ACCESS-CANARY</AccessKeyId><SecretAccessKey>FAKE-SECRET-CANARY</SecretAccessKey><SessionToken>FAKE-SESSION-CANARY</SessionToken></Credentials>"
+            for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
+                                           ("403", "<Code>AccessDenied</Code>", 0)):
+                with self.subTest(variable=variable, status=status, expected=expected):
+                    (self.directory / filename).write_text(body)
+                    result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
+                                            env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
+                                            capture_output=True, text=True)
+                    self.assertEqual(result.returncode, expected, result.stderr)
+                    for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+                        self.assertNotIn(canary, result.stdout + result.stderr)
+                    if status == "200":
+                        self.assertIn("HTTP 403, got 200", result.stderr)
+
+    def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
+        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+        start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
+        end = source.index("\n)\n", start) + 3
+        extract = source[start:end]
+        values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
+                  "SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
+                  "SubjectFromWebIdentityToken": "alice"}
+        for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
+            with self.subTest(missing=missing):
+                xml = "<Credentials>" + "".join(f"<{key}>{value}</{key}>" for key, value in values.items() if key != missing) + "</Credentials>"
+                (self.directory / "sts-good.xml").write_text(xml)
+                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
+                                        env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
+                self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
+                for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+                    self.assertNotIn(canary, result.stdout + result.stderr)
+                if missing:
+                    self.assertIn(f"missing required STS field: {missing}", result.stderr)
 
     def test_existing_evidence_directory_is_rejected(self) -> None:
-        self.artifacts.mkdir()
-        stale = self.artifacts / "suite-report.md"
-        stale.write_text("OLD RUN REPORT")
-        self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
-        self.assertEqual(stale.read_text(), "OLD RUN REPORT")
-        self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
-        (self.artifacts / "report.md").write_text("OLD RUN REPORT")
-        self.context.update({
-            "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
-        })
-        fake_bin = self.directory / "bin"
-        fake_bin.mkdir()
-        gh = fake_bin / "gh"
-        gh.write_text(
-            '#!/usr/bin/env bash\nset -euo pipefail\n'
-            'if [ "$1 $2" = "issue create" ]; then\n'
-            '  while [ "$#" -gt 0 ]; do\n'
-            '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
-            '    shift\n'
-            '  done\n'
-            'fi\n'
-        )
-        gh.chmod(0o755)
-        body = self.directory / "issue-body.md"
-        self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
-        result = self.run_step("File failure issue in rustfs/backlog")
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertNotIn("OLD RUN REPORT", body.read_text())
-        self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
+        for suffix in ("", "-scratch"):
+            self.setUp()
+            existing = Path(str(self.artifacts) + suffix)
+            existing.mkdir()
+            stale = existing / "suite-report.md"
+            stale.write_text("OLD RUN REPORT")
+            self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
+            self.assertEqual(stale.read_text(), "OLD RUN REPORT")
+            self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+            (self.artifacts / "report.md").write_text("OLD RUN REPORT")
+            self.context.update({
+                "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
+            })
+            fake_bin = self.directory / "bin"
+            fake_bin.mkdir()
+            gh = fake_bin / "gh"
+            gh.write_text(
+                '#!/usr/bin/env bash\nset -euo pipefail\n'
+                'if [ "$1 $2" = "issue create" ]; then\n'
+                '  while [ "$#" -gt 0 ]; do\n'
+                '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                '    shift\n'
+                '  done\n'
+                'fi\n'
+            )
+            gh.chmod(0o755)
+            body = self.directory / "issue-body.md"
+            self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
+            result = self.run_step("File failure issue in rustfs/backlog")
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertNotIn("OLD RUN REPORT", body.read_text())
+            self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
 
 class FunctionalWorkflowTests(unittest.TestCase):
@@ -225,7 +296,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 if suite in self.DIRECT_TESTS:
                     test = steps[self.DIRECT_TESTS[suite]]
                     self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
-                    self.assertIn("        if: always()", steps["Generate report"])
+                    self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
                 self.assertIn(condition, (
@@ -286,5 +357,379 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
 
 
+class FunctionalCaseReportTests(unittest.TestCase):
+    def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            log = root / "suite.log"
+            if text is not None:
+                log.write_text(text)
+            valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
+            return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
+
+    def test_repeated_case_executions_preserve_failure_and_context(self):
+        # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
+        log = subprocess.check_output(["bash", "-c", r'''
+log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
+log '== topology: single-single  kms-backend: local =='
+printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
+log '== topology: single-multi  kms-backend: vault-kv2 =='
+printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
+printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
+'''], text=True)
+        valid, cases, _ = self.report(log)
+        self.assertFalse(valid)
+        self.assertEqual(cases.count("| KMS-101 |"), 3)
+        self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
+        self.assertIn("roundtrip (topology: single-single  kms-backend: local) | FAIL |", cases)
+        self.assertIn("roundtrip (topology: single-multi  kms-backend: vault-kv2) | PASS |", cases)
+        self.assertNotIn("\\n", cases)
+
+    def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
+        for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
+            with self.subTest(log=text):
+                valid, cases, _ = self.report(text)
+                self.assertFalse(valid)
+                self.assertIn("## Case Summary", cases)
+        valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
+        self.assertTrue(valid)
+        self.assertIn("suite: bucket replication", cases)
+        self.assertIn("- UNSUPPORTED: 1\n", cases)
+
+    def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
+        case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
+        for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
+                                 ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
+            with self.subTest(matrix=suffix):
+                valid, _, matrix = self.report(case + suffix, matrix=True)
+                self.assertEqual(valid, expected)
+                self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
+                self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
+
+    def test_s3_case_identifiers_include_digits(self):
+        valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
+        self.assertTrue(valid)
+        self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
+
+
+class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
+    SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
+
+    def prepare(self, suite: str) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+        self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
+        self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
+        self.context.update({
+            "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
+            "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
+            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+            "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
+        })
+        self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
+        self.env = {
+            **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
+            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
+            "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
+        }
+        for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
+            self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
+        (self.directory / "scripts").mkdir()
+        (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
+        fake_bin = self.directory / "bin"
+        fake_bin.mkdir()
+        (fake_bin / "python3").symlink_to(sys.executable)
+        for command, body in (
+            ("ssh", 'printf "fixture-version\\n"\n'),
+            ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
+                   '  while [ "$#" -gt 0 ]; do\n'
+                   '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                   '    shift\n'
+                   '  done\n'
+                   'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
+        ):
+            script = fake_bin / command
+            script.write_text("#!/bin/sh\n" + body)
+            script.chmod(0o755)
+        self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
+
+    def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
+        for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")):
+            with self.subTest(suite=suite, collision=suffix or "artifact"):
+                self.prepare(suite)
+                self.assertNotIn("/tmp/rustfs-", self.source)
+                names = list(self.steps)
+                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+                if suite in FunctionalWorkflowTests.DIRECT_TESTS:
+                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+                for name, lines in self.steps.items():
+                    if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
+                    if any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("          path: |", lines)
+                        self.assertIn("          if-no-files-found: error", lines)
+                existing = Path(str(self.artifacts) + suffix)
+                existing.mkdir()
+                for filename in ("report.md", "suite.log"):
+                    (existing / filename).write_text("OLD RUN EVIDENCE")
+                self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
+                initialized = self.run_step("Initialize functional evidence")
+                self.assertNotEqual(initialized.returncode, 0)
+                self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+                issue = self.run_step("File failure issue in rustfs/backlog")
+                self.assertEqual(issue.returncode, 0, issue.stderr)
+                body = Path(self.env["CAPTURE_BODY"]).read_text()
+                self.assertNotIn("OLD RUN EVIDENCE", body)
+                self.assertIn("no report or log file was produced", body)
+                self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
+
+    def test_reports_use_only_current_complete_suite_evidence(self):
+        for suite in self.SUITES[:-1]:
+            good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
+            partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
+            if suite == "s3-compat":
+                good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
+            if suite == "upgrade":
+                good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
+            if suite == "heal":
+                good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
+                partial = "[HEAL-STEP] 1 fixture PASS\n"
+            for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
+                                 ("success", None), ("skipped", None), ("cancelled", good)):
+                with self.subTest(suite=suite, outcome=outcome, log=log):
+                    self.prepare(suite)
+                    stale = self.directory / "old-suite.log"
+                    stale.write_text("OLD RUN EVIDENCE\n" + good)
+                    self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
+                    self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                    self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
+                    self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
+                    if log is not None:
+                        Path(self.env["LOG_FILE"]).write_text(log)
+                    self.context["steps.test.outcome"] = outcome
+                    report = self.run_step("Generate report")
+                    success = outcome == "success" and log == good
+                    self.assertEqual(report.returncode == 0, success, report.stderr)
+                    contents = Path(self.env["REPORT_FILE"]).read_text()
+                    self.assertNotIn("OLD RUN EVIDENCE", contents)
+                    self.assertEqual("| PASS |" in contents, success)
+                    for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
+                                  f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
+                        self.assertIn(value, contents)
+                    self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
+                    evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
+                    if log in (good, partial):
+                        self.assertIn("| PASS |", evidence)
+                    self.assertNotIn("OLD RUN EVIDENCE", evidence)
+
+    def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                if suite == "heal":
+                    self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
+                scripts = self.directory / "auto-testing"
+                scripts.mkdir()
+                filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
+                script = scripts / filename
+                script.write_text(
+                    '#!/bin/bash\nset -euo pipefail\nlog=""\n'
+                    'while [ "$#" -gt 0 ]; do\n'
+                    '  if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
+                    '  shift\n'
+                    'done\n'
+                    '[ "$log" = "$LOG_FILE" ] || exit 31\n'
+                    'printf "CURRENT SUITE LOG\\n" > "$log"\n'
+                    'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
+                    'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
+                    'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
+                    '  mkdir -p "$RUSTFS_RESULT_DIR"\n'
+                    '  printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
+                    'fi\n'
+                )
+                script.chmod(0o755)
+                name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
+                    "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
+                )
+                result = self.run_step(name)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
+                self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1)
+                self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
+                if suite == "performance":
+                    self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
+
+    def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
+        extra = {
+            "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
+            "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"],
+            "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
+                            "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
+        }
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
+                for path in expected:
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_text("PARTIAL FAILURE DIAGNOSTIC")
+                for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
+                    directory.mkdir(exist_ok=True)
+                    (directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
+                self.assertEqual(self.uploaded_files(), expected)
+                self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
+
+    def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
+        self.prepare("kms")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        scripts = self.directory / "auto-testing"
+        scripts.mkdir()
+        # Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
+        # The Docker boundary returns synthetic credentials; setup fails before vault_stop.
+        (scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
+set -Eeuo pipefail
+TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
+trap 'rm -rf "${TEST_TMP}"' EXIT
+VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
+VAULT_CONTAINER="rustfs-vault"
+heal_run() { "$@"; }
+while [ "$#" -gt 0 ]; do
+  if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
+  shift
+done
+mkdir -p "${VAULT_DATA_DIR}"
+tmp_init="${TEST_TMP}/vault-init.json"
+tmp_err="${TEST_TMP}/vault-init.stderr"
+heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
+  vault operator init -key-shares=1 -key-threshold=1 -format=json \
+  > "${tmp_init}" 2>"${tmp_err}"
+cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
+printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
+exit 42
+''')
+        docker = self.directory / "bin/docker"
+        docker.write_text("""#!/bin/sh
+[ "$1" = exec ] || exit 99
+printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
+""")
+        docker.chmod(0o755)
+        result = self.run_step("Run KMS suite")
+        self.assertEqual(result.returncode, 42, result.stderr)
+        vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
+        self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
+        self.assertNotIn(self.artifacts, vault_init.parents)
+        self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
+        self.assertNotEqual(self.run_step("Generate report").returncode, 0)
+        self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
+        self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
+        for path in self.uploaded_files():
+            self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
+            self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
+
+    def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
+        self.prepare("heal")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        script = self.directory / "auto-testing/rustfs_heal_test.sh"
+        script.parent.mkdir()
+        # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
+        script.write_text(r'''#!/bin/bash
+set -euo pipefail
+SELECTED_STEPS=()
+PREFLIGHT=0
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
+    --log-file) LOG_FILE="$2"; shift ;;
+    --preflight) PREFLIGHT=1 ;;
+  esac
+  shift
+done
+if [ "$PREFLIGHT" -eq 1 ]; then
+  printf '\n' >> "$INVOKED_STEPS"
+  exit 0
+fi
+printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
+emit_step_result() {
+  local n="$1" desc="$2" status="$3"
+  printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
+}
+{
+  for step in "${SELECTED_STEPS[@]}"; do
+    emit_step_result "$step" "fixture step $step" PASS
+  done
+  want_all=1
+  for s in 1 2 3 4 5 6 7; do
+    [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
+  done
+  if [ "${want_all}" -eq 1 ]; then
+    printf '[HEAL-RESULT] PASS all steps passed\n'
+  fi
+} >> "$LOG_FILE"
+''')
+        script.chmod(0o755)
+        self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
+        for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
+            result = self.run_step(name)
+            self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
+        log = Path(self.env["LOG_FILE"]).read_text()
+        self.assertNotIn("[HEAL-RESULT]", log)
+        self.assertEqual(log.count("[HEAL-STEP]"), 7)
+        report = self.run_step("Generate report")
+        self.assertEqual(report.returncode, 0, report.stderr)
+        failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
+                       for step in range(1, 8)]
+        failed_logs += [
+            log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
+            log + "[HEAL-STEP] 3 later step failure FAIL\n",
+            log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
+            log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
+        ]
+        for failed_log in failed_logs:
+            with self.subTest(log=failed_log):
+                Path(self.env["LOG_FILE"]).write_text(failed_log)
+                report = self.run_step("Generate report")
+                self.assertNotEqual(report.returncode, 0, report.stderr)
+                contents = Path(self.env["REPORT_FILE"]).read_text()
+                self.assertIn("Test Step Outcome: failure", contents)
+                self.assertNotIn("| PASS |", contents)
+                if "original failure" in failed_log:
+                    self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
+                if "later step failure" in failed_log:
+                    self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
+
+    def test_performance_results_version_and_report_are_bound_to_the_run(self):
+        self.prepare("performance")
+        initialized = self.run_step("Initialize functional evidence")
+        self.assertEqual(initialized.returncode, 0, initialized.stderr)
+        self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
+        self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
+        version = self.run_step("Collect RustFS version info")
+        self.assertEqual(version.returncode, 0, version.stderr)
+        self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
+        old_summary = self.directory / "old-results/summary.md"
+        old_summary.parent.mkdir()
+        old_summary.write_text("OLD RUN EVIDENCE")
+        upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
+        self.assertNotEqual(self.run_step(upload).returncode, 0)
+        self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
+        results = Path(self.env["RUSTFS_RESULT_DIR"])
+        results.mkdir()
+        (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
+        report = self.run_step(upload)
+        self.assertEqual(report.returncode, 0, report.stderr)
+        contents = Path(self.env["REPORT_FILE"]).read_text()
+        for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
+                      "CURRENT PERFORMANCE RESULTS", "fixture-version"):
+            self.assertIn(value, contents)
+        self.assertNotIn("OLD RUN EVIDENCE", contents)
+
+
 if __name__ == "__main__":
     unittest.main()

From 1210428b6db37f7c412bb5282a1133dde0cc82ee Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:14:37 +0800
Subject: [PATCH 02/20] fix(ci): publish immutable nightly package candidates
 (#7202)

---
 .config/make/tests.mak                  |   1 +
 .github/actions/quick-checks/action.yml |   1 +
 .github/workflows/nightly-gnu.yml       |  59 ++++++-
 scripts/test_nightly_candidate.py       | 208 ++++++++++++++++++++++++
 4 files changed, 261 insertions(+), 8 deletions(-)
 create mode 100644 scripts/test_nightly_candidate.py

diff --git a/.config/make/tests.mak b/.config/make/tests.mak
index d1297e9ad..726ca04cf 100644
--- a/.config/make/tests.mak
+++ b/.config/make/tests.mak
@@ -41,6 +41,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml
index e6cc59775..18c6cb64d 100644
--- a/.github/actions/quick-checks/action.yml
+++ b/.github/actions/quick-checks/action.yml
@@ -100,6 +100,7 @@ runs:
         python3 ./scripts/check_test_wiring.py --self-test
         python3 ./scripts/check_scheduled_validation_freshness.py --self-test
         python3 ./scripts/test_security_workflow.py
+        python3 ./scripts/test_nightly_candidate.py
         python3 ./scripts/check_test_wiring.py
 
     - name: Check no planning docs committed
diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml
index 0dedbf465..1864a9715 100644
--- a/.github/workflows/nightly-gnu.yml
+++ b/.github/workflows/nightly-gnu.yml
@@ -166,8 +166,9 @@ jobs:
       # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
       # Skipped when the R2 secrets are not configured (artifact-only mode).
       - name: Upload DEB to Cloudflare R2
-        if: env.R2_ACCESS_KEY_ID != ''
+        id: publish
         env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
             exit 0
           fi
 
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y -qq awscli
-          fi
-
           export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          SOURCE_SHA="$(git rev-parse HEAD)"
+          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+            echo "Checkout SHA does not match the nightly build run" >&2
+            exit 1
+          fi
+          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
+          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
+          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
+
+          # Old AWS CLI models lack conditional PutObject support. Never fall
+          # back to an overwriting upload for a candidate.
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
+            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          "${AWS_CLI}" --version
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
+            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
+          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
+            echo "Published candidate checksum does not match the built package" >&2
+            exit 1
+          fi
+
           R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
 
           echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           # Stable "latest" alias so tests can fetch the newest nightly
           # without knowing today's date.
           echo "📤 Uploading latest alias"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
             --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           echo "✅ R2 upload complete"
 
+          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg source_sha "${SOURCE_SHA}" \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
+            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            > "${CANDIDATE_FILE}"
+          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload nightly candidate manifest
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error
+
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py
new file mode 100644
index 000000000..3e5a50401
--- /dev/null
+++ b/scripts/test_nightly_candidate.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Exercise the nightly publication step without AWS, network or package builds."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+from check_test_wiring import yaml_block
+
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
+
+
+class NightlyCandidateTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.package = self.root / "rustfs-nightly-2026-09-06.deb"
+        self.package.write_bytes(b"built package bytes\x00\xff")
+        for command in (["git", "init", "-q"], ["git", "add", self.package.name],
+                        ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
+            subprocess.run(command, cwd=self.root, check=True, capture_output=True)
+        self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
+        self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
+        self.output = self.root / "github-output"
+        self.store = self.root / "store"
+        self.shims = self.root / "fake-tools.sh"
+        self.shims.write_text(r'''aws() {
+  printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
+  if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
+  if [[ "$*" == *--generate-cli-skeleton* ]]; then
+    if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
+      printf '{}\n'
+    else
+      printf '{"IfNoneMatch":""}\n'
+    fi
+    return
+  fi
+  if [[ "$1 $2" == 's3api put-object' ]]; then
+    [[ "$FAKE_MODE" != upload-failure ]] || return 42
+    shift 2
+    local key="" body="" condition=""
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        --key) key="$2";;
+        --body) body="$2";;
+        --if-none-match) condition="$2";;
+      esac
+      shift 2
+    done
+    [[ -z "$condition" || "$condition" == '*' ]] || return 43
+    if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
+    mkdir -p "$(dirname "$FAKE_STORE/$key")"
+    cp "$body" "$FAKE_STORE/$key"
+  elif [[ "$1 $2" == 's3 cp' ]]; then
+    [[ "$FAKE_MODE" != alias-failure ]] || return 45
+    local destination="${4#s3://test-bucket/}"
+    [[ "$destination" != */ ]] || destination+="$(basename "$3")"
+    mkdir -p "$(dirname "$FAKE_STORE/$destination")"
+    cp "$3" "$FAKE_STORE/$destination"
+  else
+    return 46
+  fi
+}
+curl() {
+  local url="${!#}"
+  printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
+  [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
+  [[ "$FAKE_MODE" != missing-public-url ]] || return 22
+  if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
+  cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
+  [[ "$FAKE_MODE" != incomplete-download ]] || return 47
+}
+sudo() {
+  [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
+  [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
+}
+python3() {
+  [[ "$1 $2" == '-m venv' ]] || return 50
+  mkdir -p "$3/bin"
+  cat > "$3/bin/python" <<'SH'
+#!/usr/bin/env bash
+[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
+[[ "$FAKE_MODE" != install-failure ]] || exit 52
+: > "$FAKE_INSTALLED"
+SH
+  printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
+  chmod +x "$3/bin/python" "$3/bin/aws"
+}
+''')
+        self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
+                        R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
+                        RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
+                        FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
+        source = WORKFLOW.read_text()
+        job = yaml_block(source.splitlines(), "build", 2)
+        starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
+        self.steps = {
+            job[start].split(": ", 1)[1]: job[start:end]
+            for start, end in zip(starts, starts[1:] + [len(job)])
+        }
+        self.publish = self.steps["Upload DEB to Cloudflare R2"]
+        start = self.publish.index("        run: |") + 1
+        self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith("          "))
+
+    def run_publish(self, **overrides):
+        self.output.unlink(missing_ok=True)
+        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
+                              cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
+
+    def manifest(self):
+        output = self.output.read_text().strip()
+        self.assertTrue(output.startswith("candidate_file="), output)
+        return json.loads(Path(output.split("=", 1)[1]).read_text())
+
+    def test_success_binds_actual_package_checkout_and_attempt(self):
+        result = self.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.manifest()
+        self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
+                                   "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
+        for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
+            self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
+        self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
+
+    def test_missing_credentials_remain_artifact_only(self):
+        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+            with self.subTest(missing=key):
+                result = self.run_publish(**{key: ""})
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertFalse(self.output.exists())
+                self.assertFalse((self.root / "aws.log").exists())
+                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_publication_failures_never_emit_a_candidate(self):
+        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
+            with self.subTest(mode=mode):
+                result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
+                self.assertNotEqual(result.returncode, 0, result.stdout)
+                self.assertFalse(self.output.exists())
+                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
+        result = self.run_publish(FAKE_MODE="old-cli")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue((self.root / "installed").exists())
+        self.assertEqual(self.manifest()["package_sha256"], self.digest)
+        self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+    def test_failed_cli_bootstrap_cannot_publish(self):
+        for mode in ("bootstrap-failure", "install-failure", "broken-install"):
+            with self.subTest(mode=mode):
+                (self.root / "installed").unlink(missing_ok=True)
+                result = self.run_publish(FAKE_MODE=mode)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse(self.output.exists())
+                self.assertFalse(self.store.exists())
+                self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+    def test_checkout_sha_mismatch_fails_before_upload(self):
+        result = self.run_publish(GITHUB_SHA="f" * 40)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("Checkout SHA", result.stderr)
+        self.assertFalse(self.output.exists())
+        self.assertFalse((self.root / "aws.log").exists())
+
+    def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
+        urls = []
+        for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
+            result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            urls.append(self.manifest()["package_url"])
+        self.assertEqual(len(set(urls)), 3)
+        self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
+
+    def test_duplicate_key_is_not_overwritten_or_recertified(self):
+        result = self.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
+        stored = self.store / key
+        stored.write_bytes(b"preexisting conflicting object")
+        (self.root / "nightly-candidate-12345-1.json").unlink()
+        result = self.run_publish()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
+        self.assertFalse(self.output.exists())
+        self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_manifest_upload_requires_publication_output(self):
+        upload = self.steps["Upload nightly candidate manifest"]
+        self.assertIn("        id: publish", self.publish)
+        self.assertIn("          DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
+        self.assertIn("        if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
+        self.assertIn("          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
+        self.assertIn("          path: ${{ steps.publish.outputs.candidate_file }}", upload)
+        self.assertIn("          if-no-files-found: error", upload)
+        self.assertNotIn("        continue-on-error: true", self.publish)
+        self.assertNotIn("          overwrite: true", upload)
+
+
+if __name__ == "__main__":
+    unittest.main()

From a6b5da64f27c2c0c82fe8ce5592a0e9e80138655 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:15:03 +0800
Subject: [PATCH 03/20] fix(ci): serialize performance on shared functional VMs
 (#7204)

---
 .github/workflows/rustfs-functional-chain.yml |  17 +--
 .github/workflows/rustfs-performance-test.yml |   9 +-
 .github/workflows/rustfs-replication-test.yml |  50 ++++++--
 scripts/test_security_workflow.py             | 112 +++++++++++++++++-
 4 files changed, 157 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml
index 6ce828c1a..10b9c67ad 100644
--- a/.github/workflows/rustfs-functional-chain.yml
+++ b/.github/workflows/rustfs-functional-chain.yml
@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index 148e5ccf3..d698b27e0 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index ae8f9dc50..0faaf7809 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -330,13 +329,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index 4b4b063dd..b96b3e3f5 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Exercise functional workflow failures and security evidence without remote VMs."""
+"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
 
 from __future__ import annotations
 
@@ -272,6 +272,110 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
             self.assertNotIn("OLD RUN REPORT", body.read_text())
             self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
+    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
+        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+            with self.subTest(suite=suite):
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+                # Workflow-level concurrency covers every job, including cleanup,
+                # regardless of trigger or the runner hosting the job.
+                self.assertEqual([
+                    line.strip() for line in yaml_block(source, "concurrency", 0)
+                    if line.strip() and not line.lstrip().startswith("#")
+                ], [
+                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
+                ])
+                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
+                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
+                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
+                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
+
+    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
+            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
+                self.setUp()
+                fake_bin = self.directory / "bin"
+                fake_bin.mkdir()
+                commands = {
+                    "gh": '''#!/usr/bin/env bash
+set -euo pipefail
+if [ "$1" = api ]; then
+  printf '%s\\n' "$*" >> "$DISPATCHES"
+  attempt=$(wc -l < "$DISPATCHES")
+  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
+elif [ "$1 $2" = 'issue create' ]; then
+  printf 'issue\\n' >> "$EXECUTED"
+  while [ "$#" -gt 0 ]; do
+    if [ "$1" = --body-file ]; then
+      cat "$2" > "$CAPTURE_BODY"
+      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
+    fi
+    shift
+  done
+  exit "$ISSUE_EXIT"
+else
+  exit 99
+fi
+''',
+                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
+                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
+                }
+                for name, contents in commands.items():
+                    command = fake_bin / name
+                    command.write_text(contents)
+                    command.chmod(0o755)
+                dispatches = self.directory / "dispatches"
+                executed = self.directory / "executed"
+                body = self.directory / "issue-body.md"
+                body_path = self.directory / "issue-body-path"
+                self.env.update(
+                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
+                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
+                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
+                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
+                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+                )
+                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
+                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
+                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
+                started = self.run_step("Dispatch first suite (upgrade)")
+                self.assertEqual(started.returncode, 0, started.stderr)
+                self.assertEqual(dispatches.read_text().splitlines(), [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
+                ])
+                dispatches.unlink()
+
+                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
+                job = yaml_block(replication.splitlines(), "replication-test", 2)
+                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
+                self.steps = named_steps(job)
+                handoff = "Continue functional chain (next: Performance)"
+                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
+                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
+                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
+                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
+                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
+                failed = self.run_step("Run replication suite")
+                self.assertEqual(failed.returncode, 17, failed.stderr)
+                cleaned = self.run_step("Cleanup environment (after)")
+                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
+                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
+                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
+                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
+                forwarded = self.run_step(handoff)
+                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
+                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
+                self.assertEqual(calls, [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+                ] * (min(failed_attempts + 1, 3) if token else 0))
+                if failed_attempts == 3:
+                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
+                    self.assertIn("rustfs-chain-performance", body.read_text())
+                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
+                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+
 
 class FunctionalWorkflowTests(unittest.TestCase):
     JOBS = {
@@ -305,7 +409,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                     "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
                 ))
                 if suite != "performance":
-                    handoff = steps["Chain complete"] if suite == "replication" else next(
+                    handoff = next(
                         value for name, value in steps.items() if name.startswith("Continue functional chain")
                     )
                     self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
@@ -348,13 +452,13 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertIn("partial suite diagnostics", failed.stdout)
                 cleanup = execute("Cleanup environment (after)")
                 self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
-                handoff_name = "Chain complete" if suite == "replication" else next(
+                handoff_name = next(
                     name for name in steps if name.startswith("Continue functional chain")
                 )
                 handoff = execute(handoff_name)
                 self.assertEqual(handoff.returncode, 0, handoff.stderr)
                 markers = (root / "executed").read_text().splitlines()
-                self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
+                self.assertEqual(markers, ["cleanup", "dispatch"])
 
 
 class FunctionalCaseReportTests(unittest.TestCase):

From 8f763fb1a214a4050649758e4d45257b23aa4d5a Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:50:06 +0800
Subject: [PATCH 04/20] fix(ci): run existing script contracts in quick checks
 (#7203)

* fix(ci): share quick checks and lint workflows

* fix(ci): install actionlint from its verified release

* fix(ci): reject dependencies on required quick checks

* fix(ci): run existing script contracts in quick checks
---
 .github/actions/quick-checks/action.yml |  4 ++++
 scripts/check_test_wiring.py            | 14 +++++++++-----
 scripts/test_python_bin.sh              | 22 ++++++++--------------
 3 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml
index 18c6cb64d..c9a957f10 100644
--- a/.github/actions/quick-checks/action.yml
+++ b/.github/actions/quick-checks/action.yml
@@ -94,6 +94,10 @@ runs:
       shell: bash
       run: ./scripts/check_embedded_secrets.sh
 
+    - name: Run script contract tests
+      shell: bash
+      run: make script-tests
+
     - name: Check test wiring
       shell: bash
       run: |
diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py
index 63de463d3..ef3f15107 100755
--- a/scripts/check_test_wiring.py
+++ b/scripts/check_test_wiring.py
@@ -567,7 +567,7 @@ def check_quick_checks(root: Path) -> list[str]:
         errors.append(f"{relative}: missing composite action")
         return errors
     steps = yaml_block(runs, "steps", 2) or []
-    for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
+    for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
         step = workflow_step_block(steps, command, key="run", indent=4)
         if step is None:
             errors.append(f"{relative}: missing direct execution of {command}")
@@ -906,6 +906,7 @@ class SelfTests(unittest.TestCase):
                 "    - name: Lint workflows\n      shell: bash\n      run: shellcheck --version && actionlint\n"
                 "    - name: Error format ratchet\n      shell: bash\n"
                 "      run: ./scripts/check_error_other_format_ratchet.sh\n"
+                "    - name: Script tests\n      shell: bash\n      run: make script-tests\n"
             )
             sources = {
                 ".github/workflows/ci.yml": caller.replace(
@@ -964,6 +965,8 @@ class SelfTests(unittest.TestCase):
                 "only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"),
                 "missing shellcheck preflight": action.replace("shellcheck --version && ", ""),
                 "missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"),
+                "missing script tests": action.replace("run: make script-tests", "run: echo skipped"),
+                "swallowed script failure": action.replace("run: make script-tests", "run: make script-tests || true"),
                 "swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"),
                 "swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"),
                 "conditional lint": action.replace("run: shellcheck", "if: false\n      run: shellcheck"),
@@ -973,7 +976,7 @@ class SelfTests(unittest.TestCase):
                     "name: Lint workflows", "name: |\n        run: shellcheck --version && actionlint"
                 ).replace("\n      run: shellcheck --version && actionlint\n", "\n      run: shellcheck --version && actionlint\n        || true\n"),
             }
-            for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
+            for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
                 for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
                     mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n      run: {command}")
                 for separator in ("", "\n", "        # continued command\n"):
@@ -994,9 +997,10 @@ class SelfTests(unittest.TestCase):
             root = Path(tmp)
             (root / "scripts").mkdir()
             commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh")
-            for failing in commands:
+            (root / "Makefile").write_text(".PHONY: script-tests\nscript-tests:\n\texit 17\n")
+            for failing in (*commands, "make script-tests"):
                 with self.subTest(command=failing):
-                    run = "shellcheck --version && actionlint" if failing != commands[-1] else failing
+                    run = "shellcheck --version && actionlint" if failing in ("shellcheck", "actionlint") else failing
                     step = workflow_step_block(steps, run, key="run", indent=4)
                     self.assertIsNotNone(step)
                     run_index = next(index for index, line in enumerate(step[1]) if line.startswith("      run:"))
@@ -1011,7 +1015,7 @@ class SelfTests(unittest.TestCase):
                         cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"),
                         capture_output=True, text=True,
                     )
-                    self.assertEqual(result.returncode, 17, result.stderr)
+                    self.assertEqual(result.returncode, 2 if failing == "make script-tests" else 17, result.stderr)
 
     def test_validate_includes_quick_checks(self) -> None:
         error = "Quick Checks wiring regression"
diff --git a/scripts/test_python_bin.sh b/scripts/test_python_bin.sh
index 4c5e149fb..927412fff 100755
--- a/scripts/test_python_bin.sh
+++ b/scripts/test_python_bin.sh
@@ -62,20 +62,14 @@ exit 1
 STUB
 chmod +x "$TMP_ROOT/bin/python3"
 
-SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin"
-if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then
-    # uv is reachable even from the sandbox PATH, so the resolver would
-    # legitimately fall back to it instead of failing. Skip this case.
-    echo "ℹ️  uv is on the sandbox PATH; skipping the no-interpreter case"
-else
-    if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \
-        >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
-        fail "resolver succeeded with no usable interpreter on PATH"
-    fi
-    grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
-        || fail "missing-interpreter failure did not name the requirement"
-    grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
-        || fail "missing-interpreter failure did not point at the override"
+ln -s "$(command -v bash)" "$TMP_ROOT/bin/bash"
+if PATH="$TMP_ROOT/bin" RUSTFS_PYTHON="" "$RESOLVER" -c 'pass' \
+    >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
+    fail "resolver succeeded with no usable interpreter on PATH"
 fi
+grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
+    || fail "missing-interpreter failure did not name the requirement"
+grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
+    || fail "missing-interpreter failure did not point at the override"
 
 echo "✅ scripts/python_bin.sh resolver checks passed"

From 8fb335cf19bd2a83509410aa67d00c614829d00e Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:08:22 +0800
Subject: [PATCH 05/20] test(e2e): pin upgrade compatibility to rc.5 and cover
 bucket configuration (#7217)

test(e2e): prove bucket config survives rc.5 upgrade and rollback

Add two upgrade-compatibility scenarios pinned to the on-demand-migration
series' on-disk surfaces: BucketMetadata's 44 -> 46 msgpack keys, the
fail-closed bucket-config reads of rustfs#7172, the encryption-gated PUT
path of rustfs#7183, and the default-on migration module of rustfs#7089.

The upgrade case writes versioning, SSE-S3 default encryption, a validated
replication target plus rule, lifecycle, tags, quota, a public access block,
a bucket policy and an object lock configuration with the pinned previous
release, then asserts each one reads back unchanged on the current build,
that list-remote-targets still reports the target, that writes to the
encrypted and plain buckets keep their encryption posture, that every
pre-upgrade object including a multipart one is byte-identical, and that an
unconfigured bucket reports no migration and still answers NoSuchKey.

The rollback case is the reverse: the current build writes the 46-key blob
and the previous release must decode it by skipping the two unknown keys.
---
 .github/workflows/e2e-upgrade.yml             |  25 +-
 .../src/upgrade_compatibility_test.rs         | 691 +++++++++++++++++-
 docs/testing/ci-gates.md                      |   4 +-
 3 files changed, 711 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml
index ed420cd06..8cc1d53a5 100644
--- a/.github/workflows/e2e-upgrade.yml
+++ b/.github/workflows/e2e-upgrade.yml
@@ -19,7 +19,9 @@ on:
     paths:
       - ".github/workflows/e2e-upgrade.yml"
       - "crates/e2e_test/src/common.rs"
+      - "crates/e2e_test/src/fake_s3_target/**"
       - "crates/e2e_test/src/lib.rs"
+      - "crates/e2e_test/src/replication_extension_test.rs"
       - "crates/e2e_test/src/upgrade_compatibility_test.rs"
       - "crates/ecstore/**"
       - "crates/filemeta/**"
@@ -44,9 +46,9 @@ concurrency:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
-  UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
-  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
-  UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+  UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
+  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
+  UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
 
 jobs:
   upgrade:
@@ -55,14 +57,27 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Direct upgrade from rc.2
+          # The two `_from_rc2_` tests keep their names: they assert
+          # release-independent object contracts and pass unchanged against the
+          # newer pinned source, so renaming them would only churn history and
+          # the CI required-check names. UPGRADE_SOURCE_VERSION above is the
+          # single source of truth for which release they actually run against.
+          - name: Direct upgrade from the previous release
             cache_key: e2e-direct-upgrade
             test: direct_upgrade_from_rc2_preserves_object_contracts
             artifact: direct-upgrade
-          - name: Mixed-version rolling upgrade from rc.2
+          - name: Mixed-version rolling upgrade from the previous release
             cache_key: e2e-mixed-version-upgrade
             test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
             artifact: mixed-version-upgrade
+          - name: Bucket configuration survives the upgrade
+            cache_key: e2e-bucket-config-upgrade
+            test: direct_upgrade_from_previous_release_preserves_bucket_configuration
+            artifact: bucket-config-upgrade
+          - name: Rollback reads current bucket metadata
+            cache_key: e2e-bucket-config-rollback
+            test: rollback_to_previous_release_reads_current_bucket_metadata
+            artifact: bucket-config-rollback
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs
index 558c10d4c..9f1871552 100644
--- a/crates/e2e_test/src/upgrade_compatibility_test.rs
+++ b/crates/e2e_test/src/upgrade_compatibility_test.rs
@@ -12,19 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use crate::common::{
+    RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
+};
+use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::replication_extension_test::{
+    LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
+    BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
+    ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
+    ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
+    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
 };
+use http::{Method, StatusCode};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use tokio::task::JoinSet;
 use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
@@ -40,6 +51,32 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
 // comfortably covers that window plus CI scheduling jitter.
 const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
+// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
+const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
+const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
+const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
+const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
+const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
+const ROLLBACK_BUCKET: &str = "rollback-config-data";
+const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
+const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
+const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
+const LIFECYCLE_PREFIX: &str = "logs/";
+const LIFECYCLE_DAYS: i32 = 30;
+const BUCKET_TAG_KEY: &str = "owner";
+const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
+const OBJECT_LOCK_DAYS: i32 = 1;
+// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
+// authoritative; the quota test uses the same 30s budget.
+const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
+// Quota admission fails closed while a freshly started server has neither
+// authoritative usage nor a persisted degraded baseline for the bucket
+// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
+// window. It is a restart property, not an upgrade property — the same window
+// opens on the very first start — so the write assertions ride it out instead
+// of treating it as an upgrade failure.
+const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
+
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
         .map(PathBuf::from)
@@ -429,3 +466,653 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
 
     Ok(())
 }
+
+/// Child-process environment shared by both bucket-configuration scenarios.
+///
+/// The replication target is an in-process fake bound to `127.0.0.1`, which
+/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
+/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
+/// server's outbound health check.
+fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
+    let mut env = vec![
+        (SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
+        ("NO_PROXY", "127.0.0.1,localhost"),
+        ("HTTP_PROXY", ""),
+        ("HTTPS_PROXY", ""),
+        // Shorten the scanner cycle so the bucket's usage becomes authoritative
+        // in seconds; both `set-bucket-quota` and quota admission block on it.
+        ("RUSTFS_SCANNER_CYCLE", "1"),
+        ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
+    ];
+    env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    env.extend(replication_fast_env());
+    env
+}
+
+/// Restart `env` in place on the same data directory using an explicit binary.
+///
+/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
+/// the workspace build, which is the upgrade direction only. The rollback
+/// scenario needs the reverse: stop the current build and bring the pinned
+/// previous release up on the metadata that build just wrote.
+async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
+    env.stop_server();
+    env.start_rustfs_server_from_binary(binary, vec![], server_env).await
+}
+
+async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
+    let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
+    loop {
+        let (status, response) =
+            admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
+        if status.is_success() {
+            return Ok(());
+        }
+        if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
+            return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
+/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
+///
+/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
+/// never ends, is a genuine regression and surfaces as an error.
+async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
+    let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
+    loop {
+        let result = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(body))
+            .send()
+            .await;
+        let error = match result {
+            Ok(_) => return Ok(()),
+            Err(error) => error,
+        };
+        let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
+        if !retryable || Instant::now() >= deadline {
+            return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
+    let path = format!("/rustfs/admin/v3/quota/{bucket}");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
+    }
+    let quota: serde_json::Value = serde_json::from_str(&response)?;
+    Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
+}
+
+/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
+///
+/// Returns an error for any non-200, because rustfs#7172 made this endpoint
+/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
+/// An upgrade that misreads a blob written by the previous release therefore
+/// shows up here as an error, and a silently dropped target shows up as an
+/// empty list — the caller must distinguish the two.
+async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
+    let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    if status != StatusCode::OK {
+        return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
+    }
+    Ok(serde_json::from_str(&response)?)
+}
+
+/// Assert that `bucket` still carries exactly the replication target `arn`.
+async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
+    let targets = list_remote_targets(env, bucket).await?;
+    assert_eq!(
+        targets.len(),
+        1,
+        "{context}: list-remote-targets must still report the single configured target, got {targets:?}"
+    );
+    assert_eq!(
+        targets[0].get("arn").and_then(serde_json::Value::as_str),
+        Some(arn),
+        "{context}: the target ARN changed across the restart: {targets:?}"
+    );
+    Ok(())
+}
+
+/// Configure a replication target on `bucket` pointing at the in-process fake,
+/// then attach an enabled replication rule for it. Returns the target ARN.
+async fn configure_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target: &FakeS3Target,
+    target_bucket: &str,
+) -> Result<String, BoxError> {
+    let arn = set_replication_target_with_options(
+        env,
+        bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(env, bucket, &arn).await?;
+    Ok(arn)
+}
+
+async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
+    let configuration = ServerSideEncryptionConfiguration::builder()
+        .rules(
+            ServerSideEncryptionRule::builder()
+                .apply_server_side_encryption_by_default(
+                    ServerSideEncryptionByDefault::builder()
+                        .sse_algorithm(ServerSideEncryption::Aes256)
+                        .build()?,
+                )
+                .build(),
+        )
+        .build()?;
+    client
+        .put_bucket_encryption()
+        .bucket(bucket)
+        .server_side_encryption_configuration(configuration)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let response = client.get_bucket_encryption().bucket(bucket).send().await?;
+    let rules = response
+        .server_side_encryption_configuration()
+        .ok_or("GetBucketEncryption omitted the configuration")?
+        .rules();
+    assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
+    assert_eq!(
+        rules[0]
+            .apply_server_side_encryption_by_default()
+            .map(ServerSideEncryptionByDefault::sse_algorithm),
+        Some(&ServerSideEncryption::Aes256),
+        "{context}: the default encryption algorithm changed"
+    );
+    Ok(())
+}
+
+async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
+    let tagging = Tagging::builder()
+        .tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
+        .build()?;
+    client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
+    Ok(())
+}
+
+async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
+    let tag_set = tags.tag_set();
+    assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
+    assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
+    assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
+    Ok(())
+}
+
+async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
+    let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
+    assert_eq!(
+        versioning.status(),
+        Some(&BucketVersioningStatus::Enabled),
+        "{context}: versioning is no longer Enabled on {bucket}"
+    );
+    Ok(())
+}
+
+fn bucket_policy_document(bucket: &str) -> serde_json::Value {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Sid": "UpgradePublicRead",
+            "Effect": "Allow",
+            "Principal": { "AWS": ["*"] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
+        }]
+    })
+}
+
+/// `GET .../on-demand-migration/{bucket}/status`.
+///
+/// The migration module defaults on from rustfs#7089, so a bucket that never
+/// configured a source must still answer `configured: false` rather than
+/// engaging the migration path.
+async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
+    let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
+    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
+    );
+    let body: serde_json::Value = serde_json::from_str(&response)?;
+    assert_eq!(
+        body.get("configured"),
+        Some(&serde_json::Value::Bool(false)),
+        "a bucket upgraded from the previous release must not look migration-configured: {body}"
+    );
+    Ok(())
+}
+
+/// A GET for a key that was never written must be a plain `NoSuchKey`.
+///
+/// With the migration module on by default this is the cheap proof that an
+/// unconfigured bucket never consults a source: any migration engagement would
+/// surface as a different status or error code here.
+async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
+    let error = client
+        .get_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("a key that was never written must not be readable");
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "a missing key must stay a 404 on a bucket with no migration configuration"
+    );
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("NoSuchKey"),
+        "a missing key must stay NoSuchKey on a bucket with no migration configuration"
+    );
+    Ok(())
+}
+
+/// Bucket configuration written by the pinned previous release must survive an
+/// upgrade to the current build unchanged, and must keep working.
+///
+/// This pins the three on-disk surfaces the on-demand-migration series moved:
+///
+/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
+///   every configuration read below decodes a 44-key blob on 46-key code.
+/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
+///   public-access-block / quota blob "present but unreadable" instead of
+///   silently defaulting, and made `list-remote-targets` fail closed on it. A
+///   replication target configured by the old release must therefore still be
+///   *listed*, not dropped and not an error.
+/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
+///   encryption configuration cannot be read, so a misparsed SSE config would
+///   turn every PUT to that bucket into a 500.
+///
+/// Not covered on purpose: on-demand-migration configuration itself, which the
+/// previous release has no public API for — the reverse direction is asserted
+/// instead (an upgraded bucket reports `configured: false`).
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    // In-process: the fake target outlives both server processes, so the
+    // replication target stays reachable across the upgrade.
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
+        .await?;
+    let old_client = env.create_s3_client();
+
+    env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
+    env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
+    env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
+    old_client
+        .create_bucket()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    // Plain bucket: policy, tags, lifecycle, quota.
+    let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
+    old_client
+        .put_bucket_policy()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .policy(policy.to_string())
+        .send()
+        .await?;
+    put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
+    old_client
+        .put_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .lifecycle_configuration(
+            BucketLifecycleConfiguration::builder()
+                .rules(
+                    LifecycleRule::builder()
+                        .id(LIFECYCLE_RULE_ID)
+                        .status(ExpirationStatus::Enabled)
+                        .filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
+                        .expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
+                        .build()?,
+                )
+                .build()?,
+        )
+        .send()
+        .await?;
+    set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
+
+    // Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
+    // public access block, both of which rustfs#7172 now fails closed on.
+    put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
+    old_client
+        .put_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .public_access_block_configuration(
+            PublicAccessBlockConfiguration::builder()
+                .block_public_acls(true)
+                .ignore_public_acls(true)
+                .block_public_policy(true)
+                .restrict_public_buckets(true)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // Replicated bucket: versioning, a validated remote target, a rule.
+    enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
+    let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
+
+    // Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
+    old_client
+        .put_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .object_lock_configuration(
+            ObjectLockConfiguration::builder()
+                .object_lock_enabled(ObjectLockEnabled::Enabled)
+                .rule(
+                    ObjectLockRule::builder()
+                        .default_retention(
+                            DefaultRetention::builder()
+                                .mode(ObjectLockRetentionMode::Governance)
+                                .days(OBJECT_LOCK_DAYS)
+                                .build(),
+                        )
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let plain_key = "plain/written-by-previous";
+    let plain_bytes = b"plain object written by the previous RustFS release";
+    put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
+
+    let encrypted_key = "encrypted/written-by-previous";
+    let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
+    old_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(encrypted_key)
+        .body(ByteStream::from_static(encrypted_bytes))
+        .send()
+        .await?;
+    assert_eq!(
+        read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
+            .await?
+            .0,
+        Some(ServerSideEncryption::Aes256),
+        "the previous release must apply the bucket default encryption it just accepted"
+    );
+
+    // The multipart object lives in the default-encrypted bucket so the
+    // upgraded build has to reassemble parts *and* re-derive the object key.
+    let multipart_key = "encrypted/multipart-written-by-previous";
+    let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
+
+    let versioned_key = "versioned/written-by-previous";
+    let versioned_bytes = b"versioned object written by the previous RustFS release";
+    let versioned_id = old_client
+        .put_object()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .key(versioned_key)
+        .body(ByteStream::from_static(versioned_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    env.restart_server_preserving_data(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    // Every configuration must read back unchanged on the upgraded build.
+    let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
+    let upgraded_policy: serde_json::Value =
+        serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
+    assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
+    assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
+
+    let lifecycle = new_client
+        .get_bucket_lifecycle_configuration()
+        .bucket(CONFIG_PLAIN_BUCKET)
+        .send()
+        .await?;
+    let rules = lifecycle.rules();
+    assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
+    assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
+    assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
+    assert_eq!(
+        rules[0].expiration().and_then(LifecycleExpiration::days),
+        Some(LIFECYCLE_DAYS),
+        "the lifecycle expiration changed across the upgrade"
+    );
+
+    assert_eq!(
+        get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
+        Some(BUCKET_QUOTA_BYTES),
+        "the bucket quota changed across the upgrade"
+    );
+
+    assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
+    let public_access_block = new_client
+        .get_public_access_block()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .send()
+        .await?;
+    let public_access_block = public_access_block
+        .public_access_block_configuration()
+        .ok_or("GetPublicAccessBlock omitted the configuration")?;
+    assert_eq!(public_access_block.block_public_acls(), Some(true));
+    assert_eq!(public_access_block.ignore_public_acls(), Some(true));
+    assert_eq!(public_access_block.block_public_policy(), Some(true));
+    assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
+
+    assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
+    // rustfs#7172: neither an empty list nor an error is acceptable here.
+    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
+    let replication = new_client
+        .get_bucket_replication()
+        .bucket(CONFIG_REPLICATED_BUCKET)
+        .send()
+        .await?;
+    let replication_rules = replication
+        .replication_configuration()
+        .ok_or("GetBucketReplication omitted the configuration")?
+        .rules();
+    assert_eq!(
+        replication_rules.len(),
+        1,
+        "the replication rule count changed across the upgrade: {replication_rules:?}"
+    );
+    assert_eq!(
+        replication_rules[0].destination().map(|destination| destination.bucket()),
+        Some(target_arn.as_str()),
+        "the replication rule no longer points at the configured target"
+    );
+
+    let object_lock = new_client
+        .get_object_lock_configuration()
+        .bucket(CONFIG_LOCKED_BUCKET)
+        .send()
+        .await?;
+    let object_lock = object_lock
+        .object_lock_configuration()
+        .ok_or("GetObjectLockConfiguration omitted the configuration")?;
+    assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
+    let retention = object_lock
+        .rule()
+        .and_then(ObjectLockRule::default_retention)
+        .ok_or("the object lock configuration lost its default retention")?;
+    assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
+    assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
+
+    // rustfs#7183: a PUT into the default-encrypted bucket must still succeed
+    // and still come back encrypted.
+    let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
+    let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
+    new_client
+        .put_object()
+        .bucket(CONFIG_ENCRYPTED_BUCKET)
+        .key(post_upgrade_encrypted_key)
+        .body(ByteStream::from_static(post_upgrade_encrypted_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "a PUT after the upgrade lost the bucket default encryption"
+    );
+    assert_eq!(body, post_upgrade_encrypted_bytes);
+
+    let post_upgrade_plain_key = "plain/written-after-upgrade";
+    let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
+    put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
+    let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
+    assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
+    assert_eq!(body, post_upgrade_plain_bytes);
+
+    // Every object written by the previous release reads back byte-identical.
+    assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, encrypted_bytes);
+    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
+    assert_eq!(
+        read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
+            .await?
+            .1,
+        versioned_bytes
+    );
+
+    // rustfs#7089: the migration module is on by default, but a bucket that
+    // never configured a source behaves exactly as before.
+    assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
+    assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
+
+    replication_target.shutdown().await;
+    Ok(())
+}
+
+/// Rolling back to the pinned previous release must still read the bucket
+/// metadata the current build wrote.
+///
+/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
+/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
+/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
+/// has to skip those two unknown keys instead of failing the whole blob. If it
+/// did not, every configuration read below would come back empty or error and
+/// the rollback would silently discard the bucket's configuration.
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+
+    let replication_target = FakeS3Target::start().await?;
+    replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let server_env = bucket_config_server_env();
+    env.start_rustfs_server_with_env(vec![], &server_env).await?;
+    let new_client = env.create_s3_client();
+
+    env.create_test_bucket(ROLLBACK_BUCKET).await?;
+    enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
+    put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
+    put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
+    let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
+
+    let single_key = "rollback/single";
+    let single_bytes = b"single-part object written by the current RustFS build";
+    let single_version = new_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(single_key)
+        .body(ByteStream::from_static(single_bytes))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("versioned PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "rollback/multipart";
+    let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
+    let multipart_bytes = multipart_parts.concat();
+    write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
+
+    restart_from_binary(&mut env, &previous_binary, &server_env).await?;
+    let old_client = env.create_s3_client();
+
+    assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
+
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, single_bytes);
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
+    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
+    assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
+
+    // A PUT on the rolled-back release must still honour the encryption
+    // configuration it decoded out of the current build's metadata blob.
+    let post_rollback_key = "rollback/written-after-rollback";
+    let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
+    old_client
+        .put_object()
+        .bucket(ROLLBACK_BUCKET)
+        .key(post_rollback_key)
+        .body(ByteStream::from_static(post_rollback_bytes))
+        .send()
+        .await?;
+    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
+    assert_eq!(
+        encryption,
+        Some(ServerSideEncryption::Aes256),
+        "the rolled-back release lost the bucket default encryption"
+    );
+    assert_eq!(body, post_rollback_bytes);
+
+    replication_target.shutdown().await;
+    Ok(())
+}
diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md
index 99d589d64..81418c911 100644
--- a/docs/testing/ci-gates.md
+++ b/docs/testing/ci-gates.md
@@ -49,7 +49,7 @@ Promotion rule: never promote a report-only lane to required from one green run.
 | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
 | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
 | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
-| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from rc.2` | `e2e-upgrade.yml` `direct-upgrade` | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release |
+| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
 | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
 | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
 | PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` |
@@ -84,7 +84,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
-| `e2e-upgrade.yml` (weekly) | `direct-upgrade` | upgrade gate; server logs | no | see the PR row |
+| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
 | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch |

From 35aefbb2a52e2a48ec95973beaa39a79f407de14 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:25:44 +0800
Subject: [PATCH 06/20] fix(s3): keep ListObjects v1 local during migration
 (#7220)

* fix(s3): keep ListObjects v1 local during migration

* test(s3): use the v1 listing request DTO directly
---
 rustfs/src/app/bucket_list_through.rs | 92 +++++++++++++++++++++++++++
 rustfs/src/app/bucket_usecase.rs      | 24 +++++--
 2 files changed, 112 insertions(+), 4 deletions(-)

diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index a41b994f9..ce09fc990 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -451,6 +451,7 @@ mod tests {
     use crate::app::storage_api::test::StoragePutObjReader;
     use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
     use crate::app::storage_api::test::contract::object::ObjectIO as _;
+    use s3s::dto::ListObjectsInput;
     use std::time::Duration;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
@@ -795,6 +796,97 @@ mod tests {
         (result, requests)
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn list_objects_v1_stays_local_with_xml_safe_key_markers() {
+        run_large_stack_test("list-through-v1-local-markers", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    let (endpoint, server, stop) =
+                        list_source(std::iter::repeat(source_xml(None, false, Some("a-source")))).await;
+                    let (_state_guard, source_input) =
+                        source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await;
+                    let store = shared_gating_ecstore().await;
+                    store
+                        .put_object(
+                            &source_input.bucket,
+                            "a&local",
+                            &mut StoragePutObjReader::from_vec(vec![1]),
+                            &StorageObjectOptions::default(),
+                        )
+                        .await
+                        .expect("seed a second local object");
+
+                    for delimiter in [None, Some("/".to_string())] {
+                        let mut input = ListObjectsInput {
+                            bucket: source_input.bucket.clone(),
+                            max_keys: Some(1),
+                            delimiter,
+                            ..Default::default()
+                        };
+                        for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
+                            let request_marker = input.marker.clone().unwrap_or_default();
+                            let response = tokio::time::timeout(
+                                Duration::from_secs(10),
+                                DefaultBucketUsecase::from_global().execute_list_objects(S3Request {
+                                    input: input.clone(),
+                                    method: http::Method::GET,
+                                    uri: http::Uri::from_static("/"),
+                                    headers: HeaderMap::new(),
+                                    extensions: http::Extensions::new(),
+                                    credentials: None,
+                                    region: None,
+                                    service: None,
+                                    trailing_headers: None,
+                                }),
+                            )
+                            .await
+                            .expect("v1 pagination must finish")
+                            .expect("list-through must not change v1 listing");
+                            assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
+                            let output = response.output;
+                            let contents = output.contents.as_ref().expect("local page contents");
+                            assert_eq!(contents.len(), 1);
+                            assert_eq!(contents[0].key.as_deref(), Some(expected_key));
+                            assert_eq!(output.marker.as_deref(), Some(request_marker.as_str()));
+                            assert_eq!(output.is_truncated, Some(index == 0));
+                            assert_eq!(output.next_marker.as_deref(), (index == 0).then_some(expected_key));
+
+                            let mut xml = Vec::new();
+                            s3s::xml::Serialize::serialize(&output, &mut s3s::xml::Serializer::new(&mut xml))
+                                .expect("serialize the real v1 response");
+                            assert!(!xml.contains(&0), "XML 1.0 forbids NUL in NextMarker");
+                            let mut reader = quick_xml::Reader::from_reader(xml.as_slice());
+                            loop {
+                                if reader.read_event().expect("v1 response must be well-formed XML")
+                                    == quick_xml::events::Event::Eof
+                                {
+                                    break;
+                                }
+                            }
+                            input.marker = output.next_marker;
+                        }
+                    }
+                    stop.cancel();
+                    let requests = server.await.expect("source server must not panic");
+                    assert!(requests.is_empty(), "ListObjects v1 must issue no remote LIST requests: {requests:?}");
+                },
+            )
+            .await;
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs
index e1951cd69..fe3b11e45 100644
--- a/rustfs/src/app/bucket_usecase.rs
+++ b/rustfs/src/app/bucket_usecase.rs
@@ -2724,7 +2724,14 @@ impl DefaultBucketUsecase {
 
     #[instrument(level = "trace", skip(self, req))]
     pub async fn execute_list_objects_v2(&self, req: S3Request<ListObjectsV2Input>) -> S3Result<S3Response<ListObjectsV2Output>> {
-        // warn!("list_objects_v2 req {:?}", &req.input);
+        self.execute_list_objects_v2_inner(req, true).await
+    }
+
+    async fn execute_list_objects_v2_inner(
+        &self,
+        req: S3Request<ListObjectsV2Input>,
+        allow_list_through: bool,
+    ) -> S3Result<S3Response<ListObjectsV2Output>> {
         let ListObjectsV2Input {
             bucket,
             continuation_token,
@@ -2750,8 +2757,15 @@ impl DefaultBucketUsecase {
         // The on-demand migration envelope is decoded whether or not this
         // bucket still merges: a token handed out under `list_through` must keep
         // paginating after the policy is turned off (rustfs/backlog#2164).
-        let merged_token = list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?;
-        let (object_infos, degraded) = match list_through::list_through_state(&bucket, &req.headers) {
+        let (merged_token, source_state) = if allow_list_through {
+            (
+                list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?,
+                list_through::list_through_state(&bucket, &req.headers),
+            )
+        } else {
+            (None, None)
+        };
+        let (object_infos, degraded) = match source_state {
             Some(state) => {
                 let outcome = list_through::merged_list_objects_v2(
                     &store,
@@ -2938,7 +2952,9 @@ impl DefaultBucketUsecase {
     #[instrument(level = "debug", skip(self, req))]
     pub async fn execute_list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
         let request_marker = req.input.marker.clone();
-        let v2_resp = self.execute_list_objects_v2(req.map_input(Into::into)).await?;
+        // V1 markers are object keys, so they cannot carry the opaque merged
+        // pagination state used by V2 list-through.
+        let v2_resp = self.execute_list_objects_v2_inner(req.map_input(Into::into), false).await?;
 
         Ok(v2_resp.map_output(|v2| build_list_objects_output(v2, request_marker)))
     }

From a9f01dbbdb56f9f59123c222dd642e2a004eee55 Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Sun, 6 Sep 2026 01:26:08 +0800
Subject: [PATCH 07/20] fix(ecstore): restore odm source contract tests (#7215)

* fix(ecstore): restore odm source contract tests

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(ci): initialize replication evidence in chain test

Run the replication workflow's evidence initialization before the chain handoff self-test executes the suite step. This keeps the test model aligned with the workflow-provided LOG_FILE and TMPDIR values.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(odm): distinguish missing GCS buckets from object misses (#7221)

---------

Co-authored-by: zhi22915 <qiuzgang@gmail.com>
Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
---
 .../src/bucket/on_demand_migration/gcs.rs     | 62 ++++++++++++++++++-
 .../on_demand_migration/list_through.rs       | 16 +++--
 .../bucket/on_demand_migration/native_http.rs | 33 +++++++---
 .../on_demand_migration/source_client.rs      |  5 +-
 scripts/test_security_workflow.py             |  2 +
 5 files changed, 103 insertions(+), 15 deletions(-)

diff --git a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs
index 707e1cc82..874ddb0e3 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs
@@ -124,6 +124,18 @@ impl GcsNativeSourceBackend {
         Ok(request)
     }
 
+    async fn send_object(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
+        match self.http.send_object(request, NO_ERROR_CODE_HEADER).await {
+            Err(SourceError::NotFound) => {
+                // An XML object URL also returns 404 when its bucket is gone.
+                // Reuse the read-only listing probe before caching a key miss.
+                self.probe().await?;
+                Err(SourceError::NotFound)
+            }
+            result => result,
+        }
+    }
+
     /// Shared mapping for the XML API's HEAD and GET responses.
     fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
         if header(headers, "x-goog-encryption-key-sha256").is_some() {
@@ -164,7 +176,7 @@ impl GcsNativeSourceBackend {
 impl SourceBackend for GcsNativeSourceBackend {
     async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
         let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.send_object(request).await?;
         Self::head_from_response(response.headers())
     }
 
@@ -177,7 +189,7 @@ impl SourceBackend for GcsNativeSourceBackend {
             );
         }
         let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.send_object(request).await?;
         let head = Self::head_from_response(response.headers())?;
         let content_range = header(response.headers(), "content-range").map(str::to_string);
         Ok(SourceGet {
@@ -488,6 +500,7 @@ mod tests {
             // request; the probe is the next one on the wire.
             ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
             ScriptedResponse::new(404, Vec::new(), String::new()),
+            ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
             ScriptedResponse::new(403, Vec::new(), String::new()),
         ])
         .await;
@@ -503,4 +516,49 @@ mod tests {
         )
         .await;
     }
+
+    #[tokio::test]
+    async fn listing_404_is_not_an_object_not_found() {
+        let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(404, Vec::new(), String::new())]).await;
+        let err = backend(&endpoint)
+            .list(&SourceListRequest {
+                max_keys: 1,
+                ..Default::default()
+            })
+            .await
+            .expect_err("a failed bucket listing is not a per-object miss");
+        assert_eq!(err.class_label(), "other", "{err:?}");
+    }
+
+    #[tokio::test]
+    async fn object_404_requires_a_readable_source_bucket() {
+        for method in [Method::HEAD, Method::GET] {
+            for (probe_status, expected_class) in [
+                (200, "not_found"),
+                (404, "other"),
+                (403, "access_denied"),
+                (503, "throttled"),
+                (500, "server_error"),
+            ] {
+                let (endpoint, recorded) = scripted_server(vec![
+                    ScriptedResponse::new(404, Vec::new(), String::new()),
+                    ScriptedResponse::new(probe_status, Vec::new(), "{}".to_string()),
+                ])
+                .await;
+                let backend = backend(&endpoint);
+                let result = if method == Method::HEAD {
+                    backend.head("missing").await.map(|_| ())
+                } else {
+                    backend.get("missing", None).await.map(|_| ())
+                };
+                let error = result.expect_err("the object 404 must remain an error");
+                assert_eq!(error.class_label(), expected_class, "{method} with probe HTTP {probe_status}: {error:?}");
+                let recorded = recorded.lock().expect("recorder lock");
+                assert_eq!(recorded.len(), 2, "one bounded read-only probe per ambiguous object miss");
+                assert_eq!(recorded[0].method, method.as_str());
+                assert_eq!(recorded[1].method, "GET");
+                assert_eq!(recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1");
+            }
+        }
+    }
 }
diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs
index dd5a6a236..124d82d8f 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs
@@ -1089,10 +1089,17 @@ mod tests {
 
     #[test]
     fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
+        fn framed(payload: &str) -> String {
+            format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}")
+        }
+
         let token = progress_token(None, true, false);
         assert_eq!(
             token.encode(),
-            r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
+            concat!(
+                "\0odm-list:",
+                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
+            )
         );
         for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
             let token = progress_token(Some(count), true, false);
@@ -1100,16 +1107,17 @@ mod tests {
         }
         for version in [1, 2] {
             for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
-                let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
+                let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#));
                 assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
             }
         }
-        for encoded in [
+        for payload in [
             r#"{"t":"odm-list","v":1,"no_progress":1}"#,
             r#"{"t":"odm-list","v":2}"#,
             r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
         ] {
-            assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
+            let encoded = framed(payload);
+            assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
         }
     }
 
diff --git a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs
index 881db697d..8e5c12dd4 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs
@@ -129,6 +129,23 @@ impl NativeHttp {
         &self,
         request: reqwest::Request,
         error_code_header: &str,
+    ) -> Result<reqwest::Response, SourceError> {
+        self.send_classified(request, error_code_header, false).await
+    }
+
+    pub(super) async fn send_object(
+        &self,
+        request: reqwest::Request,
+        error_code_header: &str,
+    ) -> Result<reqwest::Response, SourceError> {
+        self.send_classified(request, error_code_header, true).await
+    }
+
+    async fn send_classified(
+        &self,
+        request: reqwest::Request,
+        error_code_header: &str,
+        not_found_on_404_without_code: bool,
     ) -> Result<reqwest::Response, SourceError> {
         let response = self.client.execute(request).await.map_err(classify_transport_error)?;
         let status = response.status();
@@ -140,14 +157,14 @@ impl NativeHttp {
             .get(error_code_header)
             .and_then(|value| value.to_str().ok())
             .map(str::to_string);
-        Err(classify_status(
-            status.as_u16(),
-            None,
-            match &code {
-                Some(code) => format!("source returned HTTP {status} ({code})"),
-                None => format!("source returned HTTP {status}"),
-            },
-        ))
+        let message = match &code {
+            Some(code) => format!("source returned HTTP {status} ({code})"),
+            None => format!("source returned HTTP {status}"),
+        };
+        match classify_status(status.as_u16(), code.as_deref(), message) {
+            SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
+            err => Err(err),
+        }
     }
 }
 
diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
index 618fff00e..6d1ba136a 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
@@ -334,8 +334,9 @@ const THROTTLE_CODES: &[&str] = &[
     "RequestLimitExceeded",
     "TooManyRequests",
     "RequestThrottled",
+    "ServerBusy",
 ];
-const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
+const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"];
 const ACCESS_DENIED_CODES: &[&str] = &[
     "AccessDenied",
     "InvalidAccessKeyId",
@@ -343,6 +344,7 @@ const ACCESS_DENIED_CODES: &[&str] = &[
     "AllAccessDisabled",
     "ExpiredToken",
     "InvalidToken",
+    "AuthorizationPermissionMismatch",
 ];
 
 pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
@@ -1817,6 +1819,7 @@ mod tests {
             ok(Vec::new(), CONTRACT_TAGGING),
             ok(Vec::new(), ""),
             status(404, ""),
+            ok(Vec::new(), ""),
             status(403, ACCESS_DENIED_BODY),
         ])
         .await;
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index b96b3e3f5..23268bf73 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -355,6 +355,8 @@ fi
                 self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
                 self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
                 self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+                initialized = self.run_step("Initialize functional evidence")
+                self.assertEqual(initialized.returncode, 0, initialized.stderr)
                 suite = self.directory / "auto-testing/rustfs-replication-test.sh"
                 suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
                 failed = self.run_step("Run replication suite")

From 1c4e9f1b652dbd846b47bd75b467dafb4d2df440 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:26:57 +0800
Subject: [PATCH 08/20] fix(odm): order initial installation against
 configuration removal (#7222)

* fix(odm): retain removal generation before initial install

* fix(odm): reserve generations only for configured buckets
---
 .../src/bucket/on_demand_migration/sys.rs     | 49 +++++++++++++------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs
index 6c348749e..61c3043c0 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs
@@ -746,8 +746,9 @@ impl OnDemandMigrationSys {
     /// (client construction is async). Requires a Tokio runtime for the
     /// install path; without one the config is logged and skipped.
     pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
-        let generation = self.next_generation();
-        let Some(config) = self.desired(config) else {
+        let config = self.desired(config);
+        let generation = self.reserve_generation(bucket, config.is_some());
+        let Some(config) = config else {
             self.remove_with_generation(bucket, generation);
             return;
         };
@@ -777,7 +778,8 @@ impl OnDemandMigrationSys {
     /// Installs, rebuilds, or removes the bucket state for `config`.
     /// Idempotent: the same config on an installed bucket is a no-op.
     pub async fn apply(&self, bucket: &str, config: Option<&OnDemandMigrationConfig>) -> ApplyOutcome {
-        let generation = self.next_generation();
+        let config = self.desired(config);
+        let generation = self.reserve_generation(bucket, config.is_some());
         self.apply_with_generation(bucket, config, generation).await
     }
 
@@ -838,7 +840,7 @@ impl OnDemandMigrationSys {
 
     /// Removes a bucket's state (idempotent), cancelling its token.
     pub fn remove(&self, bucket: &str) -> ApplyOutcome {
-        let generation = self.next_generation();
+        let generation = self.reserve_generation(bucket, false);
         self.remove_with_generation(bucket, generation)
     }
 
@@ -879,8 +881,17 @@ impl OnDemandMigrationSys {
         snapshots
     }
 
-    fn next_generation(&self) -> u64 {
-        self.generation.fetch_add(1, Ordering::Relaxed) + 1
+    fn reserve_generation(&self, bucket: &str, installing: bool) -> u64 {
+        // Reserve a desired install before its async client build, under the
+        // same lock that orders removals. Unconfigured buckets need no slot.
+        let mut buckets = self.buckets.write();
+        let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
+        if installing {
+            buckets.entry(bucket.to_string()).or_default().generation = generation;
+        } else if let Some(slot) = buckets.get_mut(bucket) {
+            slot.generation = generation;
+        }
+        generation
     }
 
     fn desired<'c>(&self, config: Option<&'c OnDemandMigrationConfig>) -> Option<&'c OnDemandMigrationConfig> {
@@ -1291,21 +1302,31 @@ mod tests {
         assert!(state.is_cancelled());
     }
 
+    #[tokio::test]
+    async fn absent_config_updates_do_not_allocate_bucket_slots() {
+        let sys = enabled_sys();
+        for index in 0..1000 {
+            let bucket = format!("unconfigured-{index}");
+            assert_eq!(sys.apply(&bucket, None).await, ApplyOutcome::NotDesired);
+            assert_eq!(sys.remove(&bucket), ApplyOutcome::NotDesired);
+        }
+        assert!(sys.buckets.read().is_empty(), "unconfigured buckets must not accumulate tombstones");
+    }
+
     #[tokio::test]
     async fn stale_install_cannot_overwrite_a_later_removal() {
         let sys = enabled_sys();
         let cfg = config(None);
-        let older = sys.next_generation();
-        let newer = sys.next_generation();
+        let older = sys.reserve_generation("b", true);
+        let newer = sys.reserve_generation("b", false);
         assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::NotDesired);
-        // The removal above did not create a slot; simulate an install that
-        // started before it and finishes after.
-        sys.apply_with_generation("b", Some(&cfg), older).await;
-        assert!(sys.state("b").is_some(), "no slot yet, so the older install lands");
+        assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);
+        assert!(sys.state("b").is_none(), "removal must supersede an in-flight first install");
 
+        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
         let installed = sys.state("b").unwrap();
-        let older = sys.next_generation();
-        let newer = sys.next_generation();
+        let older = sys.reserve_generation("b", true);
+        let newer = sys.reserve_generation("b", false);
         assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::Removed);
         assert!(installed.is_cancelled());
         assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);

From 955d49117428b88e025839375aed944fb66b077e Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:27:36 +0800
Subject: [PATCH 09/20] feat(build): make native GCS backends optional (#7223)

* feat(build): make native GCS backends optional

* test(odm): cover native Azure runtime credentials
---
 crates/ecstore/Cargo.toml                     |  5 ++-
 .../src/bucket/on_demand_migration/mod.rs     |  1 +
 .../on_demand_migration/source_client.rs      | 26 +++++++++++
 .../src/bucket/on_demand_migration/sys.rs     | 44 ++++++++++++++++++-
 crates/ecstore/src/bucket/remote_s3_client.rs |  2 +
 crates/ecstore/src/services/tier/mod.rs       |  1 +
 .../ecstore/src/services/tier/warm_backend.rs | 33 +++++++++++++-
 docs/operations/on-demand-migration.md        |  6 +++
 rustfs/Cargo.toml                             |  5 ++-
 .../src/admin/handlers/on_demand_migration.rs | 19 +++++++-
 rustfs/src/app/object/shared.rs               |  3 +-
 11 files changed, 136 insertions(+), 9 deletions(-)

diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml
index dd2063129..8dbfec7c5 100644
--- a/crates/ecstore/Cargo.toml
+++ b/crates/ecstore/Cargo.toml
@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
 # Compiles the controlled list-objects namespace-journal chaos injector into a
 # production binary (it is always available to tests). Off by default so the
 # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
@@ -212,8 +213,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
-google-cloud-storage = { workspace = true }
-google-cloud-auth = { workspace = true }
+google-cloud-storage = { workspace = true, optional = true }
+google-cloud-auth = { workspace = true, optional = true }
 faster-hex = { workspace = true }
 quick-xml = { workspace = true }
 ratelimit = { workspace = true }
diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs
index 6147f1f94..554dadd60 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs
@@ -30,6 +30,7 @@ mod backend_contract;
 pub mod backfill;
 pub mod breaker;
 pub mod config;
+#[cfg(feature = "gcs")]
 pub mod gcs;
 pub mod list_through;
 mod native_http;
diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
index 6d1ba136a..d2fc21c3e 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
@@ -26,6 +26,7 @@
 //! forwarded: v1 rejects SSE-C source objects outright.
 
 use super::azure::AzureSourceBackend;
+#[cfg(feature = "gcs")]
 use super::gcs::GcsNativeSourceBackend;
 use super::list_through::{ListPageError, validate_list_page};
 use crate::bucket::remote_s3_client::{
@@ -722,6 +723,9 @@ impl SourceClient {
                 )?;
                 Ok(Self::from_backend(Box::new(backend), spec))
             }
+            #[cfg(not(feature = "gcs"))]
+            SourceBackendSpec::Gcs(_) => Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")),
+            #[cfg(feature = "gcs")]
             SourceBackendSpec::Gcs(gcs) => {
                 let backend = GcsNativeSourceBackend::new(
                     &spec.endpoint,
@@ -1113,6 +1117,28 @@ mod tests {
         }
     }
 
+    #[cfg(not(feature = "gcs"))]
+    #[tokio::test]
+    async fn gcs_backend_not_compiled_keeps_hmac_s3_available() {
+        let mut native = spec(None);
+        native.provider = SourceProvider::GcsNative;
+        native.credentials = None;
+        native.backend = SourceBackendSpec::Gcs(GcsSourceSpec {
+            service_account_json: "{}".to_string(),
+        });
+        assert!(matches!(
+            SourceClient::new(&native).await,
+            Err(RemoteS3ClientError::BackendNotCompiled("gcs_native"))
+        ));
+
+        let mut hmac = spec(None);
+        hmac.provider = SourceProvider::Gcs;
+        hmac.endpoint = "https://storage.googleapis.com".to_string();
+        SourceClient::new(&hmac)
+            .await
+            .expect("GCS HMAC uses the always-available S3 backend");
+    }
+
     async fn scripted_client(spec: &SourceClientSpec, responses: Vec<Scripted>) -> (SourceClient, Recorded) {
         let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
         let connector = SharedHttpConnector::new(ScriptedConnector {
diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs
index 61c3043c0..ab42bd602 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs
+++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs
@@ -85,6 +85,8 @@ pub static GLOBAL_ON_DEMAND_MIGRATION_SYS: OnceLock<OnDemandMigrationSys> = Once
 /// `resolve` as [`OdmLookup::Unavailable`] and through status snapshots.
 #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
 pub enum OdmStateError {
+    #[error("the {0} backend is not included in this build")]
+    BackendNotCompiled(&'static str),
     /// `source.credentials` is `null`; the shared client builder has no
     /// anonymous mode yet (rustfs/backlog#2149 follow-up).
     #[error("anonymous source access is not supported yet; configure source credentials")]
@@ -310,11 +312,12 @@ impl BucketOdmState {
         write_back: Option<Arc<dyn OdmWriteBack>>,
     ) -> Arc<Self> {
         let spec = source_client_spec(config);
-        let client = if config.source.credentials.is_none() {
+        let client = if config.source.credentials.is_none() && !config.source.provider.is_native() {
             Err(OdmStateError::AnonymousUnsupported)
         } else {
             SourceClient::new(&spec).await.map(Arc::new).map_err(|err| match err {
                 RemoteS3ClientError::MissingCredentials => OdmStateError::AnonymousUnsupported,
+                RemoteS3ClientError::BackendNotCompiled(provider) => OdmStateError::BackendNotCompiled(provider),
                 other => OdmStateError::ClientBuild(other.to_string()),
             })
         };
@@ -1087,6 +1090,45 @@ mod tests {
         assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Rebuilt);
     }
 
+    #[tokio::test]
+    async fn native_azure_uses_provider_credentials_without_s3_credentials() {
+        let sys = enabled_sys();
+        let mut cfg = config(None);
+        cfg.source.provider = Provider::Azure;
+        cfg.source.endpoint = None;
+        cfg.source.credentials = None;
+        cfg.source.azure = Some(super::super::config::AzureSourceConfig {
+            account: "legacyaccount".to_string(),
+            account_key: Some("c2VjcmV0LWtleQ==".to_string()),
+            sas_token: None,
+        });
+        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
+        let state = ready_state(sys.resolve("b", "k"));
+        assert!(state.client().is_ok(), "native credentials must not be classified as anonymous S3");
+    }
+
+    #[cfg(not(feature = "gcs"))]
+    #[tokio::test]
+    async fn gcs_backend_not_compiled_is_unavailable_not_anonymous() {
+        let sys = enabled_sys();
+        let mut cfg = config(None);
+        cfg.source.provider = Provider::GcsNative;
+        cfg.source.credentials = None;
+        cfg.source.gcs = Some(super::super::config::GcsSourceConfig {
+            service_account_json: "{}".to_string(),
+        });
+        let encoded = cfg.to_json().expect("GCS config is serializable without the backend");
+        let restored: OnDemandMigrationConfig = serde_json::from_slice(&encoded).expect("GCS config stays readable");
+        assert_eq!(restored, cfg);
+        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
+        match sys.resolve("b", "k") {
+            Some(OdmLookup::Unavailable { error, .. }) => {
+                assert_eq!(error, OdmStateError::BackendNotCompiled("gcs_native"));
+            }
+            other => panic!("expected unavailable backend, got {other:?}"),
+        }
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     async fn singleflight_admits_one_leader_per_key() {
         let sys = enabled_sys();
diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs
index a434e09ac..6389aa67e 100644
--- a/crates/ecstore/src/bucket/remote_s3_client.rs
+++ b/crates/ecstore/src/bucket/remote_s3_client.rs
@@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec {
 
 #[derive(Debug, thiserror::Error)]
 pub enum RemoteS3ClientError {
+    #[error("the {0} backend is not included in this build")]
+    BackendNotCompiled(&'static str),
     #[error("remote endpoint requires credentials")]
     MissingCredentials,
     #[error("{0}")]
diff --git a/crates/ecstore/src/services/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs
index 8375f9dd0..bc22fb52a 100644
--- a/crates/ecstore/src/services/tier/mod.rs
+++ b/crates/ecstore/src/services/tier/mod.rs
@@ -25,6 +25,7 @@ pub(crate) mod tier_probe_intent;
 pub mod warm_backend;
 pub mod warm_backend_aliyun;
 pub mod warm_backend_azure;
+#[cfg(feature = "gcs")]
 pub mod warm_backend_gcs;
 pub mod warm_backend_huaweicloud;
 pub mod warm_backend_minio;
diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs
index 22134d744..ca4864622 100644
--- a/crates/ecstore/src/services/tier/warm_backend.rs
+++ b/crates/ecstore/src/services/tier/warm_backend.rs
@@ -19,13 +19,14 @@
 #![allow(clippy::all)]
 
 use crate::error::is_err_bucket_not_found;
+#[cfg(feature = "gcs")]
+use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
 use crate::services::tier::{
     tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
     tier_config::{TierConfig, TierType},
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
     warm_backend_azure::WarmBackendAzure,
-    warm_backend_gcs::WarmBackendGCS,
     warm_backend_huaweicloud::WarmBackendHuaweicloud,
     warm_backend_minio::WarmBackendMinIO,
     warm_backend_r2::WarmBackendR2,
@@ -912,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
                 });
             }
         }
+        #[cfg(not(feature = "gcs"))]
+        TierType::GCS => {
+            return Err(AdminError {
+                code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
+                message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
+                status_code: StatusCode::NOT_IMPLEMENTED,
+            });
+        }
+        #[cfg(feature = "gcs")]
         TierType::GCS => {
             if let Some(gcs_config) = tier.gcs.as_ref() {
                 let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
@@ -1028,6 +1038,27 @@ mod tests {
 
     const PROBE_VERSION: &str = "remote-v2";
 
+    #[cfg(not(feature = "gcs"))]
+    #[tokio::test]
+    async fn gcs_backend_not_compiled_preserves_config() {
+        let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
+        let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
+        assert_eq!(tier.tier_type, TierType::GCS);
+        let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
+        let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
+        assert_eq!(restored.tier_type, TierType::GCS);
+        let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
+        assert_eq!(restored_gcs.bucket, "archive");
+        assert_eq!(restored_gcs.creds, "secret");
+        assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
+        let error = match new_warm_backend(&tier, false).await {
+            Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
+            Err(error) => error,
+        };
+        assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
+        assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
+    }
+
     struct CountingBackend {
         put_result: fn() -> Result<String, std::io::Error>,
         removes: Arc<AtomicUsize>,
diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md
index d253e33a2..adf844700 100644
--- a/docs/operations/on-demand-migration.md
+++ b/docs/operations/on-demand-migration.md
@@ -7,6 +7,12 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t
 
 The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
 
+## Optional Google dependencies
+
+The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature.
+
+Both builds can read, redact and preserve GCS configuration. A build without `gcs` rejects native ODM client construction with `OnDemandMigrationBackendNotCompiled` (HTTP 501); persisted native sources report an unavailable client. GCS tier initialization returns `XRustFSAdminTierTypeUnsupported` (HTTP 501). Do not deploy that build to a cluster with GCS tiers containing transitioned objects: the configuration remains intact, but reading their remote data requires a GCS-capable binary. The `gcs` provider using HMAC credentials and the S3 interoperability API remains available in every build; only `gcs_native` and native GCS tier clients need the feature.
+
 ## List continuation token rollout
 
 `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.
diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml
index 6d716d63b..063ce400e 100644
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -57,7 +57,8 @@ name = "swift_object_integration_test"
 required-features = ["swift"]
 
 [features]
-default = ["ftps", "webdav"]
+default = ["ftps", "webdav", "gcs"]
+gcs = ["rustfs-ecstore/gcs"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]
 swift = ["rustfs-protocols/swift"]
@@ -66,7 +67,7 @@ sftp = ["rustfs-protocols/sftp"]
 license = []
 io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
-full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"]
+full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
 e2e-test-hooks = []
 # Shortens Connect credentials only in debug E2E builds.
 connect-e2e-short-credentials = []
diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs
index 103e6ee8d..d8bc9d713 100644
--- a/rustfs/src/admin/handlers/on_demand_migration.rs
+++ b/rustfs/src/admin/handlers/on_demand_migration.rs
@@ -90,6 +90,8 @@ const BACKFILL_OP_CANCEL: &str = "cancel";
 pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled";
 /// Error code returned when the source bucket did not answer the probe.
 pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable";
+/// Error code returned when the configured provider was excluded at build time.
+pub(crate) const ERR_CODE_BACKEND_NOT_COMPILED: &str = "OnDemandMigrationBackendNotCompiled";
 /// Error code returned by `GET` when the bucket has no configuration.
 pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration";
 /// Error code (409) returned by `start` while a backfill job holds the lease.
@@ -630,12 +632,17 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie
     }
 }
 
-/// Builder failures are input errors: the endpoint policy, the CA PEM or the
-/// credentials the operator supplied. Anonymous sources are not wired yet
+/// Distinguishes excluded backends from invalid endpoint, CA or credentials.
+/// Anonymous S3 sources are not wired yet
 /// (ODM-05 adds the credential-less path), so `MissingCredentials` is a 400
 /// naming the field instead of an opaque internal error.
 fn client_build_error(err: RemoteS3ClientError) -> S3Error {
     match err {
+        RemoteS3ClientError::BackendNotCompiled(provider) => custom_error(
+            ERR_CODE_BACKEND_NOT_COMPILED,
+            StatusCode::NOT_IMPLEMENTED,
+            format!("the {provider} backend is not included in this build; rebuild with the gcs feature"),
+        ),
         RemoteS3ClientError::MissingCredentials => admin_s3_error(
             S3ErrorCode::InvalidArgument,
             "source.credentials is required: anonymous sources are not supported yet",
@@ -1288,6 +1295,14 @@ mod tests {
         assert!(err.message().unwrap_or_default().contains("source.credentials"));
     }
 
+    #[test]
+    fn backend_not_compiled_is_distinct_from_invalid_credentials() {
+        let err = client_build_error(RemoteS3ClientError::BackendNotCompiled("gcs_native"));
+        assert_eq!(err.code(), &S3ErrorCode::Custom(ERR_CODE_BACKEND_NOT_COMPILED.into()));
+        assert_eq!(err.status_code(), Some(StatusCode::NOT_IMPLEMENTED));
+        assert!(err.message().unwrap_or_default().contains("gcs feature"));
+    }
+
     #[test]
     fn module_switch_defaults_on_and_reads_the_env() {
         temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || assert!(module_enabled()));
diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs
index 454cd9911..8f0808956 100644
--- a/rustfs/src/app/object/shared.rs
+++ b/rustfs/src/app/object/shared.rs
@@ -992,7 +992,7 @@ pub(crate) fn odm_source_error_response(policy: &PolicyConfig, class: &'static s
 /// Metrics/message label for a bucket whose source client could not be built.
 pub(crate) fn odm_state_error_class(error: &OdmStateError) -> &'static str {
     match error {
-        OdmStateError::AnonymousUnsupported => "unsupported",
+        OdmStateError::AnonymousUnsupported | OdmStateError::BackendNotCompiled(_) => "unsupported",
         OdmStateError::ClientBuild(_) => "client_build",
     }
 }
@@ -2035,6 +2035,7 @@ mod on_demand_migration_tests {
     #[test]
     fn odm_state_error_class_is_stable() {
         assert_eq!(odm_state_error_class(&OdmStateError::AnonymousUnsupported), "unsupported");
+        assert_eq!(odm_state_error_class(&OdmStateError::BackendNotCompiled("gcs_native")), "unsupported");
         assert_eq!(odm_state_error_class(&OdmStateError::ClientBuild("tls".to_string())), "client_build");
     }
 

From c9acc33720b0bb8386e72e47d1bc9c524b4e1fbb Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:28:54 +0800
Subject: [PATCH 10/20] test(odm): verify rc5 rollback configuration recovery
 (#7224)

* test(odm): verify rc5 rollback configuration recovery

* ci(e2e): run the ODM rollback recovery scenario
---
 .github/workflows/e2e-upgrade.yml             |  4 +
 .../src/upgrade_compatibility_test.rs         | 91 ++++++++++++++++++-
 docs/operations/on-demand-migration.md        |  8 ++
 3 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml
index 8cc1d53a5..324396946 100644
--- a/.github/workflows/e2e-upgrade.yml
+++ b/.github/workflows/e2e-upgrade.yml
@@ -78,6 +78,10 @@ jobs:
             cache_key: e2e-bucket-config-rollback
             test: rollback_to_previous_release_reads_current_bucket_metadata
             artifact: bucket-config-rollback
+          - name: ODM configuration recovery after rc.5 rollback
+            cache_key: e2e-odm-config-rollback
+            test: rc5_rollback_requires_restoring_odm_configuration
+            artifact: odm-config-rollback
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs
index 9f1871552..170582f6b 100644
--- a/crates/e2e_test/src/upgrade_compatibility_test.rs
+++ b/crates/e2e_test/src/upgrade_compatibility_test.rs
@@ -15,7 +15,8 @@
 use crate::common::{
     RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
 };
-use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
 use crate::replication_extension_test::{
     LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
 };
@@ -38,6 +39,7 @@ type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -277,6 +279,93 @@ async fn exercise_mixed_cluster(
     Ok(())
 }
 
+/// Pins the published old writer's limitation and the supported recovery
+/// procedure. This is not a promise that mixed-version ODM is supported.
+/// Replace the loss assertion when ODM gains independent persistence;
+/// preserving configuration across rc.5 writes is then an improvement.
+#[tokio::test]
+#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
+async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let version = tokio::process::Command::new(&previous_binary)
+        .arg("--version")
+        .output()
+        .await?;
+    assert!(version.status.success(), "previous binary must report its version");
+    assert!(
+        String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
+        "this compatibility scenario requires the published rc.5 writer"
+    );
+    let mut env = OdmTestEnv::start().await?;
+    let bucket = "odm-rc5-rollback";
+    let source_bucket = "odm-rc5-source";
+    env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
+    env.seed_source(
+        source_bucket,
+        &[SeedObject::new(
+            "source-only",
+            bytes::Bytes::from_static(b"source read after recovery"),
+        )],
+    );
+    env.rustfs.create_test_bucket(bucket).await?;
+    let saved_config = env.fake_source_spec(source_bucket);
+    assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
+    let before = env.get_config(bucket).await?;
+    assert_eq!(before.status, 200);
+    let expected_config = before
+        .json()?
+        .get("config")
+        .cloned()
+        .ok_or("configuration response omitted config")?;
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key("local")
+        .body(ByteStream::from_static(b"local data survives rollback"))
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restarted = env.get_config(bucket).await?;
+    assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
+    assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
+
+    restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(
+            Tagging::builder()
+                .tag_set(Tag::builder().key("writer").value("rc5").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let missing = env.get_config(bucket).await?;
+    assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
+    assert!(missing.body.contains("NoSuchConfiguration"));
+    assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
+
+    assert_eq!(
+        env.configure_source(bucket, &saved_config).await?.status,
+        200,
+        "restore from saved full configuration"
+    );
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restored = env.get_config(bucket).await?;
+    assert_eq!(restored.status, 200, "restored ODM configuration persists");
+    assert_eq!(restored.json()?.get("config"), Some(&expected_config));
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        read_object(&env.client, bucket, "source-only", None).await?.1,
+        b"source read after recovery"
+    );
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md
index adf844700..e9beb27d7 100644
--- a/docs/operations/on-demand-migration.md
+++ b/docs/operations/on-demand-migration.md
@@ -7,6 +7,14 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t
 
 The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
 
+## Upgrade and rollback compatibility
+
+ODM configuration is stored in two additional keys in the existing bucket metadata map. The metadata format version remains `1` for MinIO compatibility. RustFS `1.0.0-rc.5` only re-encodes its 44 known keys: a bucket configuration write through an rc.5 node discards the ODM configuration and timestamp, even if another node originally wrote them. Restarting a newer binary cannot recover the discarded values. This also means ODM is not supported during a rolling upgrade that still allows rc.5 nodes to write bucket metadata.
+
+Upgrade every node before enabling ODM. Before any rollback to rc.5, stop new migration work, retain a secure copy of the original full configuration and credentials, and disable ODM on every bucket and node. The redacted configuration GET and metadata export are not credential backups. Objects still present only at the source cannot be read through RustFS while ODM is disabled or rc.5 is running; finish migration first, redirect those reads to the source, or plan a maintenance window. After all nodes return to a compatible release, reapply and validate the saved configuration; already stored local objects remain local. Turning the global module switch off alone does not make an old metadata writer preserve these keys.
+
+The ignored `upgrade_compatibility_test::rc5_rollback_requires_restoring_odm_configuration` test pins release commit `40a2470feb567201165a5b809b7598bb4b1f68f5`, restarts against the same data directory, writes bucket tags through rc.5, and verifies configuration recovery after returning to the current binary. Set `RUSTFS_UPGRADE_SOURCE_BINARY` to that release's executable and run `cargo test -p e2e_test rc5_rollback_requires_restoring_odm_configuration -- --ignored --test-threads=1`. The test records a known old-writer limitation; it does not certify mixed-version ODM operation.
+
 ## Optional Google dependencies
 
 The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature.

From 14cef9142370c8fae473d5c15e03f1b6a5317344 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:30:18 +0800
Subject: [PATCH 11/20] fix(admin): add isolated bucket metadata diagnostics
 (#7225)

---
 docs/operations/bucket-metadata-recovery.md |  22 +
 rustfs/src/admin/handlers/bucket_meta.rs    | 729 +++++++++++++-------
 2 files changed, 516 insertions(+), 235 deletions(-)
 create mode 100644 docs/operations/bucket-metadata-recovery.md

diff --git a/docs/operations/bucket-metadata-recovery.md b/docs/operations/bucket-metadata-recovery.md
new file mode 100644
index 000000000..ff513e9db
--- /dev/null
+++ b/docs/operations/bucket-metadata-recovery.md
@@ -0,0 +1,22 @@
+# Bucket metadata diagnostics and recovery
+
+`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.
+
+To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:
+
+- Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`.
+- Readable entries under `_diagnostic/<bucket>/<config>`; target credentials remain redacted.
+- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details.
+
+`complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations.
+
+## Recover unreadable replication targets
+
+MinIO target configuration may be an array or KMS-encrypted data that RustFS cannot decode. Diagnosis preserves the failure instead of interpreting it as an empty target set.
+
+1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
+2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
+3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully.
+4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it.
+
+Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive.
diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs
index b1c4af3d2..3669f62eb 100644
--- a/rustfs/src/admin/handlers/bucket_meta.rs
+++ b/rustfs/src/admin/handlers/bucket_meta.rs
@@ -64,6 +64,9 @@ use time::OffsetDateTime;
 use tracing::warn;
 use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
 
+const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic";
+const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json";
+
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
 const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
@@ -97,9 +100,198 @@ fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) ->
     checked_raw_xml(validated, raw, deserialize::<VersioningConfiguration>)
 }
 
+async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result<Option<Vec<u8>>> {
+    match conf {
+        BUCKET_POLICY_CONFIG => {
+            let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let config_json =
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
+            Ok(Some(config_json))
+        }
+        BUCKET_NOTIFICATION_CONFIG => {
+            let config: s3s::dto::NotificationConfiguration = match metadata_sys::get_notification_config(bucket).await {
+                Ok(Some(res)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+                Ok(None) => return Ok(None),
+            };
+
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .notification_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_LIFECYCLE_CONFIG => {
+            let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                .lifecycle_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_TAGGING_CONFIG => {
+            let config: Tagging = match metadata_sys::get_tagging_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                .tagging_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_QUOTA_CONFIG_FILE => {
+            let config: BucketQuota = match metadata_sys::get_quota_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let config_json =
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+            Ok(Some(config_json))
+        }
+        OBJECT_LOCK_CONFIG => {
+            let config = match metadata_sys::get_object_lock_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .object_lock_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_SSECONFIG => {
+            let config = match metadata_sys::get_sse_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .encryption_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_VERSIONING_CONFIG => {
+            let config = match metadata_sys::get_versioning_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .versioning_config_xml
+                .clone();
+            let config_xml = checked_versioning_xml(&config, raw_config)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_REPLICATION_CONFIG => {
+            let config = match metadata_sys::get_replication_config(bucket).await {
+                Ok((res, _)) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+            let raw_config = metadata_sys::get(bucket)
+                .await
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                .replication_config_xml
+                .clone();
+            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
+
+            Ok(Some(config_xml))
+        }
+        BUCKET_TARGETS_FILE => {
+            let config: BucketTargets = match metadata_sys::get_bucket_targets_config(bucket).await {
+                Ok(res) => res,
+                Err(e) => {
+                    if e == StorageError::ConfigNotFound {
+                        return Ok(None);
+                    }
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                }
+            };
+
+            let config_json = serde_json::to_vec(&config.redacted_credentials())
+                .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+            Ok(Some(config_json))
+        }
+        _ => Ok(None),
+    }
+}
+
 #[derive(Debug, Default, serde::Deserialize)]
 pub struct ExportBucketMetadataQuery {
+    #[serde(default)]
     pub bucket: String,
+    #[serde(default)]
+    pub diagnostic: bool,
 }
 
 pub struct ExportBucketMetadata {}
@@ -169,6 +361,7 @@ impl Operation for ExportBucketMetadata {
         };
 
         let mut zip_writer = ZipWriter::new(Cursor::new(Vec::new()));
+        let mut errors = Vec::new();
 
         let confs = [
             BUCKET_POLICY_CONFIG,
@@ -186,244 +379,49 @@ impl Operation for ExportBucketMetadata {
         for bucket in buckets {
             for &conf in confs.iter() {
                 let conf_path = path_join_buf(&[bucket.name.as_str(), conf]);
-                match conf {
-                    BUCKET_POLICY_CONFIG => {
-                        let config: BucketPolicy = match metadata_sys::get_bucket_policy(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let config_json = serde_json::to_vec(&config)
-                            .map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
+                let config = match exported_bucket_config(&bucket.name, conf).await {
+                    Ok(Some(config)) => config,
+                    Ok(None) => continue,
+                    Err(error) if !query.diagnostic => return Err(error),
+                    Err(_) => {
+                        errors.push(serde_json::json!({
+                            "bucket": bucket.name,
+                            "config": conf,
+                            "code": "configuration_unavailable",
+                        }));
+                        continue;
                     }
-                    BUCKET_NOTIFICATION_CONFIG => {
-                        let config: s3s::dto::NotificationConfiguration =
-                            match metadata_sys::get_notification_config(&bucket.name).await {
-                                Ok(Some(res)) => res,
-                                Err(e) => {
-                                    if e == StorageError::ConfigNotFound {
-                                        continue;
-                                    }
-                                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                                }
-                                Ok(None) => continue,
-                            };
-
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .notification_config_xml
-                            .clone();
-                        let config_xml =
-                            checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_LIFECYCLE_CONFIG => {
-                        let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                            .lifecycle_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-                    }
-                    BUCKET_TAGGING_CONFIG => {
-                        let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                            .tagging_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-                    }
-                    BUCKET_QUOTA_CONFIG_FILE => {
-                        let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let config_json =
-                            serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    OBJECT_LOCK_CONFIG => {
-                        let config = match metadata_sys::get_object_lock_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .object_lock_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_SSECONFIG => {
-                        let config = match metadata_sys::get_sse_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .encryption_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_VERSIONING_CONFIG => {
-                        let config = match metadata_sys::get_versioning_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .versioning_config_xml
-                            .clone();
-                        let config_xml = checked_versioning_xml(&config, raw_config)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_REPLICATION_CONFIG => {
-                        let config = match metadata_sys::get_replication_config(&bucket.name).await {
-                            Ok((res, _)) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-                        let raw_config = metadata_sys::get(&bucket.name)
-                            .await
-                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                            .replication_config_xml
-                            .clone();
-                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_xml)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    BUCKET_TARGETS_FILE => {
-                        let config: BucketTargets = match metadata_sys::get_bucket_targets_config(&bucket.name).await {
-                            Ok(res) => res,
-                            Err(e) => {
-                                if e == StorageError::ConfigNotFound {
-                                    continue;
-                                }
-                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                            }
-                        };
-
-                        let config_json = serde_json::to_vec(&config.redacted_credentials())
-                            .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-                        zip_writer
-                            .start_file(conf_path, SimpleFileOptions::default())
-                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
-                        zip_writer
-                            .write_all(&config_json)
-                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
-                    }
-                    _ => {}
-                }
+                };
+                let conf_path = if query.diagnostic {
+                    path_join_buf(&[DIAGNOSTIC_EXPORT_PREFIX, &conf_path])
+                } else {
+                    conf_path
+                };
+                zip_writer
+                    .start_file(conf_path, SimpleFileOptions::default())
+                    .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
+                zip_writer
+                    .write_all(&config)
+                    .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
             }
         }
 
+        if query.diagnostic {
+            let manifest = serde_json::to_vec(&serde_json::json!({
+                "version": 1,
+                "mode": "diagnostic",
+                "complete": errors.is_empty(),
+                "errors": errors,
+            }))
+            .map_err(|e| s3_error!(InternalError, "failed to serialize diagnostic manifest: {e}"))?;
+            zip_writer
+                .start_file(DIAGNOSTIC_EXPORT_MANIFEST, SimpleFileOptions::default())
+                .map_err(|e| s3_error!(InternalError, "failed to start diagnostic manifest: {e}"))?;
+            zip_writer
+                .write_all(&manifest)
+                .map_err(|e| s3_error!(InternalError, "failed to write diagnostic manifest: {e}"))?;
+        }
+
         let zip_bytes = zip_writer
             .finish()
             .map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
@@ -431,8 +429,17 @@ impl Operation for ExportBucketMetadata {
         header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
         header.insert(
             CONTENT_DISPOSITION,
-            "attachment; filename=bucket-meta.zip".parse().expect("valid header value"),
+            if query.diagnostic {
+                "attachment; filename=bucket-meta-diagnostic.zip"
+            } else {
+                "attachment; filename=bucket-meta.zip"
+            }
+            .parse()
+            .expect("valid header value"),
         );
+        if query.diagnostic {
+            header.insert("x-rustfs-bucket-metadata-export", "diagnostic".parse().expect("valid header value"));
+        }
         header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
         Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
     }
@@ -499,6 +506,18 @@ impl Operation for ImportBucketMetadata {
             file_contents.push((file_path, content));
         }
 
+        // Reject the whole archive before creating buckets or writing configs,
+        // even when the marker is malformed or follows ordinary config entries.
+        if file_contents.iter().any(|(path, _)| {
+            path == DIAGNOSTIC_EXPORT_MANIFEST
+                || path == DIAGNOSTIC_EXPORT_PREFIX
+                || path
+                    .strip_prefix(DIAGNOSTIC_EXPORT_PREFIX)
+                    .is_some_and(|suffix| suffix.starts_with('/'))
+        }) {
+            return Err(s3_error!(InvalidRequest, "diagnostic bucket metadata archives cannot be imported"));
+        }
+
         let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
         let quota_fleet_proof =
             if durable_quota_import {
@@ -1419,6 +1438,246 @@ mod backup_zip_compatibility_tests {
         assert_eq!(response.output.0, StatusCode::OK);
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn diagnostic_export_isolated_errors_and_import_recovers_unreadable_targets() {
+        const HEALTHY: &str = "diagnostic-healthy";
+        const UNREADABLE: &str = "diagnostic-unreadable";
+        const REPLACEMENT_TARGETS: &[u8] = br#"{"targets":[]}"#;
+        const SECRET: &str = "diagnostic-must-not-expose-this-secret";
+
+        let _ = rustfs_credentials::init_global_action_credentials(
+            Some(ROOT_ACCESS_KEY.to_string()),
+            Some(ROOT_SECRET_KEY.to_string()),
+        );
+        let temp = tempfile::tempdir().expect("create diagnostic export test root");
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(temp.path())
+            .disk_count(1)
+            .build()
+            .await;
+        env.make_bucket(HEALTHY, false).await;
+        env.make_bucket(UNREADABLE, false).await;
+        rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore))
+            .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
+            .await
+            .expect("seed IAM format");
+        let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
+            .await
+            .expect("build test IAM");
+        publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
+            Arc::clone(&env.ecstore),
+            iam,
+            Arc::new(rustfs_kms::KmsServiceManager::new()),
+        )));
+        metadata_sys::update(HEALTHY, BUCKET_VERSIONING_CONFIG, VERSIONING_XML.to_vec())
+            .await
+            .expect("seed healthy bucket config");
+
+        let minio_blob = hex_simd::decode_to_vec(
+            include_str!("../../../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex").trim(),
+        )
+        .expect("decode real MinIO metadata fixture");
+        let minio_metadata = BucketMetadata::unmarshal(&minio_blob[4..]).expect("read MinIO metadata fixture");
+        let targets_array = format!(r#"[{{"credentials":{{"secretKey":"{SECRET}"}}}}]"#).into_bytes();
+        for unreadable_targets in [targets_array, minio_metadata.bucket_targets_config_json] {
+            let mut metadata = metadata_sys::get_config_from_disk(UNREADABLE)
+                .await
+                .expect("load bucket before simulating MinIO targets");
+            let incarnation = metadata.bucket_incarnation_id;
+            metadata.bucket_targets_config_json = unreadable_targets.clone();
+            metadata
+                .save_with_store(Arc::clone(&env.ecstore))
+                .await
+                .expect("persist unreadable targets fixture");
+            crate::storage::storage_api::set_bucket_metadata(UNREADABLE.to_string(), metadata)
+                .await
+                .expect("publish unreadable targets fixture");
+            assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err());
+
+            let strict_error = ExportBucketMetadata {}
+                .call(
+                    admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
+                    Params::new(),
+                )
+                .await
+                .expect_err("a complete export must fail closed on unreadable targets");
+            assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError);
+
+            let response = ExportBucketMetadata {}
+                .call(
+                    admin_request(
+                        Method::GET,
+                        Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
+                        Vec::new(),
+                    ),
+                    Params::new(),
+                )
+                .await
+                .expect("one unreadable bucket must not abort diagnostic export");
+            assert_eq!(response.output.0, StatusCode::OK);
+            assert_eq!(response.headers["x-rustfs-bucket-metadata-export"], "diagnostic");
+            assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta-diagnostic.zip");
+            let bytes = response.output.1.collect().await.expect("read diagnostic archive").to_bytes();
+            let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open diagnostic archive");
+            let mut files = HashMap::new();
+            for index in 0..archive.len() {
+                let mut file = archive.by_index(index).expect("read diagnostic entry");
+                let mut content = Vec::new();
+                file.read_to_end(&mut content).expect("read diagnostic config");
+                assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes()));
+                assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/"));
+                files.insert(file.name().to_string(), content);
+            }
+            assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML);
+            assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}")));
+            assert!(!files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_TARGETS_FILE}")));
+            let manifest: serde_json::Value =
+                serde_json::from_slice(&files[DIAGNOSTIC_EXPORT_MANIFEST]).expect("decode diagnostic manifest");
+            assert_eq!(
+                manifest,
+                serde_json::json!({
+                    "version": 1,
+                    "mode": "diagnostic",
+                    "complete": false,
+                    "errors": [{ "bucket": UNREADABLE, "config": BUCKET_TARGETS_FILE, "code": "configuration_unavailable" }],
+                })
+            );
+
+            let error = ImportBucketMetadata {}
+                .call(
+                    admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
+                    Params::new(),
+                )
+                .await
+                .expect_err("diagnostic exports are never backups");
+            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+            assert_eq!(
+                metadata_sys::get_config_from_disk(UNREADABLE)
+                    .await
+                    .expect("load rejected import state")
+                    .bucket_targets_config_json,
+                unreadable_targets
+            );
+
+            import_archive(zip_with_entries(UNREADABLE, &[(BUCKET_TARGETS_FILE, REPLACEMENT_TARGETS)])).await;
+            let recovered = metadata_sys::get_config_from_disk(UNREADABLE)
+                .await
+                .expect("read recovered targets from disk");
+            assert_eq!(recovered.bucket_incarnation_id, incarnation);
+            assert_eq!(recovered.bucket_targets_config_json, REPLACEMENT_TARGETS);
+            assert!(
+                metadata_sys::get_bucket_targets_config(UNREADABLE)
+                    .await
+                    .expect("existing import API must recover targets readers")
+                    .is_empty()
+            );
+        }
+
+        // The marker may be malformed, come last, or be removed while the
+        // reserved directory remains. None may allow an earlier config write.
+        for marker in [
+            DIAGNOSTIC_EXPORT_MANIFEST.to_string(),
+            DIAGNOSTIC_EXPORT_PREFIX.to_string(),
+            format!("{DIAGNOSTIC_EXPORT_PREFIX}/bucket/config"),
+        ] {
+            let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
+            writer
+                .start_file(format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default())
+                .expect("start ordinary config before diagnostic marker");
+            writer
+                .write_all(b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+                .expect("write ordinary config before diagnostic marker");
+            writer
+                .start_file(
+                    format!("diagnostic-never-created/{BUCKET_VERSIONING_CONFIG}"),
+                    SimpleFileOptions::default(),
+                )
+                .expect("start a nonexistent bucket config before diagnostic marker");
+            writer.write_all(VERSIONING_XML).expect("write nonexistent bucket config");
+            writer
+                .start_file(marker, SimpleFileOptions::default())
+                .expect("start diagnostic marker");
+            writer.write_all(b"not json").expect("write malformed diagnostic marker");
+            let error = ImportBucketMetadata {}
+                .call(
+                    admin_request(
+                        Method::PUT,
+                        Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"),
+                        writer.finish().expect("finish marked archive").into_inner(),
+                    ),
+                    Params::new(),
+                )
+                .await
+                .expect_err("diagnostic preflight must reject before any config write");
+            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+            assert_eq!(
+                metadata_sys::get_config_from_disk(HEALTHY)
+                    .await
+                    .expect("read healthy config after rejected import")
+                    .versioning_config_xml,
+                VERSIONING_XML
+            );
+            assert!(
+                env.ecstore
+                    .get_bucket_info("diagnostic-never-created", &BucketOptions::default())
+                    .await
+                    .is_err(),
+                "diagnostic preflight must reject before bucket creation"
+            );
+        }
+
+        let response = ExportBucketMetadata {}
+            .call(
+                admin_request(
+                    Method::GET,
+                    Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
+                    Vec::new(),
+                ),
+                Params::new(),
+            )
+            .await
+            .expect("diagnostic export after recovery");
+        let bytes = response
+            .output
+            .1
+            .collect()
+            .await
+            .expect("read complete diagnostic archive")
+            .to_bytes();
+        let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open complete diagnostic archive");
+        let manifest: serde_json::Value = serde_json::from_reader(
+            archive
+                .by_name(DIAGNOSTIC_EXPORT_MANIFEST)
+                .expect("complete diagnostic manifest"),
+        )
+        .expect("parse complete diagnostic manifest");
+        assert_eq!(manifest["complete"], true);
+        let error = ImportBucketMetadata {}
+            .call(
+                admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
+                Params::new(),
+            )
+            .await
+            .expect_err("complete diagnostics must still reject import");
+        assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
+
+        let response = ExportBucketMetadata {}
+            .call(
+                admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
+                Params::new(),
+            )
+            .await
+            .expect("ordinary cluster export must work after API recovery");
+        assert!(!response.headers.contains_key("x-rustfs-bucket-metadata-export"));
+        assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta.zip");
+        let bytes = response.output.1.collect().await.expect("read ordinary archive").to_bytes();
+        let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("open ordinary archive");
+        assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err());
+        assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok());
+        assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok());
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn g_zip_001_002_003_use_real_admin_archive_and_persistence_paths() {

From 8fc1c9281e80487382af00373fa33e08ca835fff Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:31:58 +0800
Subject: [PATCH 12/20] refactor(odm): move migration orchestration into
 application (#7226)

* refactor(odm): move migration orchestration into application

* style(odm): format relocated listing test imports
---
 Cargo.lock                                    |   4 +-
 crates/ecstore/Cargo.toml                     |   1 -
 crates/ecstore/src/api/mod.rs                 |  72 ++------
 crates/ecstore/src/bucket/metadata.rs         |  89 +++-------
 crates/ecstore/src/bucket/metadata_sys.rs     | 152 ++++++----------
 crates/ecstore/src/bucket/mod.rs              |   1 -
 crates/ecstore/src/bucket/remote_s3_client.rs |   6 +-
 crates/obs/src/metrics/mod.rs                 |   1 +
 crates/obs/src/metrics/storage_api.rs         | 168 ++++++------------
 .../background-services-inventory.md          |   6 +-
 docs/architecture/crate-boundaries.md         |  25 +++
 .../ecstore-api-facade-inventory.md           |   7 +
 .../remote-credential-sealing-adr.md          |   8 +-
 docs/operations/on-demand-migration.md        |   6 +-
 rustfs/Cargo.toml                             |   5 +-
 .../src/admin/handlers/on_demand_migration.rs |  39 ++--
 rustfs/src/admin/storage_api.rs               |  40 +----
 rustfs/src/app/bucket_list_through.rs         |  18 +-
 rustfs/src/app/object/get.rs                  |   8 +-
 rustfs/src/app/object/head.rs                 |  12 +-
 .../src/app/object/on_demand_migration_put.rs |   6 +-
 rustfs/src/app/object/shared.rs               |   4 +-
 rustfs/src/app/storage_api.rs                 |  33 ----
 rustfs/src/lib.rs                             |   1 +
 .../src}/on_demand_migration/azure.rs         |  10 +-
 .../on_demand_migration/backend_contract.rs   |   2 +-
 .../src}/on_demand_migration/backfill.rs      |  32 ++--
 .../src}/on_demand_migration/breaker.rs       |   0
 .../src}/on_demand_migration/config.rs        |  83 +++++++--
 .../src}/on_demand_migration/gcs.rs           |   8 +-
 .../src}/on_demand_migration/list_through.rs  |   0
 rustfs/src/on_demand_migration/metrics.rs     | 147 +++++++++++++++
 .../src}/on_demand_migration/mod.rs           |  15 +-
 .../src}/on_demand_migration/native_http.rs   |   2 +-
 .../on_demand_migration/negative_cache.rs     |   0
 .../src}/on_demand_migration/pull.rs          |   4 +-
 .../src}/on_demand_migration/source_client.rs |   6 +-
 .../src}/on_demand_migration/stats.rs         |   0
 rustfs/src/on_demand_migration/storage_api.rs |  28 +++
 .../src}/on_demand_migration/sys.rs           |  51 +++++-
 .../on_demand_migration/test_http_fixture.rs  |   0
 rustfs/src/startup_background.rs              |   7 +-
 rustfs/src/startup_bucket_metadata.rs         |   6 +-
 rustfs/src/storage/storage_api.rs             |  14 +-
 rustfs/src/storage_api.rs                     |  32 +++-
 45 files changed, 619 insertions(+), 540 deletions(-)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/azure.rs (99%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/backend_contract.rs (99%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/backfill.rs (98%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/breaker.rs (100%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/config.rs (94%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/gcs.rs (98%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/list_through.rs (100%)
 create mode 100644 rustfs/src/on_demand_migration/metrics.rs
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/mod.rs (84%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/native_http.rs (99%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/negative_cache.rs (100%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/pull.rs (99%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/source_client.rs (99%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/stats.rs (100%)
 create mode 100644 rustfs/src/on_demand_migration/storage_api.rs
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/sys.rs (96%)
 rename {crates/ecstore/src/bucket => rustfs/src}/on_demand_migration/test_http_fixture.rs (100%)

diff --git a/Cargo.lock b/Cargo.lock
index 76072f5da..c6d279229 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9493,6 +9493,8 @@ dependencies = [
  "atomic_enum",
  "aws-config",
  "aws-sdk-s3",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
  "axum",
  "base64-simd",
  "bytes",
@@ -9506,6 +9508,7 @@ dependencies = [
  "futures",
  "futures-lite",
  "futures-util",
+ "google-cloud-auth",
  "hashbrown 0.17.1",
  "hex-simd",
  "hmac 0.13.0",
@@ -9792,7 +9795,6 @@ dependencies = [
  "path-absolutize",
  "pin-project-lite",
  "proptest",
- "quick-xml",
  "rand 0.10.2",
  "ratelimit",
  "rcgen",
diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml
index 8dbfec7c5..4925b283e 100644
--- a/crates/ecstore/Cargo.toml
+++ b/crates/ecstore/Cargo.toml
@@ -216,7 +216,6 @@ serde_urlencoded.workspace = true
 google-cloud-storage = { workspace = true, optional = true }
 google-cloud-auth = { workspace = true, optional = true }
 faster-hex = { workspace = true }
-quick-xml = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
 
diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs
index e752713ff..b19397a7f 100644
--- a/crates/ecstore/src/api/mod.rs
+++ b/crates/ecstore/src/api/mod.rs
@@ -146,69 +146,23 @@ pub mod bucket {
         };
     }
 
-    pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
-            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
-            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
-            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
-            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
-            SourceLatencySnapshot, source_backend_spec, source_client_spec,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
-            ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig,
-            Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig,
-            ValidationContext,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
-            PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
-            WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
-            idle_guarded_body,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
-            ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
-            MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
-            decode_continuation_token, source_list_plan,
-        };
-        pub mod backfill {
-            pub use crate::bucket::on_demand_migration::backfill::{
-                BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
-                BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
-                BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
-                BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
-                BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
-                StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
-                read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
-            };
-        }
-        pub mod source_client {
-            pub use crate::bucket::on_demand_migration::source_client::{
-                AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError,
-                SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse,
-                SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style,
-            };
-        }
-    }
-
     pub mod metadata_sys {
-        #[cfg(feature = "test-util")]
-        pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
         pub use crate::bucket::metadata_sys::{
-            BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
+            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
             capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
             get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
             get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
             get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
-            get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
-            get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
-            reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
-            update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
-            update_under_transaction_lock,
+            get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
+            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
+            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
+            update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
+            update_quota_if_incarnation, update_under_transaction_lock,
         };
+        #[cfg(feature = "test-util")]
+        pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
     }
 
     pub mod migration {
@@ -251,7 +205,7 @@ pub mod bucket {
     pub mod remote_s3_client {
         pub use crate::bucket::remote_s3_client::{
             PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
-            validate_remote_endpoint,
+            build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
         };
     }
 
@@ -497,9 +451,9 @@ pub mod object {
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
         SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
-        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
-        unregister_object_mutation_hook,
+        ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
+        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
+        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs
index 41b2afdc5..09030ba12 100644
--- a/crates/ecstore/src/bucket/metadata.rs
+++ b/crates/ecstore/src/bucket/metadata.rs
@@ -489,28 +489,17 @@ impl BucketMetadata {
         !self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
     }
 
-    /// Parsed per-bucket durability override, if a valid one is stored.
-    ///
-    /// Absent/empty/unparsable payloads all mean "no override" (the bucket
-    /// follows the global durability mode); a parse failure is logged so a
-    /// corrupted entry cannot silently change fsync behavior.
-    /// Parsed on-demand migration config, if one is stored.
-    ///
-    /// `Ok(None)` means no config (absent or cleared). A stored payload that
-    /// does not parse is an error, never a default: the runtime must not
-    /// pull from a source it cannot describe.
-    pub fn on_demand_migration_config(
-        &self,
-    ) -> std::result::Result<
-        Option<super::on_demand_migration::OnDemandMigrationConfig>,
-        super::on_demand_migration::OnDemandMigrationConfigError,
-    > {
-        if self.on_demand_migration_config_json.is_empty() {
-            return Ok(None);
-        }
-        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
+    /// Opaque application-owned configuration with its persisted update time.
+    /// Empty bytes mean absent or cleared; decoding belongs to the consumer.
+    pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
+        (!self.on_demand_migration_config_json.is_empty()).then_some((
+            self.on_demand_migration_config_json.as_slice(),
+            self.on_demand_migration_config_updated_at,
+        ))
     }
 
+    /// Parsed per-bucket durability override, if a valid one is stored.
+    /// Invalid payloads follow the global mode after logging a parse failure.
     pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
         if self.durability_config_json.is_empty() {
             return None;
@@ -916,13 +905,6 @@ impl BucketMetadata {
                 self.durability_config_updated_at = updated;
             }
             BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
-                // Structural check only (shape, unknown fields); the
-                // deployment-relative rules run in the admin handler with a
-                // `ValidationContext`. A blob this build cannot read must not
-                // be persisted for every later reader to trip over.
-                if !data.is_empty() {
-                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
-                }
                 self.on_demand_migration_config_json = data;
                 self.on_demand_migration_config_updated_at = updated;
             }
@@ -1978,51 +1960,30 @@ mod test {
 
     const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
-    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
-    /// extension entry that round-trips through `update_config` and the
-    /// msgpack codec, clears on delete, and never parses corruption into a
-    /// default.
+    /// The metadata codec preserves application-owned bytes and timestamps.
     #[test]
     fn on_demand_migration_config_round_trips_and_tracks_updates() {
-        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
-
         let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
-
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+        assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("valid config is accepted");
-        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
-
+            .expect("opaque config is accepted");
+        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
+        assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
         let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
         assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(
-            back.on_demand_migration_config_updated_at.unix_timestamp(),
-            bm.on_demand_migration_config_updated_at.unix_timestamp()
-        );
-        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
-
-        // A blob this build cannot read is rejected at the write boundary
-        // rather than persisted for every reader to trip over.
-        let before = bm.on_demand_migration_config_json.clone();
-        assert!(
-            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
-                .is_err()
-        );
-        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
-
-        // Delete clears the entry.
-        let stamped = bm.on_demand_migration_config_updated_at;
+        assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
         bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
         assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
         assert!(bm.on_demand_migration_config_updated_at >= stamped);
-
-        // Corruption that bypassed `update_config` (disk, another writer)
-        // is a typed error, never a default.
-        bm.on_demand_migration_config_json = b"not-json".to_vec();
-        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
+        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
+            .unwrap();
+        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
+        assert_eq!(
+            back.on_demand_migration_config_json, b"not-json",
+            "metadata must not reinterpret application bytes"
+        );
     }
 
     /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
@@ -2034,7 +1995,7 @@ mod test {
         let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
         assert!(bm.on_demand_migration_config_json.is_empty());
         assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
+        assert_eq!(bm.on_demand_migration_config(), None);
 
         bm.default_timestamps();
         assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs
index 4a53927f9..5e30841ad 100644
--- a/crates/ecstore/src/bucket/metadata_sys.rs
+++ b/crates/ecstore/src/bucket/metadata_sys.rs
@@ -19,7 +19,6 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
-use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
@@ -49,6 +48,11 @@ use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};
 use uuid::Uuid;
 
+/// Opaque bucket configuration notifications for application-owned services.
+/// `None` withdraws a configuration; consumers validate nonempty bytes.
+pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime)>) + Send + Sync>;
+pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
+
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
 
 #[cfg(any(test, feature = "test-util"))]
@@ -395,39 +399,20 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
-/// Publish the bucket's on-demand migration config (or its absence) to the
-/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
-///
-/// Called from the same five cache-install paths as
-/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
-/// published as `None`: the runtime must stop pulling for that bucket rather
-/// than keep an older config or guess.
+/// Publish application-owned bytes on every cache install path.
 fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
-        return;
-    };
-    match bm.on_demand_migration_config() {
-        Ok(config) => hook(bucket, config.as_ref()),
-        Err(err) => {
-            warn!(
-                event = "bucket_metadata_parse_failed",
-                component = "ecstore",
-                subsystem = "bucket_metadata",
-                bucket = %bucket,
-                config = "on_demand_migration",
-                error = %err,
-                "Failed to parse bucket metadata config"
-            );
-            hook(bucket, None);
-        }
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(
+            bucket,
+            super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
+            bm.on_demand_migration_config(),
+        );
     }
 }
 
-/// Withdraw a bucket's on-demand migration config when its metadata leaves
-/// the cache.
 fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
-        hook(bucket, None);
+    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
+        hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
     }
 }
 
@@ -1049,15 +1034,24 @@ pub async fn get_durability_config(
 }
 
 /// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. A stored payload that does not parse
-/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
+pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
 
     bucket_meta_sys.get_on_demand_migration_config(bucket).await
 }
 
+/// Resolve opaque configuration from the store's own metadata system.
+pub async fn get_on_demand_migration_config_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+    let sys = bucket_metadata_sys_of(ctx)?;
+    let lock = sys.read().await;
+    lock.get_on_demand_migration_config(bucket).await
+}
+
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2579,29 +2573,27 @@ impl BucketMetadataSys {
     }
 
     /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(
-        &self,
-        bucket: &str,
-    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
+    pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        let config = bm.on_demand_migration_config().map_err(Error::other)?;
-        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
+        Ok(bm
+            .on_demand_migration_config()
+            .map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
     }
 }
 
 /// Test-only fixture shared with sibling modules (e.g. the quota checker
 /// tests): a 4-disk `ECStore` on an isolated instance context, so tests
 /// exercising the metadata system never touch ambient process state.
-#[cfg(test)]
-pub(crate) mod test_support {
+#[cfg(any(test, feature = "test-util"))]
+pub mod test_support {
     use super::*;
     use crate::disk::endpoint::Endpoint;
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::store::init_local_disks_with_instance_ctx;
 
-    pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+    pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
         let mut dirs = Vec::with_capacity(4);
         let mut endpoints = Vec::with_capacity(4);
         for disk_idx in 0..4 {
@@ -4387,17 +4379,21 @@ mod tests {
 
     /// Every `(bucket, config)` the recording hook has seen. Tests filter by
     /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
+    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<(Vec<u8>, OffsetDateTime)>)>> = std::sync::Mutex::new(Vec::new());
 
     fn install_recording_odm_hook() {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
-            Box::new(|bucket, config| {
-                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
+        BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
+            Box::new(|bucket, config_file, config| {
+                assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
+                ODM_HOOK_CALLS
+                    .lock()
+                    .unwrap()
+                    .push((bucket.to_string(), config.map(|(bytes, stamp)| (bytes.to_vec(), stamp))));
             })
         });
     }
 
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
+    fn odm_hook_calls(bucket: &str) -> Vec<Option<(Vec<u8>, OffsetDateTime)>> {
         ODM_HOOK_CALLS
             .lock()
             .unwrap()
@@ -4407,54 +4403,6 @@ mod tests {
             .collect()
     }
 
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
-
     /// rustfs/backlog#2148: the publish hook fires on every path that
     /// installs bucket metadata into the cache (set, initial load, peer
     /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
@@ -4468,11 +4416,15 @@ mod tests {
         for dir in &dirs {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
         }
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
+
         let expect_publish = |before: usize, label: &str| {
             let calls = odm_hook_calls(bucket);
             assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
+            assert_eq!(
+                calls.last().unwrap().as_ref().map(|(bytes, _)| bytes.as_slice()),
+                Some(ODM_JSON),
+                "{label} must publish the stored bytes"
+            );
         };
 
         // set (via persist_new_and_set, which installs through `set`).
@@ -4518,14 +4470,18 @@ mod tests {
         assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
         assert_eq!(calls.last().unwrap(), &None);
 
-        // A corrupt payload is withdrawn, never published as a config.
+        // Opaque bytes reach the application even if they are not valid JSON.
         let mut corrupt = BucketMetadata::new(bucket);
         corrupt.on_demand_migration_config_json = b"not-json".to_vec();
         let before = odm_hook_calls(bucket).len();
         lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
         let calls = odm_hook_calls(bucket);
         assert_eq!(calls.len(), before + 1);
-        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
+        assert_eq!(
+            calls.last().unwrap().as_ref().map(|(bytes, _)| bytes.as_slice()),
+            Some(b"not-json".as_slice()),
+            "the application validates opaque config bytes"
+        );
     }
 
     #[tokio::test]
diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs
index e93419cf2..da42fc43e 100644
--- a/crates/ecstore/src/bucket/mod.rs
+++ b/crates/ecstore/src/bucket/mod.rs
@@ -26,7 +26,6 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
-pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;
diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs
index 6389aa67e..1385cea80 100644
--- a/crates/ecstore/src/bucket/remote_s3_client.rs
+++ b/crates/ecstore/src/bucket/remote_s3_client.rs
@@ -283,9 +283,7 @@ impl Intercept for UserAgentSuffixInterceptor {
 
 /// Builds the SDK config for `spec` without finalizing it, so callers can add
 /// interceptors or (in tests) swap the HTTP client before `build()`.
-pub(crate) async fn build_remote_s3_config(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
+pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
     let Some(credentials) = &spec.credentials else {
         return Err(RemoteS3ClientError::MissingCredentials);
     };
@@ -525,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
     Ok(())
 }
 
-pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
+pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
     validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
 }
 
diff --git a/crates/obs/src/metrics/mod.rs b/crates/obs/src/metrics/mod.rs
index aea11381d..009cc94f2 100644
--- a/crates/obs/src/metrics/mod.rs
+++ b/crates/obs/src/metrics/mod.rs
@@ -38,3 +38,4 @@ pub(crate) use storage_api::metrics::{
     obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
     obs_transition_state_handle,
 };
+pub use storage_api::register_on_demand_migration_metrics_source;
diff --git a/crates/obs/src/metrics/storage_api.rs b/crates/obs/src/metrics/storage_api.rs
index e69cc64d8..44b1f0834 100644
--- a/crates/obs/src/metrics/storage_api.rs
+++ b/crates/obs/src/metrics/storage_api.rs
@@ -17,13 +17,6 @@ use std::time::Duration;
 
 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
-use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{
-    BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
-};
-use rustfs_ecstore::api::bucket::on_demand_migration::{
-    BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
-    OnDemandMigrationSys as SourceOnDemandMigrationSys,
-};
 use rustfs_ecstore::api::bucket::replication::{
     BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
     MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
@@ -44,9 +37,7 @@ pub(crate) use rustfs_ecstore::api::runtime::{
 pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
 use rustfs_storage_api as storage_contracts;
 
-use crate::metrics::collectors::{
-    OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
-};
+use crate::metrics::collectors::{OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBucketStats};
 
 #[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
@@ -465,70 +456,37 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
     buckets
 }
 
-fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
-    let stats = snapshot.stats;
-    OnDemandMigrationBucketStats {
-        bucket: snapshot.bucket,
-        requests_total: stats.requests_total,
-        pulled_bytes_total: stats.pulled_bytes_total,
-        pulled_objects_total: stats.pulled_objects_total,
-        pull_failures_total: stats.pull_failures_total,
-        inflight_pulls: stats.inflight_pulls,
-        queue_depth: stats.queue_depth,
-        source_latency_buckets: stats
-            .source_latency
-            .buckets
-            .into_iter()
-            .map(|bucket| (bucket.le_ms, bucket.count))
-            .collect(),
-        source_latency_count: stats.source_latency.count,
-        source_latency_sum_ms: stats.source_latency.sum_ms,
-        breaker_state: match stats.breaker_state {
-            SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
-            SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
-            SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
-        },
-    }
+struct OnDemandMigrationMetricsSource {
+    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
+    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
 }
 
-/// Every bucket with live on-demand migration state on this node, sorted by
-/// name. Empty while the module switch is off.
-pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
-    SourceOnDemandMigrationSys::get()
-        .snapshot()
-        .into_iter()
-        .map(on_demand_migration_stats_from_snapshot)
-        .collect()
-}
+static ON_DEMAND_MIGRATION_METRICS_SOURCE: std::sync::OnceLock<OnDemandMigrationMetricsSource> = std::sync::OnceLock::new();
 
-fn on_demand_migration_backfill_stats_from_checkpoint(
-    bucket: String,
-    checkpoint: SourceBackfillCheckpoint,
-) -> OdmBackfillBucketStats {
-    OdmBackfillBucketStats {
-        bucket,
-        state: checkpoint.state.as_str().to_string(),
-        listed: checkpoint.listed,
-        enqueued: checkpoint.enqueued,
-        pulled: checkpoint.pulled,
-        skipped_existing: checkpoint.skipped_existing,
-        failed: checkpoint.failed,
-        bytes: checkpoint.bytes,
-    }
-}
-
-/// Backfill jobs running on this node, sorted by bucket. Empty until the
-/// runner is installed, and empty again once a job finishes: the series are
-/// per-node job progress, not a cluster-wide history.
-pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
-    let buckets = source_global_backfill_runner()
-        .map(|runner| {
-            runner
-                .local_job_snapshots()
-                .into_iter()
-                .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
-                .collect()
+/// Register the application-owned ODM snapshots before starting the collector.
+pub fn register_on_demand_migration_metrics_source(
+    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
+    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
+) -> bool {
+    ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .set(OnDemandMigrationMetricsSource {
+            snapshot,
+            backfill_snapshot,
         })
+        .is_ok()
+}
+
+pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
+    ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .get()
+        .map(|source| (source.snapshot)())
+        .unwrap_or_default()
+}
+
+pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
+    let buckets = ON_DEMAND_MIGRATION_METRICS_SOURCE
+        .get()
+        .map(|source| (source.backfill_snapshot)())
         .unwrap_or_default();
     OdmBackfillRuntimeStats { server, buckets }
 }
@@ -580,6 +538,31 @@ pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_ra
 mod tests {
     use super::*;
 
+    #[test]
+    fn on_demand_migration_callbacks_supply_runtime_snapshots() {
+        assert!(register_on_demand_migration_metrics_source(
+            || vec![OnDemandMigrationBucketStats {
+                bucket: "configured".into(),
+                pulled_bytes_total: 4096,
+                ..Default::default()
+            }],
+            || vec![OdmBackfillBucketStats {
+                bucket: "backfill".into(),
+                pulled: 3,
+                ..Default::default()
+            }],
+        ));
+        let snapshot = obs_on_demand_migration_snapshot();
+        assert_eq!(snapshot.len(), 1);
+        assert_eq!(snapshot[0].bucket, "configured");
+        assert_eq!(snapshot[0].pulled_bytes_total, 4096);
+        let backfill = obs_on_demand_migration_backfill_snapshot("node-a".into());
+        assert_eq!(backfill.server, "node-a");
+        assert_eq!(backfill.buckets.len(), 1);
+        assert_eq!(backfill.buckets[0].bucket, "backfill");
+        assert_eq!(backfill.buckets[0].pulled, 3);
+    }
+
     #[test]
     fn obs_replication_numeric_conversions_floor_negative_values() {
         assert_eq!(i64_to_u64_floor_zero(-1), 0);
@@ -772,51 +755,6 @@ mod tests {
         assert_eq!(snapshot.mrf_last_flush_duration_millis, 4);
     }
 
-    #[test]
-    fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
-        // Built from JSON: the snapshot's timestamps use `time`, which obs does not depend on.
-        let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
-            "bucket": "photos",
-            "provider": "minio",
-            "endpoint_host": "source.example.com",
-            "applied_at": "2026-09-02T10:00:00Z",
-            "client_error": null,
-            "negative_cache_entries": 0,
-            "inflight_keys": 1,
-            "max_concurrent_pulls": 8,
-            "stats": {
-                "requests_total": {"get": {"source_hit": 2}},
-                "pulled_bytes_total": 4096,
-                "pulled_objects_total": {"inline": 1},
-                "pull_failures_total": {"source_timeout": 1},
-                "inflight_pulls": 1,
-                "queue_depth": 2,
-                "source_latency": {
-                    "buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
-                    "count": 3,
-                    "sum_ms": 90753
-                },
-                "last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
-                "breaker_state": "open"
-            }
-        }))
-        .expect("runtime snapshot decodes");
-
-        let stats = on_demand_migration_stats_from_snapshot(snapshot);
-
-        assert_eq!(stats.bucket, "photos");
-        assert_eq!(stats.requests_total["get"]["source_hit"], 2);
-        assert_eq!(stats.pulled_bytes_total, 4096);
-        assert_eq!(stats.pulled_objects_total["inline"], 1);
-        assert_eq!(stats.pull_failures_total["source_timeout"], 1);
-        assert_eq!(stats.inflight_pulls, 1);
-        assert_eq!(stats.queue_depth, 2);
-        assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
-        assert_eq!(stats.source_latency_count, 3);
-        assert_eq!(stats.source_latency_sum_ms, 90_753);
-        assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
-    }
-
     #[test]
     fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
         let snapshot = bucket_replication_stats_snapshot_from_parts(
diff --git a/docs/architecture/background-services-inventory.md b/docs/architecture/background-services-inventory.md
index 772a761c0..01b612a3a 100644
--- a/docs/architecture/background-services-inventory.md
+++ b/docs/architecture/background-services-inventory.md
@@ -13,8 +13,8 @@ Operator-facing behaviour, configuration, and troubleshooting for these services
 
 | Service | Desired source | Current-status inputs | Status surface | Side effects |
 |---|---|---|---|---|
-| Write-back pull pipeline (`crates/ecstore/src/bucket/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `crates/ecstore/src/bucket/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
-| Backfill job (module under `crates/ecstore/src/bucket/on_demand_migration/`, rustfs/backlog#2159 — not yet in the tree) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
-| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159 — not yet in the tree) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |
+| Write-back pull pipeline (`rustfs/src/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `rustfs/src/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
+| Backfill job (module under `rustfs/src/on_demand_migration/`, rustfs/backlog#2159) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
+| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |
 
 The pull pipeline has no separate loop of its own: a bucket's queue dispatcher starts lazily on the first background pull and is cancelled when the bucket's state is rebuilt or removed, and each inline pull commits in a task that outlives its request so a client disconnect cannot truncate the stored object. Neither the switch nor the config is re-read by the workers: the bucket-metadata publish hook rebuilds the state, which is the only desired-state path.
diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md
index fb94ed0aa..6fdca19ab 100644
--- a/docs/architecture/crate-boundaries.md
+++ b/docs/architecture/crate-boundaries.md
@@ -84,3 +84,28 @@ The server-config model (`Config`, `KV`, `KVS`) and the global server-config sna
 ## Required Architecture Documents
 
 The guard requires the documents and section headings listed in its `require_source_contains` entries (`scripts/check_architecture_migration_rules.sh`); the directory index is [README.md](README.md).
+
+## On-Demand Migration Service
+
+`rustfs/src/on_demand_migration/` owns source clients, pull scheduling, list
+merging, runtime state and backfill orchestration. Its `storage_api.rs` is the
+only ECStore facade boundary. Object write-back still enters the application's
+internal PUT and multipart use cases, including the atomic create-only commit,
+delete-marker protection, encryption, quota and notification rules.
+
+ECStore stores the existing ODM bytes and update timestamp without interpreting
+the JSON. Every metadata cache install or removal publishes those bytes through
+`BUCKET_CONFIG_PUBLISH_HOOK`; the application decodes them and synchronously
+withdraws corrupt configurations. Configuration writes validate structure and
+deployment constraints in the admin use case before the incarnation-fenced
+metadata update. Backfill reads metadata from its store's instance context and
+preserves the checkpoint ETag compare-and-set, lease and tail-drained writes.
+
+Observability owns its metric DTOs and accepts application snapshot callbacks;
+it does not depend on the ODM runtime. The application registers both bucket
+and backfill snapshots during startup, before metadata and metric collection.
+
+This boundary does not change `.metadata.bin`, the ODM wire format or the
+backfill checkpoint format. An older binary may still discard unknown metadata
+fields when it rewrites a bucket; service relocation does not make mixed-version
+configuration writes or rollback preserve ODM configuration.
diff --git a/docs/architecture/ecstore-api-facade-inventory.md b/docs/architecture/ecstore-api-facade-inventory.md
index 7fbd79c52..71b9dcbb8 100644
--- a/docs/architecture/ecstore-api-facade-inventory.md
+++ b/docs/architecture/ecstore-api-facade-inventory.md
@@ -63,3 +63,10 @@ Lifecycle, replication, and `SetDisks` split blockers, extracted contracts, and
 4. Do not replace `SetDisks` with multiple runtime structs in one change; move one operation family only after contracts and focused tests exist.
 5. Remove or narrow one facade group per change so rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility.
 6. Keep `api::bucket`, `api::config`, `api::disk`, and `api::tier` on explicit submodules and symbol lists; do not restore `pub use crate::<owner>::{...}` whole-module passthroughs for those groups.
+
+### On-Demand Migration
+
+`rustfs/src/on_demand_migration/storage_api.rs` owns the service's storage facade
+imports: opaque bucket configuration, shared remote S3 client construction,
+namespace locking, object options and metadata-object persistence. ODM types
+are owned by the application and are no longer exported through ECStore.
diff --git a/docs/architecture/remote-credential-sealing-adr.md b/docs/architecture/remote-credential-sealing-adr.md
index 618011fec..4a21029b3 100644
--- a/docs/architecture/remote-credential-sealing-adr.md
+++ b/docs/architecture/remote-credential-sealing-adr.md
@@ -1,7 +1,7 @@
 # Remote Credential Sealing ADR
 
 **Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another.
-**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `crates/ecstore/src/bucket/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
+**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `rustfs/src/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
 
 ## Recommendation
 
@@ -37,7 +37,7 @@ Two of the three are not files at all. `bucket-targets.json` and `on-demand-migr
 | Store | Reached as | Actually persisted at | Written by | Container |
 |---|---|---|---|---|
 | Replication and ILM targets | `BUCKET_TARGETS_FILE` | `BucketMetadata::bucket_targets_config_json`, msgpack field `BucketTargetsConfigJSON` | `BucketMetadata::update_config`, then `BucketMetadata::save_with_store`; `crates/ecstore/src/bucket/metadata_sys.rs` serializes the update under a transaction lock | `{BUCKET_META_PREFIX}/{bucket}/{BUCKET_METADATA_FILE}` in `RUSTFS_META_BUCKET` (`crates/ecstore/src/disk/mod.rs`) |
-| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; `update_config` additionally refuses a blob this build cannot parse | same blob as above |
+| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; the application validates structure and deployment constraints before persistence | same blob as above |
 | Remote tiers | `TIER_CONFIG_FILE` | its own object, a four-byte `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header followed by an `rmp_serde` payload of `ExternalTierConfigMgr` | `TierConfigMgr` through `encode_external_tiering_config_blob`, under `tier_config_lock_path` | `tier_config_path` under `CONFIG_PREFIX` in `RUSTFS_META_BUCKET` |
 
 The consequence of the first two sharing a blob is that any change to how that blob parses has a blast radius covering policy, lifecycle, versioning, object lock and everything else in `BucketMetadata` — not just credentials.
@@ -48,7 +48,7 @@ Three things hold the line today, and all three keep working whether or not seal
 
 - **The reserved bucket.** `RUSTFS_META_BUCKET` is `.rustfs.sys`; `is_reserved_or_invalid_bucket` keeps it off the S3 surface, and the admin inspect archive in `rustfs/src/admin/handlers/inspect_archive.rs` runs its request through a strict bucket-name check that a dot-prefixed reserved name does not pass.
 - **Admin authorization** on every route that can read or write one of the three configurations.
-- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
+- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `rustfs/src/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
 
 So no API returns a stored secret. The bytes are reachable by reading the drives, and that is the boundary sealing is proposed to move.
 
@@ -74,7 +74,7 @@ The envelope deliberately does **not** carry its own scope. A scope read out of
 
 ## Why a hook instead of a dependency
 
-`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
+`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `BUCKET_CONFIG_PUBLISH_HOOK` in `crates/ecstore/src/bucket/metadata_sys.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
 
 ## Compatibility, per store, because the three differ
 
diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md
index e9beb27d7..2932ecb3f 100644
--- a/docs/operations/on-demand-migration.md
+++ b/docs/operations/on-demand-migration.md
@@ -1,7 +1,7 @@
 # On-Demand Migration
 
 **Use this when:** you are moving an existing S3-compatible bucket into RustFS without a stop-the-world copy, or you are debugging a bucket that serves reads from an external source (424 `SourceUnavailable`, an open circuit breaker, missing pulled objects, a source 403).
-**Source of truth:** `crates/ecstore/src/bucket/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.
+**Source of truth:** `rustfs/src/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.
 
 On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** to a local RustFS bucket. When a client GETs a key that does not exist locally, RustFS fetches it from the source, streams it to the client, and stores it locally in the same pass; every later read is served locally. It is a pull-style, lazy migration path — the RustFS equivalent of Cloudflare R2 Sippy, Tigris shadow buckets, and Alibaba Cloud OSS / Tencent COS mirror-back-to-origin.
 
@@ -111,7 +111,7 @@ Read-through only migrates what clients touch. The background backfill job walks
 
 ## Configuration reference
 
-The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `crates/ecstore/src/bucket/on_demand_migration/config.rs`.
+The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `rustfs/src/on_demand_migration/config.rs`.
 
 | Field | Type | Default | Bounds / rules |
 |---|---|---|---|
@@ -168,7 +168,7 @@ Validation also rejects two shapes outright: a source whose endpoint and bucket
 | `azure` | Optional; derived as `https://<account>.blob.core.windows.net` | Native Blob REST, not S3 | Unused; write `auto` | Needs `source.azure`; the container is `source.bucket`. Reads need `Read` on the blob and `List` on the container, plus `Tags` when `policy.copy_tags` is on | None yet: no interop job covers Azure |
 | `gcs_native` | Optional; derived as `https://storage.googleapis.com` | Native GCS API, not S3 | Unused; write `auto` | Needs `source.gcs`. Reads use the XML API for objects and `objects.list` for listings, both with an OAuth token minted from the service-account key; the key needs `storage.objects.get` and `storage.objects.list` | None yet: no interop job covers native GCS |
 
-Every backend answers the same trait contract, pinned by `backend_contract.rs` in `crates/ecstore/src/bucket/on_demand_migration/`, and the three differences that contract allows are the ones documented here.
+Every backend answers the same trait contract, pinned by `backend_contract.rs` in `rustfs/src/on_demand_migration/`, and the three differences that contract allows are the ones documented here.
 
 `azure` differs in two of them. Its ETag is a concurrency token rather than a digest of the bytes, so it is stored as `odm-source-etag` provenance and never used as the expected MD5 of a pulled object — the write-back integrity check falls back to the local digest. And its listing paginates only with an opaque marker: there is no "start after this key" form, so a caller that asks for one gets `Unsupported` instead of a listing that silently starts over.
 
diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml
index 063ce400e..03b6b378e 100644
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -58,7 +58,7 @@ required-features = ["swift"]
 
 [features]
 default = ["ftps", "webdav", "gcs"]
-gcs = ["rustfs-ecstore/gcs"]
+gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]
 swift = ["rustfs-protocols/swift"]
@@ -288,6 +288,9 @@ reqwest = { workspace = true, features = ["json", "stream"] }
 socket2 = { workspace = true, features = ["all"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
 tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
+aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
+aws-smithy-types = { workspace = true }
+google-cloud-auth = { workspace = true, optional = true }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "time"] }
diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs
index d8bc9d713..3e331b26f 100644
--- a/rustfs/src/admin/handlers/on_demand_migration.rs
+++ b/rustfs/src/admin/handlers/on_demand_migration.rs
@@ -38,16 +38,6 @@ use crate::admin::runtime_sources::{
 };
 use crate::admin::storage_api::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
 use crate::admin::storage_api::bucket::metadata_sys;
-use crate::admin::storage_api::bucket::on_demand_migration::backfill::{
-    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
-};
-use crate::admin::storage_api::bucket::on_demand_migration::source_client::{
-    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
-};
-use crate::admin::storage_api::bucket::on_demand_migration::{
-    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
-    source_backend_spec,
-};
 use crate::admin::storage_api::bucket::remote_s3_client::{
     PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
 };
@@ -58,6 +48,16 @@ use crate::admin::utils::{extract_query_params, read_compatible_admin_body};
 use crate::error::ApiError;
 use crate::license::license_check;
 use crate::module_switches::{ENV_ON_DEMAND_MIGRATION_ENABLED, on_demand_migration_enabled_from_env};
+use crate::on_demand_migration::backfill::{
+    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
+};
+use crate::on_demand_migration::source_client::{
+    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
+};
+use crate::on_demand_migration::{
+    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
+    source_backend_spec,
+};
 use crate::server::ADMIN_PREFIX;
 use hyper::{Method, StatusCode};
 use matchit::Params;
@@ -580,7 +580,7 @@ async fn validate_config(bucket: &str, config: &OnDemandMigrationConfig) -> S3Re
 }
 
 fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
-    use crate::admin::storage_api::bucket::on_demand_migration::Provider;
+    use crate::on_demand_migration::Provider;
     match config.source.provider {
         Provider::S3 => SourceProvider::S3,
         Provider::Aws => SourceProvider::Aws,
@@ -787,7 +787,7 @@ impl Operation for GetBucketOnDemandMigrationHandler {
         let bucket = bucket_from_params(&params)?;
         let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
 
-        let Some((config, updated_at)) = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
+        let Some((config, updated_at)) = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
             admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
         })?
         else {
@@ -846,7 +846,7 @@ impl Operation for GetBucketOnDemandMigrationStatusHandler {
         let bucket = bucket_from_params(&params)?;
         let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
 
-        let config = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
+        let config = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
             admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
         })?;
         let runtime = OnDemandMigrationSys::get().bucket_snapshot(&bucket);
@@ -1763,6 +1763,19 @@ mod store_tests {
                 assert_eq!(body["inflight_pulls"], Value::from(0));
                 assert_eq!(body["queue_depth"], Value::from(0));
 
+                // A malformed replacement cannot overwrite the saved source.
+                let before = metadata_sys::get(BUCKET).await.expect("saved metadata");
+                for invalid in [b"not-json".to_vec(), br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec()] {
+                    let err = SetBucketOnDemandMigrationHandler {}
+                        .call(root_request(Method::PUT, config_uri(""), invalid), bucket_params(&router))
+                        .await
+                        .expect_err("malformed replacement must be rejected");
+                    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+                    let after = metadata_sys::get(BUCKET).await.expect("saved metadata remains readable");
+                    assert_eq!(after.on_demand_migration_config_json, before.on_demand_migration_config_json);
+                    assert_eq!(after.on_demand_migration_config_updated_at, before.on_demand_migration_config_updated_at);
+                }
+
                 // The peer fan-out ran: the single unreachable peer is reported.
                 let context = crate::admin::runtime_sources::current_app_context();
                 let err = reload_peers(context.as_deref(), BUCKET)
diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs
index ea7ae56b5..01caaa95f 100644
--- a/rustfs/src/admin/storage_api.rs
+++ b/rustfs/src/admin/storage_api.rs
@@ -20,8 +20,8 @@ use time::OffsetDateTime;
 
 mod ecstore_bucket {
     pub(crate) use crate::storage::storage_api::ecstore_bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, on_demand_migration, quota,
-        remote_s3_client, replication, target, utils, versioning, versioning_sys,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
+        replication, target, utils, versioning, versioning_sys,
     };
 }
 
@@ -284,35 +284,6 @@ pub(crate) mod durability {
     pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
 }
 
-pub(crate) mod on_demand_migration {
-    pub(crate) type OdmBucketSnapshot = super::ecstore_bucket::on_demand_migration::OdmBucketSnapshot;
-    pub(crate) type OnDemandMigrationConfig = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfig;
-    pub(crate) type OnDemandMigrationConfigError = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfigError;
-    pub(crate) type OnDemandMigrationSys = super::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
-    pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle;
-    pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider;
-    pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>;
-    pub(crate) use super::ecstore_bucket::on_demand_migration::source_backend_spec;
-
-    pub(crate) mod backfill {
-        pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint;
-        pub(crate) type BackfillError = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillError;
-        pub(crate) type BackfillRequest = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillRequest;
-        pub(crate) type BackfillState = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillState;
-        pub(crate) type SkipExisting = super::super::ecstore_bucket::on_demand_migration::backfill::SkipExisting;
-        pub(crate) use super::super::ecstore_bucket::on_demand_migration::backfill::global_backfill_runner;
-    }
-
-    pub(crate) mod source_client {
-        pub(crate) type SourceClient = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClient;
-        pub(crate) type SourceClientSpec = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClientSpec;
-        pub(crate) type SourceError = super::super::ecstore_bucket::on_demand_migration::source_client::SourceError;
-        pub(crate) type SourceProbe = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProbe;
-        pub(crate) type SourceProvider = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProvider;
-        pub(crate) type SourceTimeouts = super::super::ecstore_bucket::on_demand_migration::source_client::SourceTimeouts;
-    }
-}
-
 pub(crate) mod remote_s3_client {
     pub(crate) type PathStyle = super::ecstore_bucket::remote_s3_client::PathStyle;
     pub(crate) type RemoteCredentials = super::ecstore_bucket::remote_s3_client::RemoteCredentials;
@@ -455,12 +426,6 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
     }
 
-    pub(crate) async fn get_on_demand_migration_config(
-        bucket: &str,
-    ) -> Result<Option<(super::on_demand_migration::OnDemandMigrationConfig, OffsetDateTime)>> {
-        super::ecstore_bucket::metadata_sys::get_on_demand_migration_config(bucket).await
-    }
-
     pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
     }
@@ -913,7 +878,6 @@ pub(crate) mod bucket {
     pub(crate) use super::lifecycle;
     pub(crate) use super::metadata;
     pub(crate) use super::metadata_sys;
-    pub(crate) use super::on_demand_migration;
     pub(crate) use super::quota;
     pub(crate) use super::remote_s3_client;
     pub(crate) use super::replication;
diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index ce09fc990..c385fdb9c 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -25,11 +25,6 @@
 use super::storage_api::bucket_usecase::ECStore;
 use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
 use super::storage_api::bucket_usecase::StorageObjectOptions;
-use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
-    BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
-    MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
-    SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
-};
 use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
 use super::storage_api::bucket_usecase::contract::list::{ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _};
 use super::storage_api::bucket_usecase::contract::object::ObjectOperations as _;
@@ -37,6 +32,11 @@ use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Result};
 use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params;
 use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class};
 use crate::error::ApiError;
+use crate::on_demand_migration::{
+    BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
+    MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
+    SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
+};
 use futures::StreamExt;
 use http::HeaderMap;
 use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header};
@@ -443,14 +443,14 @@ mod tests {
     use super::*;
     use crate::app::bucket_usecase::DefaultBucketUsecase;
     use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
-    use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
-        FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
-        SourceCredentials, TlsConfig,
-    };
     use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
     use crate::app::storage_api::test::StoragePutObjReader;
     use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
     use crate::app::storage_api::test::contract::object::ObjectIO as _;
+    use crate::on_demand_migration::{
+        FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
+        SourceCredentials, TlsConfig,
+    };
     use s3s::dto::ListObjectsInput;
     use std::time::Duration;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs
index 65039350c..591c4b4c6 100644
--- a/rustfs/src/app/object/get.rs
+++ b/rustfs/src/app/object/get.rs
@@ -15,11 +15,11 @@
 //! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning.
 
 use super::*;
-use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+use crate::on_demand_migration::WriteBackBody;
+use crate::on_demand_migration::{
     BucketOdmState, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, PullError, PullLeader, PullOutcome, PullReason, PullSlot,
     RangeGetPolicy, SourceBody, SourceClient, SourceError, SourceGet, SourceHead, commit_inline, idle_guarded_body,
 };
-use crate::app::storage_api::object_usecase::on_demand_migration::WriteBackBody;
 use rustfs_rio::{TeeOptions, TeePrimary, tee_reader_with_options};
 use tokio_stream::wrappers::ReceiverStream;
 
@@ -4798,11 +4798,11 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
 #[cfg(test)]
 mod on_demand_migration_tests {
     use super::*;
-    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+    use crate::on_demand_migration::{
         BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
         Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
     };
-    use crate::app::storage_api::object_usecase::on_demand_migration::{
+    use crate::on_demand_migration::{
         LocalObject, OdmWriteBack, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
     };
     use async_trait::async_trait;
diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs
index 6d877af8a..7cb38bbbc 100644
--- a/rustfs/src/app/object/head.rs
+++ b/rustfs/src/app/object/head.rs
@@ -15,7 +15,7 @@
 //! HeadObject path.
 
 use super::*;
-use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+use crate::on_demand_migration::{
     BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead,
 };
 
@@ -638,7 +638,7 @@ impl DefaultObjectUsecase {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
+    use crate::on_demand_migration::{
         BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
         Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
     };
@@ -739,11 +739,9 @@ mod tests {
             user_metadata: HashMap::from([("owner".to_string(), "alice".to_string())]),
             version_id: Some("v1".to_string()),
             storage_class: Some("STANDARD_IA".to_string()),
-            sse: Some(
-                crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse::Kms {
-                    key_id: Some("key-1".to_string()),
-                },
-            ),
+            sse: Some(crate::on_demand_migration::source_client::SourceSse::Kms {
+                key_id: Some("key-1".to_string()),
+            }),
             is_multipart_etag: true,
             etag_is_opaque: false,
         }
diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs
index 7edea71f8..50e87b5cf 100644
--- a/rustfs/src/app/object/on_demand_migration_put.rs
+++ b/rustfs/src/app/object/on_demand_migration_put.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 //! On-demand migration write-back (rustfs/backlog#2153): the app-layer
-//! [`OdmWriteBack`] the ecstore pull pipeline stores source objects with.
+//! [`OdmWriteBack`] the migration service stores source objects with.
 //!
 //! Every write goes through the internal put entry points, so a pulled
 //! object is indistinguishable from a client PUT: bucket default SSE, quota,
@@ -34,7 +34,7 @@
 use super::*;
 
 use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
-use crate::app::storage_api::object_usecase::on_demand_migration::{
+use crate::on_demand_migration::{
     LocalObject, OdmWriteBack, SourceHead, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
     is_multipart_etag,
 };
@@ -297,7 +297,6 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
 mod tests {
     use super::*;
     use crate::app::storage_api::multipart_usecase::contract::multipart::MultipartOperations as _;
-    use crate::app::storage_api::object_usecase::on_demand_migration::{PullFailureReason, SourceSse};
     use crate::app::storage_api::s3::{
         BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
         ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
@@ -306,6 +305,7 @@ mod tests {
     use crate::app::storage_api::test::bucket::utils::serialize;
     use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
     use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
+    use crate::on_demand_migration::{PullFailureReason, SourceSse};
     use http::Method;
     use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, contains_key_str, get_str};
     use sha2::{Digest as Sha256Digest, Sha256};
diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs
index 8f0808956..8df039d14 100644
--- a/rustfs/src/app/object/shared.rs
+++ b/rustfs/src/app/object/shared.rs
@@ -15,9 +15,7 @@
 //! Cross-cutting helpers shared by the object use-case modules.
 
 use super::*;
-use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
-    OdmStateError, PolicyConfig, SourceErrorPolicy, SourceHead,
-};
+use crate::on_demand_migration::{OdmStateError, PolicyConfig, SourceErrorPolicy, SourceHead};
 
 pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-current-version-id";
 
diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs
index 383df36a1..6c810e213 100644
--- a/rustfs/src/app/storage_api.rs
+++ b/rustfs/src/app/storage_api.rs
@@ -628,26 +628,6 @@ pub(crate) mod bucket {
         }
     }
 
-    pub(crate) mod on_demand_migration {
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::{
-            SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
-        };
-        #[cfg(test)]
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
-            BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig,
-            PathStyle, Provider, SourceConfig, SourceCredentials, TlsConfig,
-        };
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
-            BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OnDemandMigrationSys, PolicyConfig,
-            PullError, PullLeader, PullOutcome, PullReason, PullSlot, RangeGetPolicy, SourceBody, SourceErrorPolicy,
-            commit_inline, idle_guarded_body,
-        };
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
-            ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
-            MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
-        };
-    }
-
     pub(crate) mod policy_sys {
         pub(crate) type PolicySys = crate::storage::storage_api::ecstore_bucket::policy_sys::PolicySys;
     }
@@ -1181,19 +1161,6 @@ pub(crate) mod bucket_usecase {
 pub(crate) mod object_usecase {
     pub(crate) use super::storage_contracts::BUCKET_LIFECYCLE_LOCK_OBJECT;
 
-    pub(crate) mod on_demand_migration {
-        #[cfg(test)]
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::PullFailureReason;
-        #[cfg(test)]
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse;
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::{
-            SourceHead, is_multipart_etag,
-        };
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
-            LocalObject, OdmWriteBack, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
-        };
-    }
-
     pub(crate) mod object_cache {
         #[cfg(test)]
         pub(crate) use crate::storage::storage_api::ecstore_object::GetObjectBodySource;
diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs
index 95b0c0fba..77c387412 100644
--- a/rustfs/src/lib.rs
+++ b/rustfs/src/lib.rs
@@ -93,6 +93,7 @@ pub(crate) mod kms_rekey;
 pub mod license;
 pub mod memory_observability;
 pub mod module_switches;
+pub mod on_demand_migration;
 pub mod profiling;
 #[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
 pub mod protocols;
diff --git a/crates/ecstore/src/bucket/on_demand_migration/azure.rs b/rustfs/src/on_demand_migration/azure.rs
similarity index 99%
rename from crates/ecstore/src/bucket/on_demand_migration/azure.rs
rename to rustfs/src/on_demand_migration/azure.rs
index 08e687797..25d4cb0a1 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/azure.rs
+++ b/rustfs/src/on_demand_migration/azure.rs
@@ -40,8 +40,8 @@ use super::source_client::{
     AzureAuth, AzureSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
     SourceTimeouts, range_header_value,
 };
-use crate::bucket::remote_s3_client::RemoteS3ClientError;
-use crate::storage_api_contracts::range::HTTPRangeSpec;
+use super::storage_api::HTTPRangeSpec;
+use super::storage_api::remote_s3_client::RemoteS3ClientError;
 use hmac::{Hmac, Mac, digest::KeyInit};
 use http::{HeaderMap, HeaderValue, Method};
 use quick_xml::Reader;
@@ -549,9 +549,9 @@ fn leaf_text(reader: &mut Reader<&[u8]>, end: quick_xml::name::QName<'_>) -> Res
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::bucket::on_demand_migration::source_client::SourceError;
-    use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
+    use crate::on_demand_migration::source_client::SourceError;
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
 
     const LIST_PAGE: &str = r#"<?xml version="1.0" encoding="utf-8"?>
 <EnumerationResults ServiceEndpoint="https://acct.blob.core.windows.net/" ContainerName="legacy">
diff --git a/crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs b/rustfs/src/on_demand_migration/backend_contract.rs
similarity index 99%
rename from crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs
rename to rustfs/src/on_demand_migration/backend_contract.rs
index a8e1b1337..3a99cd549 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/backend_contract.rs
+++ b/rustfs/src/on_demand_migration/backend_contract.rs
@@ -27,7 +27,7 @@
 //! and whether the provider can resume a listing from a key.
 
 use super::source_client::{SourceBackend, SourceError, SourceListRequest};
-use crate::storage_api_contracts::range::HTTPRangeSpec;
+use super::storage_api::HTTPRangeSpec;
 use std::collections::HashMap;
 
 /// The single object every fixture serves.
diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/rustfs/src/on_demand_migration/backfill.rs
similarity index 98%
rename from crates/ecstore/src/bucket/on_demand_migration/backfill.rs
rename to rustfs/src/on_demand_migration/backfill.rs
index 3a0ccfd69..8cc384ec8 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs
+++ b/rustfs/src/on_demand_migration/backfill.rs
@@ -45,16 +45,12 @@
 
 use super::pull::{EnqueueOutcome, PullReason, QueuedPullOutcome};
 use super::source_client::{SourceError, SourcePage};
+use super::storage_api::{
+    BUCKET_META_PREFIX, ECStore, HTTPPreconditions, NamespaceLocking as _, ObjectOperations as _, ObjectOptions,
+    RUSTFS_META_BUCKET, StorageError, WriteCompletion, get_lock_acquire_timeout, get_on_demand_migration_config_in,
+    local_node_name, read_config_with_metadata, save_config_with_opts,
+};
 use super::sys::{BucketOdmState, OnDemandMigrationSys};
-use crate::bucket::metadata_sys::bucket_metadata_sys_of;
-use crate::config::com::{read_config_with_metadata, save_config_with_opts};
-use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
-use crate::error::Error as StorageError;
-use crate::object_api::ObjectOptions;
-use crate::runtime::sources::local_node_name;
-use crate::set_disk::get_lock_acquire_timeout;
-use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions, object::ObjectOperations as _};
-use crate::store::ECStore;
 use async_trait::async_trait;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
@@ -474,12 +470,10 @@ impl BackfillContext for BucketBackfillContext {
     }
 
     async fn config_updated_at(&self) -> Result<Option<OffsetDateTime>, StorageError> {
-        let sys = bucket_metadata_sys_of(&self.api.ctx)?;
-        let guard = sys.read().await;
-        Ok(guard
-            .get_on_demand_migration_config(self.state.bucket())
-            .await?
-            .map(|(_, updated_at)| updated_at))
+        Ok(
+            super::config::decode_stored_config(get_on_demand_migration_config_in(&self.api.ctx, self.state.bucket()).await?)?
+                .map(|(_, updated_at)| updated_at),
+        )
     }
 }
 
@@ -683,7 +677,7 @@ async fn write_checkpoint(
     };
     let opts = ObjectOptions {
         max_parity: true,
-        write_completion: crate::object_api::WriteCompletion::TailDrained,
+        write_completion: WriteCompletion::TailDrained,
         http_preconditions: Some(preconditions),
         ..Default::default()
     };
@@ -1506,10 +1500,10 @@ pub async fn run_backfill_recovery_loop(runner: Arc<BackfillRunner>, cancel: Can
 
 #[cfg(test)]
 mod tests {
+    use super::super::storage_api::test_support::isolated_store_over_temp_disks;
     use super::*;
-    use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
-    use crate::bucket::on_demand_migration::source_client::SourceObject;
-    use crate::bucket::on_demand_migration::sys::PullError;
+    use crate::on_demand_migration::source_client::SourceObject;
+    use crate::on_demand_migration::sys::PullError;
     use std::collections::{BTreeSet, HashSet};
     use std::sync::atomic::AtomicBool;
 
diff --git a/crates/ecstore/src/bucket/on_demand_migration/breaker.rs b/rustfs/src/on_demand_migration/breaker.rs
similarity index 100%
rename from crates/ecstore/src/bucket/on_demand_migration/breaker.rs
rename to rustfs/src/on_demand_migration/breaker.rs
diff --git a/crates/ecstore/src/bucket/on_demand_migration/config.rs b/rustfs/src/on_demand_migration/config.rs
similarity index 94%
rename from crates/ecstore/src/bucket/on_demand_migration/config.rs
rename to rustfs/src/on_demand_migration/config.rs
index d00f08c2c..28b67819c 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/config.rs
+++ b/rustfs/src/on_demand_migration/config.rs
@@ -14,16 +14,34 @@
 
 //! Bucket-level On-Demand Migration configuration: wire model (JSON stored
 //! under `on-demand-migration.json`), pure validation, credential redaction,
-//! and the publish hook the runtime registers into (rustfs/backlog#2148).
+//! and persisted-config decoding (rustfs/backlog#2148).
 //!
 //! The persisted blob is not encrypted; it shares the trust boundary of
 //! `bucket-targets.json` and `tier-config.bin`.
 
 use serde::{Deserialize, Serialize};
 use std::fmt;
-use std::sync::OnceLock;
 use url::Url;
 
+/// Decode bytes only at the service boundary, preserving typed corruption errors.
+pub(super) fn decode_stored_config(
+    stored: Option<(Vec<u8>, time::OffsetDateTime)>,
+) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
+    stored
+        .map(|(bytes, updated_at)| {
+            OnDemandMigrationConfig::from_json(&bytes)
+                .map(|config| (config, updated_at))
+                .map_err(super::storage_api::StorageError::other)
+        })
+        .transpose()
+}
+
+pub(crate) async fn get_config(
+    bucket: &str,
+) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
+    decode_stored_config(super::storage_api::get_on_demand_migration_config(bucket).await?)
+}
+
 /// The only wire version this build reads and writes.
 pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1;
 
@@ -786,16 +804,6 @@ impl EndpointKey {
     }
 }
 
-/// Signature of the runtime publish hook: called with the bucket name and
-/// its parsed config (`None` when absent, cleared, or unreadable) every time
-/// the bucket's metadata is installed into or removed from the cache.
-pub type ConfigPublishHook = Box<dyn Fn(&str, Option<&OnDemandMigrationConfig>) + Send + Sync>;
-
-/// Registration point for the runtime (`OnDemandMigrationSys`). Until it is
-/// set, metadata publishes are no-ops for ODM, so this crate carries no
-/// runtime dependency and the config layer stays inert.
-pub static ON_DEMAND_MIGRATION_CONFIG_HOOK: OnceLock<ConfigPublishHook> = OnceLock::new();
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1501,4 +1509,55 @@ mod tests {
         assert!(!rendered.contains("topsecret"), "{rendered}");
         assert!(!rendered.contains("SK"), "{rendered}");
     }
+    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
+    /// stored payload it cannot parse as a typed error, never as a default
+    /// and never as `ConfigNotFound`.
+    #[tokio::test]
+    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
+        use super::super::storage_api::StorageError as Error;
+        use super::super::storage_api::test_support::{
+            BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata, BucketMetadataSys, isolated_store_over_temp_disks,
+        };
+        use std::sync::Arc;
+        const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
+
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let bucket = "odm-accessor";
+
+        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
+        assert_eq!(
+            decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap()).unwrap(),
+            None
+        );
+
+        let mut corrupt = BucketMetadata::new(bucket);
+        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
+        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
+        let err = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
+            .expect_err("corrupt config must not read as a default");
+        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
+        let typed = match &err {
+            Error::Io(io) => io
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
+            _ => None,
+        };
+        assert!(
+            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
+            "typed parse error must survive the Result boundary, got: {err:?}"
+        );
+
+        let mut valid = BucketMetadata::new(bucket);
+        valid
+            .update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
+            .unwrap();
+        let stamped = valid.on_demand_migration_config_updated_at;
+        sys.set(bucket.to_string(), Arc::new(valid)).await;
+        let (config, updated_at) = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
+            .unwrap()
+            .expect("stored config is returned");
+        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
+        assert_eq!(updated_at, stamped);
+    }
 }
diff --git a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs b/rustfs/src/on_demand_migration/gcs.rs
similarity index 98%
rename from crates/ecstore/src/bucket/on_demand_migration/gcs.rs
rename to rustfs/src/on_demand_migration/gcs.rs
index 874ddb0e3..bce647b97 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs
+++ b/rustfs/src/on_demand_migration/gcs.rs
@@ -43,8 +43,8 @@ use super::source_client::{
     GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
     SourceTimeouts, range_header_value,
 };
-use crate::bucket::remote_s3_client::RemoteS3ClientError;
-use crate::storage_api_contracts::range::HTTPRangeSpec;
+use super::storage_api::HTTPRangeSpec;
+use super::storage_api::remote_s3_client::RemoteS3ClientError;
 use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
 use google_cloud_auth::credentials::{CacheableResource, Credentials};
 use http::{HeaderMap, HeaderValue, Method};
@@ -318,8 +318,8 @@ fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
     use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
 
     const LIST_PAGE_ONE: &str = r#"{
diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/rustfs/src/on_demand_migration/list_through.rs
similarity index 100%
rename from crates/ecstore/src/bucket/on_demand_migration/list_through.rs
rename to rustfs/src/on_demand_migration/list_through.rs
diff --git a/rustfs/src/on_demand_migration/metrics.rs b/rustfs/src/on_demand_migration/metrics.rs
new file mode 100644
index 000000000..cf6e035cc
--- /dev/null
+++ b/rustfs/src/on_demand_migration/metrics.rs
@@ -0,0 +1,147 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Projection of application runtime state onto observability-owned DTOs.
+
+use crate::on_demand_migration::backfill::{
+    BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
+};
+use crate::on_demand_migration::{
+    BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
+    OnDemandMigrationSys as SourceOnDemandMigrationSys,
+};
+use rustfs_obs::metrics::{
+    OdmBackfillBucketStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
+    register_on_demand_migration_metrics_source,
+};
+
+pub(super) fn register() {
+    register_on_demand_migration_metrics_source(snapshot, backfill_snapshot);
+}
+
+fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
+    let stats = snapshot.stats;
+    OnDemandMigrationBucketStats {
+        bucket: snapshot.bucket,
+        requests_total: stats.requests_total,
+        pulled_bytes_total: stats.pulled_bytes_total,
+        pulled_objects_total: stats.pulled_objects_total,
+        pull_failures_total: stats.pull_failures_total,
+        inflight_pulls: stats.inflight_pulls,
+        queue_depth: stats.queue_depth,
+        source_latency_buckets: stats
+            .source_latency
+            .buckets
+            .into_iter()
+            .map(|bucket| (bucket.le_ms, bucket.count))
+            .collect(),
+        source_latency_count: stats.source_latency.count,
+        source_latency_sum_ms: stats.source_latency.sum_ms,
+        breaker_state: match stats.breaker_state {
+            SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
+            SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
+            SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
+        },
+    }
+}
+
+/// Every bucket with live on-demand migration state on this node, sorted by
+/// name. Empty while the module switch is off.
+fn snapshot() -> Vec<OnDemandMigrationBucketStats> {
+    SourceOnDemandMigrationSys::get()
+        .snapshot()
+        .into_iter()
+        .map(on_demand_migration_stats_from_snapshot)
+        .collect()
+}
+
+fn on_demand_migration_backfill_stats_from_checkpoint(
+    bucket: String,
+    checkpoint: SourceBackfillCheckpoint,
+) -> OdmBackfillBucketStats {
+    OdmBackfillBucketStats {
+        bucket,
+        state: checkpoint.state.as_str().to_string(),
+        listed: checkpoint.listed,
+        enqueued: checkpoint.enqueued,
+        pulled: checkpoint.pulled,
+        skipped_existing: checkpoint.skipped_existing,
+        failed: checkpoint.failed,
+        bytes: checkpoint.bytes,
+    }
+}
+
+/// Backfill jobs running on this node, sorted by bucket. Empty until the
+/// runner is installed, and empty again once a job finishes: the series are
+/// per-node job progress, not a cluster-wide history.
+fn backfill_snapshot() -> Vec<OdmBackfillBucketStats> {
+    source_global_backfill_runner()
+        .map(|runner| {
+            runner
+                .local_job_snapshots()
+                .into_iter()
+                .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
+                .collect()
+        })
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
+        // Pin the runtime wire snapshot and its observability projection together.
+        let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
+            "bucket": "photos",
+            "provider": "minio",
+            "endpoint_host": "source.example.com",
+            "applied_at": "2026-09-02T10:00:00Z",
+            "client_error": null,
+            "negative_cache_entries": 0,
+            "inflight_keys": 1,
+            "max_concurrent_pulls": 8,
+            "stats": {
+                "requests_total": {"get": {"source_hit": 2}},
+                "pulled_bytes_total": 4096,
+                "pulled_objects_total": {"inline": 1},
+                "pull_failures_total": {"source_timeout": 1},
+                "inflight_pulls": 1,
+                "queue_depth": 2,
+                "source_latency": {
+                    "buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
+                    "count": 3,
+                    "sum_ms": 90753
+                },
+                "last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
+                "breaker_state": "open"
+            }
+        }))
+        .expect("runtime snapshot decodes");
+
+        let stats = on_demand_migration_stats_from_snapshot(snapshot);
+
+        assert_eq!(stats.bucket, "photos");
+        assert_eq!(stats.requests_total["get"]["source_hit"], 2);
+        assert_eq!(stats.pulled_bytes_total, 4096);
+        assert_eq!(stats.pulled_objects_total["inline"], 1);
+        assert_eq!(stats.pull_failures_total["source_timeout"], 1);
+        assert_eq!(stats.inflight_pulls, 1);
+        assert_eq!(stats.queue_depth, 2);
+        assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
+        assert_eq!(stats.source_latency_count, 3);
+        assert_eq!(stats.source_latency_sum_ms, 90_753);
+        assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
+    }
+}
diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/rustfs/src/on_demand_migration/mod.rs
similarity index 84%
rename from crates/ecstore/src/bucket/on_demand_migration/mod.rs
rename to rustfs/src/on_demand_migration/mod.rs
index 554dadd60..823f37e99 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs
+++ b/rustfs/src/on_demand_migration/mod.rs
@@ -33,11 +33,13 @@ pub mod config;
 #[cfg(feature = "gcs")]
 pub mod gcs;
 pub mod list_through;
+mod metrics;
 mod native_http;
 pub mod negative_cache;
 pub mod pull;
 pub mod source_client;
 pub mod stats;
+mod storage_api;
 pub mod sys;
 #[cfg(test)]
 mod test_http_fixture;
@@ -47,9 +49,9 @@ pub use breaker::{
     BreakerState, BreakerTransition, BreakerVerdict,
 };
 pub use config::{
-    AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
-    ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider,
-    RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
+    AzureSourceConfig, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig,
+    OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, SourceCredentials,
+    SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
 };
 pub use list_through::{
     FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
@@ -63,6 +65,9 @@ pub use pull::{
     PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
     WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
 };
+pub use source_client::{
+    SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceSse, is_multipart_etag,
+};
 pub use stats::{
     GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
     PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
@@ -72,3 +77,7 @@ pub use sys::{
     OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
     source_client_spec,
 };
+
+pub(crate) fn register_metrics() {
+    metrics::register();
+}
diff --git a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs b/rustfs/src/on_demand_migration/native_http.rs
similarity index 99%
rename from crates/ecstore/src/bucket/on_demand_migration/native_http.rs
rename to rustfs/src/on_demand_migration/native_http.rs
index 8e5c12dd4..ae084d084 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs
+++ b/rustfs/src/on_demand_migration/native_http.rs
@@ -26,7 +26,7 @@
 //! the log line and the admin response.
 
 use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
-use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
+use super::storage_api::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_smithy_types::body::SdkBody;
 use futures::StreamExt;
diff --git a/crates/ecstore/src/bucket/on_demand_migration/negative_cache.rs b/rustfs/src/on_demand_migration/negative_cache.rs
similarity index 100%
rename from crates/ecstore/src/bucket/on_demand_migration/negative_cache.rs
rename to rustfs/src/on_demand_migration/negative_cache.rs
diff --git a/crates/ecstore/src/bucket/on_demand_migration/pull.rs b/rustfs/src/on_demand_migration/pull.rs
similarity index 99%
rename from crates/ecstore/src/bucket/on_demand_migration/pull.rs
rename to rustfs/src/on_demand_migration/pull.rs
index 8d18bdbdd..9d52af009 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/pull.rs
+++ b/rustfs/src/on_demand_migration/pull.rs
@@ -1093,7 +1093,7 @@ impl OnDemandMigrationSys {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::on_demand_migration::config::{
+    use crate::on_demand_migration::config::{
         FilterConfig, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, PolicyConfig, Provider, SourceConfig,
         SourceCredentials, TlsConfig,
     };
@@ -1499,7 +1499,7 @@ mod tests {
         assert_eq!(failures(&state).get("queue_full"), Some(&1));
         assert!(!queue.is_stopped());
 
-        assert_eq!(sys.remove(BUCKET), crate::bucket::on_demand_migration::ApplyOutcome::Removed);
+        assert_eq!(sys.remove(BUCKET), crate::on_demand_migration::ApplyOutcome::Removed);
         tokio::time::timeout(Duration::from_secs(5), queue.wait_until_stopped())
             .await
             .expect("dispatcher and in-flight job must exit after cancel");
diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/rustfs/src/on_demand_migration/source_client.rs
similarity index 99%
rename from crates/ecstore/src/bucket/on_demand_migration/source_client.rs
rename to rustfs/src/on_demand_migration/source_client.rs
index d2fc21c3e..9f3e050c9 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+++ b/rustfs/src/on_demand_migration/source_client.rs
@@ -29,10 +29,10 @@ use super::azure::AzureSourceBackend;
 #[cfg(feature = "gcs")]
 use super::gcs::GcsNativeSourceBackend;
 use super::list_through::{ListPageError, validate_list_page};
-use crate::bucket::remote_s3_client::{
+use super::storage_api::HTTPRangeSpec;
+use super::storage_api::remote_s3_client::{
     PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
 };
-use crate::storage_api_contracts::range::HTTPRangeSpec;
 use aws_sdk_s3::Client as S3Client;
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::get_object::GetObjectOutput;
@@ -995,7 +995,7 @@ fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
+    use crate::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
     use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn};
     use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
     use aws_smithy_runtime_api::client::result::ConnectorError;
diff --git a/crates/ecstore/src/bucket/on_demand_migration/stats.rs b/rustfs/src/on_demand_migration/stats.rs
similarity index 100%
rename from crates/ecstore/src/bucket/on_demand_migration/stats.rs
rename to rustfs/src/on_demand_migration/stats.rs
diff --git a/rustfs/src/on_demand_migration/storage_api.rs b/rustfs/src/on_demand_migration/storage_api.rs
new file mode 100644
index 000000000..d7917ad29
--- /dev/null
+++ b/rustfs/src/on_demand_migration/storage_api.rs
@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Storage capabilities used by on-demand migration orchestration.
+
+#[cfg(test)]
+pub(super) use crate::storage_api::on_demand_migration::test_support;
+pub(super) use crate::storage_api::on_demand_migration::{
+    BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_META_PREFIX, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, HTTPPreconditions, HTTPRangeSpec,
+    NamespaceLocking, ObjectOperations, ObjectOptions, RUSTFS_META_BUCKET, StorageError, WriteCompletion,
+    get_lock_acquire_timeout, get_on_demand_migration_config, get_on_demand_migration_config_in, read_config_with_metadata,
+    remote_s3_client, save_config_with_opts,
+};
+
+pub(super) async fn local_node_name() -> String {
+    rustfs_common::get_global_local_node_name().await
+}
diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/rustfs/src/on_demand_migration/sys.rs
similarity index 96%
rename from crates/ecstore/src/bucket/on_demand_migration/sys.rs
rename to rustfs/src/on_demand_migration/sys.rs
index ab42bd602..d529594d4 100644
--- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs
+++ b/rustfs/src/on_demand_migration/sys.rs
@@ -19,7 +19,7 @@
 //! [`SourceClient`], a circuit breaker, a negative cache, a per-key
 //! singleflight table, a pull concurrency limit and counters. Its lifecycle
 //! follows the bucket metadata cache through the publish hook registered in
-//! [`ON_DEMAND_MIGRATION_CONFIG_HOOK`]; the hook fires on every cache install
+//! [`BUCKET_CONFIG_PUBLISH_HOOK`]; the hook fires on every cache install
 //! path (initial load, admin update, peer reload, refresh loop, lazy load).
 //!
 //! Change detection compares the config by value (`PartialEq`) rather than
@@ -41,9 +41,7 @@
 
 use super::backfill::{PriorityPullPermits, PullPermit, PullPriority};
 use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
-use super::config::{
-    ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
-};
+use super::config::{OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig};
 use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
 use super::negative_cache::NegativeCache;
 use super::pull::{OdmWriteBack, PullQueue};
@@ -52,9 +50,10 @@ use super::source_client::{
     SourceTimeouts,
 };
 use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
-use crate::bucket::remote_s3_client::{
+use super::storage_api::remote_s3_client::{
     PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
 };
+use super::storage_api::{BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
 use parking_lot::{Mutex, RwLock};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -740,11 +739,34 @@ impl OnDemandMigrationSys {
     /// Registers `publish` as the bucket-metadata publish hook. Returns
     /// `false` when a hook was already registered.
     pub fn register_config_hook(&'static self) -> bool {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK
-            .set(Box::new(move |bucket, config| self.publish(bucket, config)))
+        BUCKET_CONFIG_PUBLISH_HOOK
+            .set(Box::new(move |bucket, config_file, stored| {
+                if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
+                    self.publish_stored(bucket, stored.map(|(bytes, _)| bytes));
+                }
+            }))
             .is_ok()
     }
 
+    /// Corrupt persisted bytes withdraw state synchronously, just like deletion.
+    fn publish_stored(&'static self, bucket: &str, stored: Option<&[u8]>) {
+        match stored.map(OnDemandMigrationConfig::from_json).transpose() {
+            Ok(config) => self.publish(bucket, config.as_ref()),
+            Err(err) => {
+                warn!(
+                    event = EVENT_ODM_BUCKET_STATE_APPLIED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
+                    result = "invalid",
+                    bucket = %bucket,
+                    error = %err,
+                    "Failed to parse on-demand migration config"
+                );
+                self.publish(bucket, None);
+            }
+        }
+    }
+
     /// Hook entry point: removals apply immediately, installs are spawned
     /// (client construction is async). Requires a Tokio runtime for the
     /// install path; without one the config is logged and skipped.
@@ -951,8 +973,8 @@ impl OnDemandMigrationSys {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
-    use crate::bucket::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
+    use crate::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
+    use crate::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
     use std::sync::atomic::AtomicUsize;
     use tokio::sync::Barrier;
 
@@ -1344,6 +1366,17 @@ mod tests {
         assert!(state.is_cancelled());
     }
 
+    #[tokio::test]
+    async fn corrupt_stored_config_withdraws_runtime_state() {
+        let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
+        let cfg = config(None);
+        assert_eq!(sys.apply("corrupt", Some(&cfg)).await, ApplyOutcome::Installed);
+        let state = sys.state("corrupt").expect("state installed");
+        sys.publish_stored("corrupt", Some(b"not-json"));
+        assert!(sys.state("corrupt").is_none(), "corruption cannot keep an older source active");
+        assert!(state.is_cancelled(), "corruption cancels in-flight work");
+    }
+
     #[tokio::test]
     async fn absent_config_updates_do_not_allocate_bucket_slots() {
         let sys = enabled_sys();
diff --git a/crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs b/rustfs/src/on_demand_migration/test_http_fixture.rs
similarity index 100%
rename from crates/ecstore/src/bucket/on_demand_migration/test_http_fixture.rs
rename to rustfs/src/on_demand_migration/test_http_fixture.rs
diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs
index 01b0886de..393b7ad3d 100644
--- a/rustfs/src/startup_background.rs
+++ b/rustfs/src/startup_background.rs
@@ -17,10 +17,11 @@ use crate::module_switches::{
     bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env,
     is_on_demand_migration_module_enabled, scanner_enabled_from_env,
 };
-use crate::storage_api::startup::background::{
-    BackfillRunner, ECStore, OnDemandMigrationSys, SysBackfillContexts, install_global_backfill_runner,
-    set_workload_admission_snapshot_provider, spawn_backfill_recovery_loop,
+use crate::on_demand_migration::OnDemandMigrationSys;
+use crate::on_demand_migration::backfill::{
+    BackfillRunner, SysBackfillContexts, install_global_backfill_runner, spawn_backfill_recovery_loop,
 };
+use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider};
 use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
 use rustfs_heal::{
diff --git a/rustfs/src/startup_bucket_metadata.rs b/rustfs/src/startup_bucket_metadata.rs
index 1c3cdb270..a85d71ab3 100644
--- a/rustfs/src/startup_bucket_metadata.rs
+++ b/rustfs/src/startup_bucket_metadata.rs
@@ -14,10 +14,11 @@
 
 use crate::app::object::OnDemandMigrationWriteBack;
 use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled};
+use crate::on_demand_migration::OnDemandMigrationSys;
 use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
 use crate::storage_api::startup::bucket_metadata::{
-    ECStore, Error as StorageError, OnDemandMigrationSys, Result as StorageResult, get_global_replication_pool,
-    init_bucket_metadata_sys, reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
+    ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys,
+    reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
 };
 use std::{
     io::{Error as IoError, Result as IoResult},
@@ -98,6 +99,7 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
 /// `OnDemandMigrationSys` with a usable write-back (rustfs/backlog#2152).
 /// Idempotent across embedded and server startups.
 fn init_on_demand_migration_runtime() {
+    crate::on_demand_migration::register_metrics();
     let enabled = on_demand_migration_enabled_from_env();
     set_on_demand_migration_module_enabled(enabled);
     let sys = OnDemandMigrationSys::get();
diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs
index 2216db7f4..fe9a55641 100644
--- a/rustfs/src/storage/storage_api.rs
+++ b/rustfs/src/storage/storage_api.rs
@@ -407,8 +407,8 @@ pub(crate) mod ecstore_bucket {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
     pub(crate) use rustfs_ecstore::api::bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, on_demand_migration,
-        policy_sys, remote_s3_client, replication, tagging, target, utils,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
+        remote_s3_client, replication, tagging, target, utils,
     };
     pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
 }
@@ -466,10 +466,10 @@ pub(crate) mod ecstore_data_usage {
 #[allow(unused_imports)]
 pub(crate) mod ecstore_disk {
     pub(crate) use rustfs_ecstore::api::disk::{
-        BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
-        FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
-        ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
-        get_object_disk_read_timeout, validate_batch_read_version_item_count,
+        BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo,
+        DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction,
+        RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken,
+        UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count,
     };
     pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
 }
@@ -560,7 +560,7 @@ pub(crate) mod ecstore_object {
     pub(crate) use rustfs_ecstore::api::object::{
         EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
         ObjectEncryptionResolver, ObjectMutationHook, PrepareSelectObjectSnapshotError, ReadEncryptionMaterial,
-        ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, get_object_body_cache_plaintext_len,
+        ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, WriteCompletion, get_object_body_cache_plaintext_len,
         lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
         unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs
index 4f824f8da..60de2c850 100644
--- a/rustfs/src/storage_api.rs
+++ b/rustfs/src/storage_api.rs
@@ -278,10 +278,6 @@ pub(crate) mod startup {
     }
 
     pub(crate) mod background {
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::backfill::{
-            BackfillRunner, SysBackfillContexts, install_global_backfill_runner, spawn_backfill_recovery_loop,
-        };
         pub(crate) use crate::storage::storage_api::{
             BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider,
         };
@@ -294,7 +290,6 @@ pub(crate) mod startup {
             }
         }
 
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
         pub(crate) use crate::storage::storage_api::{
             ECStore, Error, Result, get_global_replication_pool, init_bucket_metadata_sys,
             reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
@@ -415,3 +410,30 @@ pub(crate) mod table {
         get_lock_acquire_timeout, table_catalog_path_hash,
     };
 }
+
+pub(crate) mod on_demand_migration {
+    pub(crate) use crate::storage::storage_api::ECStore;
+    pub(crate) use crate::storage::storage_api::StorageObjectOptions as ObjectOptions;
+    pub(crate) use crate::storage::storage_api::contract::{
+        namespace::NamespaceLocking, object::HTTPPreconditions, object::ObjectOperations, range::HTTPRangeSpec,
+    };
+    pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
+    pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{
+        BUCKET_CONFIG_PUBLISH_HOOK, get_on_demand_migration_config, get_on_demand_migration_config_in,
+    };
+    pub(crate) use crate::storage::storage_api::ecstore_bucket::remote_s3_client;
+    pub(crate) use crate::storage::storage_api::ecstore_config::com::{read_config_with_metadata, save_config_with_opts};
+    pub(crate) use crate::storage::storage_api::ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+    pub(crate) use crate::storage::storage_api::ecstore_error::Error as StorageError;
+    pub(crate) use crate::storage::storage_api::ecstore_object::WriteCompletion;
+    pub(crate) use crate::storage::storage_api::ecstore_set_disk::get_lock_acquire_timeout;
+
+    #[cfg(test)]
+    pub(crate) mod test_support {
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::{
+            BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata,
+        };
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::BucketMetadataSys;
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+    }
+}

From 30ab919bb33e8b68ec803ae0005913978fbfb4fe Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:35:05 +0800
Subject: [PATCH 13/20] fix(ci): route v1 listing test types through
 application bridge (#7227)

---
 rustfs/src/app/bucket_list_through.rs | 7 ++++---
 rustfs/src/app/storage_api.rs         | 4 ++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index c385fdb9c..586f5b0ae 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -443,7 +443,9 @@ mod tests {
     use super::*;
     use crate::app::bucket_usecase::DefaultBucketUsecase;
     use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
-    use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
+    use crate::app::storage_api::bucket_usecase::s3::{
+        ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response, XmlSerialize, XmlSerializer,
+    };
     use crate::app::storage_api::test::StoragePutObjReader;
     use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
     use crate::app::storage_api::test::contract::object::ObjectIO as _;
@@ -451,7 +453,6 @@ mod tests {
         FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
         SourceCredentials, TlsConfig,
     };
-    use s3s::dto::ListObjectsInput;
     use std::time::Duration;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
@@ -864,7 +865,7 @@ mod tests {
                             assert_eq!(output.next_marker.as_deref(), (index == 0).then_some(expected_key));
 
                             let mut xml = Vec::new();
-                            s3s::xml::Serialize::serialize(&output, &mut s3s::xml::Serializer::new(&mut xml))
+                            XmlSerialize::serialize(&output, &mut XmlSerializer::new(&mut xml))
                                 .expect("serialize the real v1 response");
                             assert!(!xml.contains(&0), "XML 1.0 forbids NUL in NextMarker");
                             let mut reader = quick_xml::Reader::from_reader(xml.as_slice());
diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs
index 6c810e213..da7af7c86 100644
--- a/rustfs/src/app/storage_api.rs
+++ b/rustfs/src/app/storage_api.rs
@@ -27,12 +27,16 @@ pub(crate) fn EndpointServerPools(
 /// S3 wire types for app-layer modules, funneled here so new files stay off
 /// the direct s3s surface (s3s footprint ratchet, `scripts/check_s3s_footprint.sh`).
 pub(crate) mod s3 {
+    #[cfg(test)]
+    pub(crate) use s3s::dto::ListObjectsInput;
     #[cfg(test)]
     pub(crate) use s3s::dto::{
         BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
         ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
         ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
     };
+    #[cfg(test)]
+    pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};
     pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
     #[cfg(test)]
     pub(crate) use s3s::{S3Request, S3Response};

From 037354cec08180d6a5eb3390aac24534dc096ac6 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:37:02 +0800
Subject: [PATCH 14/20] fix(build): declare relocated migration service
 dependencies (#7229)

---
 Cargo.lock        | 2 ++
 rustfs/Cargo.toml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index c6d279229..aeadb7866 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9527,6 +9527,7 @@ dependencies = [
  "metrics",
  "metrics-util",
  "mime_guess",
+ "moka",
  "opentelemetry",
  "opentelemetry_sdk",
  "p256 0.14.0",
@@ -9619,6 +9620,7 @@ dependencies = [
  "urlencoding",
  "uuid",
  "x509-parser",
+ "xxhash-rust",
  "zeroize",
  "zip",
  "zstd 0.14.0",
diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml
index 03b6b378e..aabf2d865 100644
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -291,6 +291,8 @@ tokio-rustls = { workspace = true, default-features = false, features = ["loggin
 aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 aws-smithy-types = { workspace = true }
 google-cloud-auth = { workspace = true, optional = true }
+moka = { workspace = true, features = ["sync"] }
+xxhash-rust = { workspace = true, features = ["xxh3"] }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "time"] }

From 6d8606412eabb6403212c81edb28d4fa73a8af55 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 01:43:14 +0800
Subject: [PATCH 15/20] fix(odm): compile relocated instance-bound backfill
 service (#7230)

---
 Cargo.lock                                 | 1 +
 crates/ecstore/src/bucket/metadata_sys.rs  | 7 ++-----
 crates/ecstore/src/store/init.rs           | 5 +++++
 crates/madmin/src/on_demand_migration.rs   | 2 +-
 rustfs/Cargo.toml                          | 1 +
 rustfs/src/on_demand_migration/backfill.rs | 4 ++--
 6 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index aeadb7866..a9fac1098 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9503,6 +9503,7 @@ dependencies = [
  "clap",
  "const-str",
  "datafusion",
+ "faster-hex",
  "flatbuffers",
  "flate2",
  "futures",
diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs
index 5e30841ad..7b36a1647 100644
--- a/crates/ecstore/src/bucket/metadata_sys.rs
+++ b/crates/ecstore/src/bucket/metadata_sys.rs
@@ -1043,11 +1043,8 @@ pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<
 }
 
 /// Resolve opaque configuration from the store's own metadata system.
-pub async fn get_on_demand_migration_config_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
-    let sys = bucket_metadata_sys_of(ctx)?;
+pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+    let sys = bucket_metadata_sys_of(&api.ctx)?;
     let lock = sys.read().await;
     lock.get_on_demand_migration_config(bucket).await
 }
diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs
index 63d2b5fdd..764e48391 100644
--- a/crates/ecstore/src/store/init.rs
+++ b/crates/ecstore/src/store/init.rs
@@ -353,6 +353,11 @@ async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken)
 }
 
 impl ECStore {
+    /// Shutdown token owned by this store instance.
+    pub fn background_cancel_token(&self) -> Option<CancellationToken> {
+        self.ctx.background_cancel_token()
+    }
+
     /// Validate topology and process storage-class overrides before any disk is opened.
     pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
         let drive_counts = startup_pool_drive_counts(endpoint_pools);
diff --git a/crates/madmin/src/on_demand_migration.rs b/crates/madmin/src/on_demand_migration.rs
index 0fce8af6c..e92899717 100644
--- a/crates/madmin/src/on_demand_migration.rs
+++ b/crates/madmin/src/on_demand_migration.rs
@@ -17,7 +17,7 @@
 //! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}`,
 //! `GET .../status`, `POST .../backfill?op=start|cancel` and
 //! `GET .../backfill` (ODM-12), mirroring the server's config model
-//! (`crates/ecstore/src/bucket/on_demand_migration/config.rs`) and handler
+//! (`rustfs/src/on_demand_migration/config.rs`) and handler
 //! responses (`rustfs/src/admin/handlers/on_demand_migration.rs`). The SDK
 //! owns its own copies, madmin-go style; the fixtures under
 //! `fixtures/on_demand_migration/` are the contract both sides pin
diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml
index aabf2d865..52a7e6598 100644
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -276,6 +276,7 @@ rcgen = { workspace = true }
 # Async Runtime and Networking
 async-trait = { workspace = true }
 axum.workspace = true
+faster-hex.workspace = true
 futures.workspace = true
 futures-lite.workspace = true
 futures-util.workspace = true
diff --git a/rustfs/src/on_demand_migration/backfill.rs b/rustfs/src/on_demand_migration/backfill.rs
index 8cc384ec8..f2d14b61d 100644
--- a/rustfs/src/on_demand_migration/backfill.rs
+++ b/rustfs/src/on_demand_migration/backfill.rs
@@ -471,7 +471,7 @@ impl BackfillContext for BucketBackfillContext {
 
     async fn config_updated_at(&self) -> Result<Option<OffsetDateTime>, StorageError> {
         Ok(
-            super::config::decode_stored_config(get_on_demand_migration_config_in(&self.api.ctx, self.state.bucket()).await?)?
+            super::config::decode_stored_config(get_on_demand_migration_config_in(&self.api, self.state.bucket()).await?)?
                 .map(|(_, updated_at)| updated_at),
         )
     }
@@ -1472,7 +1472,7 @@ impl Job {
 /// Spawns [`run_backfill_recovery_loop`] on the store's shutdown token;
 /// `false` (nothing spawned) when the store has no background token.
 pub fn spawn_backfill_recovery_loop(runner: Arc<BackfillRunner>) -> bool {
-    let Some(cancel) = runner.api.ctx.background_cancel_token() else {
+    let Some(cancel) = runner.api.background_cancel_token() else {
         return false;
     };
     tokio::spawn(run_backfill_recovery_loop(runner, cancel));

From dd368f0f5b7d9d50c98246c1d1694724a331be32 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 02:05:43 +0800
Subject: [PATCH 16/20] fix(odm): fence source work against bucket recreation
 (#7231)

* fix(odm): fence backfill checkpoints by bucket incarnation

* fix(odm): bind source work to the bucket incarnation

* fix(odm): retain checkpoint fences through owned commit tails

* docs(odm): explain application service and incarnation boundaries

* test(odm): probe lifecycle fence after checkpoint waiter aborts

* fix(odm): defer source identity errors past local reads

* docs(metadata): clarify MinIO target recovery limits

* fix(odm): keep source-free reads independent of capture errors

* fix(odm): retain one source policy snapshot across lookup

* test(odm): name recorded metadata hook snapshots
---
 crates/ecstore/src/bucket/metadata_sys.rs     |  27 +-
 crates/ecstore/src/store/bucket.rs            |   4 +-
 crates/ecstore/src/store/bucket_fence.rs      |  40 +-
 crates/ecstore/src/store/mod.rs               |   1 +
 docs/architecture/crate-boundaries.md         |  13 +
 docs/operations/bucket-metadata-recovery.md   |   2 +-
 rustfs/src/app/bucket_list_through.rs         | 170 ++++++++-
 rustfs/src/app/bucket_usecase.rs              |  11 +-
 rustfs/src/app/object/get.rs                  |  34 +-
 rustfs/src/app/object/head.rs                 |  52 ++-
 rustfs/src/app/object/internal_put.rs         |  42 +-
 rustfs/src/app/object/mod.rs                  |   4 +-
 .../src/app/object/on_demand_migration_put.rs | 219 +++++++++--
 rustfs/src/app/object/put.rs                  |   9 +-
 rustfs/src/app/storage_api.rs                 |  18 +-
 rustfs/src/on_demand_migration/backfill.rs    | 284 +++++++++++---
 rustfs/src/on_demand_migration/pull.rs        |  11 +-
 rustfs/src/on_demand_migration/sys.rs         | 177 ++++++++-
 rustfs/src/storage/access.rs                  | 360 +++++++++++++++++-
 rustfs/src/storage/storage_api.rs             |   5 +-
 rustfs/src/storage_api.rs                     |   4 +-
 21 files changed, 1320 insertions(+), 167 deletions(-)

diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs
index 7b36a1647..17f185b63 100644
--- a/crates/ecstore/src/bucket/metadata_sys.rs
+++ b/crates/ecstore/src/bucket/metadata_sys.rs
@@ -50,7 +50,7 @@ use uuid::Uuid;
 
 /// Opaque bucket configuration notifications for application-owned services.
 /// `None` withdraws a configuration; consumers validate nonempty bytes.
-pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime)>) + Send + Sync>;
+pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
 pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
 
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
@@ -405,7 +405,8 @@ fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
         hook(
             bucket,
             super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
-            bm.on_demand_migration_config(),
+            bm.on_demand_migration_config()
+                .map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
         );
     }
 }
@@ -4374,23 +4375,26 @@ mod tests {
 
     const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
 
+    type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
+    type RecordedOdmHookCall = (String, RecordedOdmConfig);
+
     /// Every `(bucket, config)` the recording hook has seen. Tests filter by
     /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<(Vec<u8>, OffsetDateTime)>)>> = std::sync::Mutex::new(Vec::new());
+    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
 
     fn install_recording_odm_hook() {
         BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
             Box::new(|bucket, config_file, config| {
                 assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
-                ODM_HOOK_CALLS
-                    .lock()
-                    .unwrap()
-                    .push((bucket.to_string(), config.map(|(bytes, stamp)| (bytes.to_vec(), stamp))));
+                ODM_HOOK_CALLS.lock().unwrap().push((
+                    bucket.to_string(),
+                    config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
+                ));
             })
         });
     }
 
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<(Vec<u8>, OffsetDateTime)>> {
+    fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
         ODM_HOOK_CALLS
             .lock()
             .unwrap()
@@ -4414,18 +4418,21 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
         }
 
+        let incarnation = Uuid::new_v4();
         let expect_publish = |before: usize, label: &str| {
             let calls = odm_hook_calls(bucket);
             assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
             assert_eq!(
-                calls.last().unwrap().as_ref().map(|(bytes, _)| bytes.as_slice()),
+                calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
                 Some(ODM_JSON),
                 "{label} must publish the stored bytes"
             );
+            assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
         };
 
         // set (via persist_new_and_set, which installs through `set`).
         let mut bm = BucketMetadata::new(bucket);
+        bm.bucket_incarnation_id = incarnation;
         bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
             .unwrap();
         let writer = BucketMetadataSys::new(ecstore.clone());
@@ -4475,7 +4482,7 @@ mod tests {
         let calls = odm_hook_calls(bucket);
         assert_eq!(calls.len(), before + 1);
         assert_eq!(
-            calls.last().unwrap().as_ref().map(|(bytes, _)| bytes.as_slice()),
+            calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
             Some(b"not-json".as_slice()),
             "the application validates opaque config bytes"
         );
diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs
index 210118cda..c648255ad 100644
--- a/crates/ecstore/src/store/bucket.rs
+++ b/crates/ecstore/src/store/bucket.rs
@@ -329,11 +329,11 @@ impl ECStore {
     /// reuse its result, which is sound because bucket deletion/recreation
     /// requires the lifecycle WRITE lock and therefore cannot have run while
     /// any read guard was continuously held.
-    pub(crate) async fn acquire_bucket_incarnation_fence(
+    pub async fn acquire_bucket_incarnation_fence(
         &self,
         bucket: &str,
         expected: uuid::Uuid,
-    ) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
+    ) -> Result<super::BucketIncarnationFenceGuard> {
         let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
         let pieces = super::bucket_fence::FencePieces {
             registry: self.bucket_fence_registry.clone(),
diff --git a/crates/ecstore/src/store/bucket_fence.rs b/crates/ecstore/src/store/bucket_fence.rs
index 1800d53a3..66b1f9d01 100644
--- a/crates/ecstore/src/store/bucket_fence.rs
+++ b/crates/ecstore/src/store/bucket_fence.rs
@@ -150,7 +150,7 @@ impl BucketFenceRegistry {
 /// A held bucket lifecycle read lock plus its registration in the fence
 /// registry. Dropping the guard deregisters it; the memo is cleared when the
 /// last guard for the bucket drops (or a lost lock is observed).
-pub(crate) struct BucketIncarnationFenceGuard {
+pub struct BucketIncarnationFenceGuard {
     inner: Option<NamespaceLockGuard>,
     registry: Arc<BucketFenceRegistry>,
     bucket: String,
@@ -158,6 +158,14 @@ pub(crate) struct BucketIncarnationFenceGuard {
 }
 
 impl BucketIncarnationFenceGuard {
+    /// Propagate lifecycle lock loss into the storage commit checks.
+    /// The caller still owns this guard until the complete write tail drains.
+    pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
+        if let Some(guard) = self.namespace_lock_guard() {
+            opts.add_bucket_lifecycle_lock_guard(guard);
+        }
+    }
+
     pub(crate) fn is_lock_lost(&self) -> bool {
         self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
     }
@@ -346,6 +354,36 @@ mod tests {
         first_pieces.abandon("b", first.token);
     }
 
+    #[tokio::test]
+    async fn checkpoint_options_inherit_bucket_fence_lock_loss() {
+        let lock = NamespaceLock::new("bucket-fence-options".to_string(), Arc::new(LocalClient::new()));
+        let inner = lock
+            .acquire_guard(&lock_request("options"))
+            .await
+            .expect("acquire")
+            .expect("quorum");
+        let pieces = FencePieces {
+            registry: Arc::default(),
+            inner,
+        };
+        let registration = pieces.enter("b");
+        let fence = pieces.into_guard("b", registration.token);
+        let mut opts = crate::object_api::ObjectOptions::default();
+        fence.attach_to_object_options(&mut opts);
+        let inherited = opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .expect("checkpoint inherits lifecycle guard");
+        assert!(!inherited.is_lock_lost());
+        tokio::time::timeout(
+            Duration::from_secs(2),
+            fence.namespace_lock_guard().expect("held guard").lock_lost_notified(),
+        )
+        .await
+        .expect("distributed guard expires");
+        assert!(inherited.is_lock_lost(), "the actual pre-rename options must observe lifecycle lock loss");
+    }
+
     #[test]
     fn buckets_are_isolated() {
         let reg = BucketFenceRegistry::default();
diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs
index b2f2965f6..b54a69438 100644
--- a/crates/ecstore/src/store/mod.rs
+++ b/crates/ecstore/src/store/mod.rs
@@ -417,6 +417,7 @@ const MAX_UPLOADS_LIST: usize = 10000;
 mod bucket;
 mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
+pub use bucket_fence::BucketIncarnationFenceGuard;
 mod heal;
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md
index 6fdca19ab..09a308d8f 100644
--- a/docs/architecture/crate-boundaries.md
+++ b/docs/architecture/crate-boundaries.md
@@ -87,6 +87,12 @@ The guard requires the documents and section headings listed in its `require_sou
 
 ## On-Demand Migration Service
 
+Read-through, backfill and external pull orchestration belong in an application
+service under `rustfs/src/<service>/`. ECStore owns the storage primitives they
+need, including atomic commits, lifecycle locks and on-disk metadata. A service
+may use these primitives without moving its provider clients or scheduling
+policy into the engine.
+
 `rustfs/src/on_demand_migration/` owns source clients, pull scheduling, list
 merging, runtime state and backfill orchestration. Its `storage_api.rs` is the
 only ECStore facade boundary. Object write-back still enters the application's
@@ -101,6 +107,13 @@ deployment constraints in the admin use case before the incarnation-fenced
 metadata update. Backfill reads metadata from its store's instance context and
 preserves the checkpoint ETag compare-and-set, lease and tail-drained writes.
 
+An ODM runtime is bound to the bucket incarnation published with its metadata,
+not just its name. Source reads and write-back reject a different incarnation.
+Checkpoint writes hold the user bucket's lifecycle fence through their complete
+commit and read-back, even if their caller stops waiting; the storage commit
+also observes lock loss. Deleting and recreating a bucket must not let work for
+its previous incarnation repopulate objects or checkpoints.
+
 Observability owns its metric DTOs and accepts application snapshot callbacks;
 it does not depend on the ODM runtime. The application registers both bucket
 and backfill snapshots during startup, before metadata and metric collection.
diff --git a/docs/operations/bucket-metadata-recovery.md b/docs/operations/bucket-metadata-recovery.md
index ff513e9db..92227c359 100644
--- a/docs/operations/bucket-metadata-recovery.md
+++ b/docs/operations/bucket-metadata-recovery.md
@@ -12,7 +12,7 @@ To inspect readable configurations while identifying failures, use the same auth
 
 ## Recover unreadable replication targets
 
-MinIO target configuration may be an array or KMS-encrypted data that RustFS cannot decode. Diagnosis preserves the failure instead of interpreting it as an empty target set.
+RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support.
 
 1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
 2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index 586f5b0ae..86497e36f 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -28,7 +28,7 @@ use super::storage_api::bucket_usecase::StorageObjectOptions;
 use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
 use super::storage_api::bucket_usecase::contract::list::{ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _};
 use super::storage_api::bucket_usecase::contract::object::ObjectOperations as _;
-use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Result};
+use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Request, S3Result};
 use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params;
 use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class};
 use crate::error::ApiError;
@@ -101,16 +101,38 @@ fn invalid_continuation_token(err: &ListThroughTokenError) -> S3Error {
 /// bucket has no source, `list_through` is off, or the request carries the
 /// `source-proxy-request` anti-loop marker and therefore comes from a peer
 /// that must be answered locally.
-pub(crate) fn list_through_state(bucket: &str, headers: &HeaderMap) -> Option<Arc<BucketOdmState>> {
-    if get_header(headers, SUFFIX_SOURCE_PROXY_REQUEST).is_some() {
-        return None;
+pub(crate) async fn list_through_state<T>(
+    store: &ECStore,
+    bucket: &str,
+    req: &S3Request<T>,
+    params: &ListObjectsV2Params,
+) -> S3Result<Option<Arc<BucketOdmState>>> {
+    if get_header(&req.headers, SUFFIX_SOURCE_PROXY_REQUEST).is_some() {
+        return Ok(None);
     }
     let sys = OnDemandMigrationSys::get();
     if !sys.is_module_enabled() {
-        return None;
+        return Ok(None);
     }
-    let state = sys.state(bucket)?;
-    state.config().policy.list_through.then_some(state)
+    let Some(state) = sys.state(bucket).filter(|state| state.config().policy.list_through) else {
+        return Ok(None);
+    };
+    if params.max_keys == 0
+        || matches!(
+            source_list_plan(&params.prefix, state.config().filter.prefix.as_deref(), params.delimiter.as_deref()),
+            SourceListPlan::Skip,
+        )
+    {
+        return Ok(None);
+    }
+    let Some(expected_incarnation) = super::storage_api::bucket_usecase::access::odm_read_generation(req, bucket)? else {
+        return Ok(None);
+    };
+    let incarnation = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?;
+    if incarnation != expected_incarnation {
+        return Ok(None);
+    }
+    Ok(state.filter_incarnation(incarnation))
 }
 
 /// A merged page plus whether the source had to be left out of it.
@@ -444,10 +466,11 @@ mod tests {
     use crate::app::bucket_usecase::DefaultBucketUsecase;
     use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
     use crate::app::storage_api::bucket_usecase::s3::{
-        ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response, XmlSerialize, XmlSerializer,
+        GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
+        XmlSerialize, XmlSerializer,
     };
     use crate::app::storage_api::test::StoragePutObjReader;
-    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use crate::app::storage_api::test::contract::object::ObjectIO as _;
     use crate::on_demand_migration::{
         FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
@@ -727,7 +750,12 @@ mod tests {
                 ..Default::default()
             },
         };
-        sys.apply(&bucket, Some(&config)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket identity"),
+            Some(&config),
+        )
+        .await;
         assert!(
             sys.state(&bucket).expect("ODM state installed").client().is_ok(),
             "fake source client must build"
@@ -797,6 +825,128 @@ mod tests {
         (result, requests)
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn stale_bucket_state_cannot_send_get_head_or_list_to_the_source() {
+        run_large_stack_test("list-through-incarnation", || async {
+            temp_env::async_with_vars(
+                [
+                    ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
+                    ("HTTP_PROXY", None),
+                    ("HTTPS_PROXY", None),
+                    ("ALL_PROXY", None),
+                    ("http_proxy", None),
+                    ("https_proxy", None),
+                    ("all_proxy", None),
+                    ("NO_PROXY", Some("*")),
+                    ("no_proxy", Some("*")),
+                ],
+                async {
+                    let (endpoint, server, stop) = list_source(std::iter::repeat(source_xml(None, false, Some("source")))).await;
+                    let (_guard, input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await;
+                    let sys = OnDemandMigrationSys::get();
+                    let old = sys.state(&input.bucket).expect("original source state");
+                    let store = shared_gating_ecstore().await;
+                    let get = S3Request {
+                        input: GetObjectInput {
+                            bucket: input.bucket.clone(),
+                            key: "missing".into(),
+                            ..Default::default()
+                        },
+                        method: http::Method::GET,
+                        uri: http::Uri::from_static("/missing"),
+                        headers: HeaderMap::new(),
+                        extensions: http::Extensions::new(),
+                        credentials: None,
+                        region: None,
+                        service: None,
+                        trailing_headers: None,
+                    };
+                    let authorized_generation =
+                        super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
+                            &store,
+                            &get,
+                            &input.bucket,
+                        )
+                        .await
+                        .expect("capture the identity before authorization");
+                    store
+                        .delete_bucket(
+                            &input.bucket,
+                            &DeleteBucketOptions {
+                                force: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("delete original bucket");
+                    store
+                        .make_bucket(&input.bucket, &MakeBucketOptions::default())
+                        .await
+                        .expect("recreate bucket");
+                    let replacement = store
+                        .bucket_incarnation_id(&input.bucket)
+                        .await
+                        .expect("replacement identity");
+                    assert_ne!(replacement, old.incarnation_id());
+                    sys.remove(&input.bucket);
+                    let mut without_source = get.clone();
+                    super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
+                        &store,
+                        &mut without_source,
+                        &input.bucket,
+                    )
+                    .await;
+                    for state_incarnation in [old.incarnation_id(), replacement] {
+                        sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
+                            .await;
+                        // Both a stale runtime and a newly published replacement must reject
+                        // requests already authorized for the deleted incarnation.
+                        for capture in 0..3 {
+                            if capture == 0 && state_incarnation == replacement {
+                                continue;
+                            }
+                            let mut get = if capture == 2 { without_source.clone() } else { get.clone() };
+                            if capture == 1 {
+                                get.extensions.insert(authorized_generation.clone());
+                            }
+                            let mut head = get.clone().map_input(|_| HeadObjectInput {
+                                bucket: input.bucket.clone(),
+                                key: "missing".into(),
+                                ..Default::default()
+                            });
+                            head.method = http::Method::HEAD;
+                            let mut list = get.clone().map_input(|_| input.clone());
+                            list.uri = http::Uri::from_static("/?list-type=2");
+                            let usecase = crate::app::object::DefaultObjectUsecase::from_global();
+                            let get_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_get_object(get))
+                                .await
+                                .expect("GET stays local")
+                                .expect_err("local object is absent");
+                            assert_eq!(*get_error.code(), S3ErrorCode::NoSuchKey);
+                            let head_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_head_object(head))
+                                .await
+                                .expect("HEAD stays local")
+                                .expect_err("local object is absent");
+                            assert_eq!(*head_error.code(), S3ErrorCode::NoSuchKey);
+                            let listing = tokio::time::timeout(
+                                Duration::from_secs(10),
+                                DefaultBucketUsecase::from_global().execute_list_objects_v2(list),
+                            )
+                            .await
+                            .expect("LIST stays local")
+                            .expect("replacement bucket lists locally");
+                            assert_eq!(listing.output.key_count, Some(0));
+                        }
+                    }
+                    stop.cancel();
+                    assert!(server.await.expect("source server must remain unused").is_empty());
+                },
+            )
+            .await;
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn list_objects_v1_stays_local_with_xml_safe_key_markers() {
diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs
index fe3b11e45..0459e2ea6 100644
--- a/rustfs/src/app/bucket_usecase.rs
+++ b/rustfs/src/app/bucket_usecase.rs
@@ -20,7 +20,7 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
 use super::storage_api::bucket_usecase::access::ReqInfo;
 use super::storage_api::bucket_usecase::access::{
     authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
-    prepare_list_buckets_iam_authorization, req_info_ref,
+    prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
@@ -2729,7 +2729,7 @@ impl DefaultBucketUsecase {
 
     async fn execute_list_objects_v2_inner(
         &self,
-        req: S3Request<ListObjectsV2Input>,
+        mut req: S3Request<ListObjectsV2Input>,
         allow_list_through: bool,
     ) -> S3Result<S3Response<ListObjectsV2Output>> {
         let ListObjectsV2Input {
@@ -2742,7 +2742,7 @@ impl DefaultBucketUsecase {
             prefix,
             start_after,
             ..
-        } = req.input;
+        } = req.input.clone();
 
         let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
 
@@ -2757,10 +2757,13 @@ impl DefaultBucketUsecase {
         // The on-demand migration envelope is decoded whether or not this
         // bucket still merges: a token handed out under `list_through` must keep
         // paginating after the policy is turned off (rustfs/backlog#2164).
+        if allow_list_through {
+            prepare_odm_read_generation(&store, &mut req, &bucket).await;
+        }
         let (merged_token, source_state) = if allow_list_through {
             (
                 list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?,
-                list_through::list_through_state(&bucket, &req.headers),
+                list_through::list_through_state(&store, &bucket, &req, &params).await?,
             )
         } else {
             (None, None)
diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs
index 591c4b4c6..aee2fabb1 100644
--- a/rustfs/src/app/object/get.rs
+++ b/rustfs/src/app/object/get.rs
@@ -3843,11 +3843,11 @@ impl DefaultObjectUsecase {
         if !odm_get_may_consult_source(opts, part_number) {
             return None;
         }
-        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
-        let (state, client) = match odm_get_verdict(lookup) {
-            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
-            OdmGetVerdict::Consult { state, client } => (state, client),
-        };
+        let sys = OnDemandMigrationSys::get();
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
         let policy = &state.config().policy;
         // The read path reports a latest delete marker as a plain 404, so the
         // marker is classified here, and only where one can exist.
@@ -3861,6 +3861,24 @@ impl DefaultObjectUsecase {
                 None => return Some(OdmGetOutcome::RetryLocal),
             }
         }
+        let expected_incarnation = match odm_read_generation(req, bucket) {
+            Ok(Some(incarnation)) => incarnation,
+            Ok(None) => return None,
+            Err(err) => return Some(OdmGetOutcome::Respond(Err(err))),
+        };
+        match store.bucket_incarnation_id(bucket).await {
+            Ok(current) if current == expected_incarnation => {}
+            Ok(_) => return None,
+            Err(err) => return Some(OdmGetOutcome::Respond(Err(ApiError::from(err).into()))),
+        }
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
+        let (state, client) = match odm_get_verdict(lookup) {
+            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
+            OdmGetVerdict::Consult { state, client } => (state, client),
+        };
         let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         let reply = odm_get_from_source(&state, client.as_ref(), &req.headers, key, range, request_context).await;
         Some(match reply {
@@ -3895,7 +3913,7 @@ impl DefaultObjectUsecase {
         result
     }
 
-    async fn execute_get_object_inner(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
+    async fn execute_get_object_inner(&self, mut req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
         let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event();
 
         if let Some(context) = &self.context {
@@ -3981,6 +3999,8 @@ impl DefaultObjectUsecase {
                 return Self::complete_get_object_error(helper, err);
             }
         };
+        let bucket = req.input.bucket.clone();
+        prepare_odm_read_generation(&store, &mut req, &bucket).await;
         if let Some(request_context_start) = request_context_start {
             rustfs_io_metrics::record_get_object_stage_duration(
                 "s3_handler",
@@ -4919,7 +4939,7 @@ mod on_demand_migration_tests {
             Err(WriteBackError::Local("multipart is not part of the inline path".to_string()))
         }
 
-        async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, _upload_id: &str) -> Result<(), WriteBackError> {
+        async fn abort_multipart_upload(&self, _request: &WriteBackRequest, _upload_id: &str) -> Result<(), WriteBackError> {
             Ok(())
         }
     }
diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs
index 7cb38bbbc..f20eb895a 100644
--- a/rustfs/src/app/object/head.rs
+++ b/rustfs/src/app/object/head.rs
@@ -146,6 +146,8 @@ impl DefaultObjectUsecase {
     /// `None` means the runtime does not intervene and the caller keeps its
     /// original 404. The source answer is never written back or queued.
     async fn on_demand_migration_head(
+        req: &S3Request<HeadObjectInput>,
+        store: &ECStore,
         bucket: &str,
         key: &str,
         opts: &ObjectOptions,
@@ -154,7 +156,33 @@ impl DefaultObjectUsecase {
         if !odm_request_may_consult_source(opts) {
             return None;
         }
-        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
+        let sys = OnDemandMigrationSys::get();
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
+        let policy = &state.config().policy;
+        if !odm_policy_admits_miss(policy, miss) {
+            return None;
+        }
+        if policy.head == HeadPolicy::LocalOnly {
+            state.stats().record_request(OdmOp::Head, OdmOutcome::Filtered);
+            return None;
+        }
+        let expected_incarnation = match odm_read_generation(req, bucket) {
+            Ok(Some(incarnation)) => incarnation,
+            Ok(None) => return None,
+            Err(err) => return Some(Err(err)),
+        };
+        match store.bucket_incarnation_id(bucket).await {
+            Ok(current) if current == expected_incarnation => {}
+            Ok(_) => return None,
+            Err(err) => return Some(Err(ApiError::from(err).into())),
+        }
+        if !sys.is_module_enabled() {
+            return None;
+        }
+        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
         match odm_head_verdict(lookup, miss) {
             OdmHeadVerdict::Ignore => None,
             OdmHeadVerdict::Fail(err) => Some(Err(err)),
@@ -269,7 +297,7 @@ impl DefaultObjectUsecase {
     }
 
     #[instrument(level = "debug", skip(self, req))]
-    pub async fn execute_head_object(&self, req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
+    pub async fn execute_head_object(&self, mut req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
         if let Some(context) = &self.context {
             let _ = context.object_store();
         }
@@ -314,6 +342,8 @@ impl DefaultObjectUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        prepare_odm_read_generation(&store, &mut req, &bucket).await;
+
         // Modification Points: Explicitly handles get_object_info errors, distinguishing between object absence and other errors
         let lookup = store.get_object_info(&bucket, &key, &opts).await;
         // Single classification point for the on-demand migration gate
@@ -347,7 +377,7 @@ impl DefaultObjectUsecase {
                         return result;
                     }
                     if let Some(miss) = odm_miss
-                        && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
+                        && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
                     {
                         return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                     }
@@ -362,7 +392,7 @@ impl DefaultObjectUsecase {
                 // A latest delete marker is a local miss the source may still
                 // answer when the bucket policy says so.
                 if let Some(miss) = odm_miss
-                    && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
+                    && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
                 {
                     return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                 }
@@ -1053,7 +1083,12 @@ mod tests {
             head: HeadPolicy::LocalOnly,
             ..Default::default()
         });
-        sys.apply(&bucket, Some(&cfg)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            Some(&cfg),
+        )
+        .await;
         let state = sys.state(&bucket).expect("bucket runtime installed");
 
         // Local hit: served locally, the runtime is never entered.
@@ -1109,7 +1144,12 @@ mod tests {
         );
 
         cfg.policy.respect_local_delete_marker = false;
-        sys.apply(&bucket, Some(&cfg)).await;
+        sys.apply_for_incarnation(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            Some(&cfg),
+        )
+        .await;
         let err = Box::pin(usecase.execute_head_object(head_input(&bucket, "present", None)))
             .await
             .expect_err("local_only still answers 404");
diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs
index bfd01429f..eb93d2610 100644
--- a/rustfs/src/app/object/internal_put.rs
+++ b/rustfs/src/app/object/internal_put.rs
@@ -48,6 +48,8 @@ use http::HeaderName;
 /// other internal provenance is written verbatim.
 pub(crate) struct InternalPutContext {
     pub(crate) bucket: String,
+    /// Pins background work to its original bucket across deletion and recreation.
+    pub(crate) expected_bucket_incarnation_id: Option<Uuid>,
     pub(crate) key: String,
     /// Plaintext object length. The single-object path requires it, exactly
     /// like S3 PutObject rejects an unknown `Content-Length`.
@@ -239,6 +241,7 @@ impl DefaultObjectUsecase {
         let start_time = Instant::now();
         let InternalPutContext {
             bucket,
+            expected_bucket_incarnation_id,
             key,
             size,
             expected_md5_hex,
@@ -296,6 +299,7 @@ impl DefaultObjectUsecase {
                 principal_id,
                 emit_events,
                 preserve_delete_marker,
+                expected_bucket_incarnation_id,
             },
         };
         let committed = self
@@ -367,6 +371,8 @@ impl DefaultObjectUsecase {
             .await
             .map_err(ApiError::from)?;
 
+        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
+
         let dsc = must_replicate_object(
             &ctx.bucket,
             &ctx.key,
@@ -428,7 +434,10 @@ impl DefaultObjectUsecase {
         let bucket = ctx.bucket.as_str();
         let key = ctx.key.as_str();
         let store = self.object_store().ok_or_else(not_initialized)?;
-        let mut opts = ObjectOptions::default();
+        let mut opts = ObjectOptions {
+            expected_bucket_incarnation_id: ctx.expected_bucket_incarnation_id,
+            ..Default::default()
+        };
         let session = store
             .get_multipart_info(bucket, key, upload_id, &opts)
             .await
@@ -542,6 +551,7 @@ impl DefaultObjectUsecase {
         }
         let mut opts =
             get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?;
+        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
         opts.preserve_etag = ctx.preserve_etag.clone();
         opts.preserve_delete_marker = ctx.preserve_delete_marker;
         let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
@@ -699,10 +709,24 @@ impl DefaultObjectUsecase {
     }
 
     /// Discard an internal multipart upload and its staged parts.
-    pub(crate) async fn internal_abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), ApiError> {
+    pub(crate) async fn internal_abort_multipart_upload(
+        &self,
+        bucket: &str,
+        key: &str,
+        upload_id: &str,
+        expected_bucket_incarnation_id: Option<Uuid>,
+    ) -> Result<(), ApiError> {
         let store = self.object_store().ok_or_else(not_initialized)?;
         store
-            .abort_multipart_upload(bucket, key, upload_id, &ObjectOptions::default())
+            .abort_multipart_upload(
+                bucket,
+                key,
+                upload_id,
+                &ObjectOptions {
+                    expected_bucket_incarnation_id,
+                    ..Default::default()
+                },
+            )
             .await
             .map_err(ApiError::from)?;
         rustfs_scanner::record_dirty_usage_bucket(bucket);
@@ -756,6 +780,7 @@ mod tests {
     fn internal_context(bucket: &str, key: &str, body: &[u8]) -> InternalPutContext {
         InternalPutContext {
             bucket: bucket.to_string(),
+            expected_bucket_incarnation_id: None,
             key: key.to_string(),
             size: Some(body.len() as u64),
             expected_md5_hex: Some(md5_hex(body)),
@@ -1138,9 +1163,14 @@ mod tests {
         ))
         .await
         .expect("part of the aborted upload must stage");
-        Box::pin(usecase.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id))
-            .await
-            .expect("internal abort must succeed");
+        Box::pin(usecase.internal_abort_multipart_upload(
+            &bucket,
+            &ctx.key,
+            &aborted_upload_id,
+            ctx.expected_bucket_incarnation_id,
+        ))
+        .await
+        .expect("internal abort must succeed");
         let uploads = Box::pin(store.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100))
             .await
             .expect("list multipart uploads after abort");
diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs
index d5751d5c4..b9ea67269 100644
--- a/rustfs/src/app/object/mod.rs
+++ b/rustfs/src/app/object/mod.rs
@@ -21,8 +21,8 @@ use crate::storage_api::table::get_bucket_metadata;
 
 use super::storage_api::object_usecase::access::{
     PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
-    has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized,
-    replication_request_authorized, req_info_mut, req_info_ref,
+    has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
+    recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::object_usecase::bucket::quota::BucketQuota;
diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs
index 50e87b5cf..86ba3972a 100644
--- a/rustfs/src/app/object/on_demand_migration_put.rs
+++ b/rustfs/src/app/object/on_demand_migration_put.rs
@@ -172,6 +172,7 @@ pub(super) async fn write_back_context(request: &WriteBackRequest, single_part:
     };
     InternalPutContext {
         bucket: request.bucket.clone(),
+        expected_bucket_incarnation_id: Some(request.bucket_incarnation_id),
         key: request.key.clone(),
         size: Some(head.size),
         expected_md5_hex: single_part.then(|| expected_md5_hex(head)).flatten(),
@@ -285,9 +286,9 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
             .map_err(write_back_error)
     }
 
-    async fn abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), WriteBackError> {
+    async fn abort_multipart_upload(&self, request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError> {
         self.usecase()
-            .internal_abort_multipart_upload(bucket, key, upload_id)
+            .internal_abort_multipart_upload(&request.bucket, &request.key, upload_id, Some(request.bucket_incarnation_id))
             .await
             .map_err(write_back_error)
     }
@@ -303,7 +304,7 @@ mod tests {
         ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
     };
     use crate::app::storage_api::test::bucket::utils::serialize;
-    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
     use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
     use crate::on_demand_migration::{PullFailureReason, SourceSse};
     use http::Method;
@@ -345,9 +346,10 @@ mod tests {
         }
     }
 
-    fn request(bucket: &str, key: &str, head: SourceHead) -> WriteBackRequest {
+    fn request(bucket: &str, bucket_incarnation_id: Uuid, key: &str, head: SourceHead) -> WriteBackRequest {
         WriteBackRequest {
             bucket: bucket.to_string(),
+            bucket_incarnation_id,
             key: key.to_string(),
             head,
             source_label: SOURCE_LABEL.to_string(),
@@ -475,7 +477,15 @@ mod tests {
         let body = b"pulled from the legacy bucket".to_vec();
         let head = source_head(&body);
         let outcome = write_back
-            .put_object(&request(&bucket, "dir/obj.txt", head.clone()), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "dir/obj.txt",
+                    head.clone(),
+                ),
+                body_stream(&body),
+            )
             .await
             .expect("write-back must commit");
         assert_eq!(outcome.etag, head.etag, "single-part source ETag is preserved");
@@ -518,7 +528,12 @@ mod tests {
             .await
             .expect("bucket");
         let write_back = OnDemandMigrationWriteBack::new();
-        let req = request(bucket, "key", source_head(b"source"));
+        let req = request(
+            bucket,
+            store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"),
+            "key",
+            source_head(b"source"),
+        );
         assert!(matches!(
             write_back.put_object(&req, body_stream(b"source")).await,
             Err(WriteBackError::Unsupported(_))
@@ -534,6 +549,75 @@ mod tests {
         assert_nothing_left(&store, bucket, "key").await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn stale_write_back_cannot_mutate_a_recreated_bucket() {
+        let (store, bucket) = write_back_test_bucket("odm-wb-incarnation", false).await;
+        let old_id = store.bucket_incarnation_id(&bucket).await.expect("old bucket incarnation");
+        let stale = request(&bucket, old_id, "object", source_head(b"source"));
+        let (resume, wait) = tokio::sync::oneshot::channel();
+        let delayed = {
+            let stale = stale.clone();
+            tokio::spawn(async move {
+                wait.await.expect("resume old source pull");
+                OnDemandMigrationWriteBack::new()
+                    .put_object(&stale, body_stream(b"source"))
+                    .await
+            })
+        };
+        store
+            .delete_bucket(
+                &bucket,
+                &DeleteBucketOptions {
+                    force: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("delete original bucket");
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let new_id = store.bucket_incarnation_id(&bucket).await.expect("replacement incarnation");
+        assert_ne!(old_id, new_id);
+        resume.send(()).expect("release delayed pull");
+        assert!(delayed.await.expect("delayed pull task").is_err());
+        assert_nothing_left(&store, &bucket, "object").await;
+
+        let write_back = OnDemandMigrationWriteBack::new();
+        assert!(write_back.create_multipart_upload(&stale).await.is_err());
+        let current = request(&bucket, new_id, "object", source_head(b"current"));
+        let upload = write_back
+            .create_multipart_upload(&current)
+            .await
+            .expect("create replacement upload");
+        // A stale capability must fail independently of whether its upload ID
+        // happens to name a valid session in the replacement bucket.
+        assert!(
+            write_back
+                .upload_part(&stale, &upload, 1, 6, body_stream(b"source"))
+                .await
+                .is_err()
+        );
+        let part = write_back
+            .upload_part(&current, &upload, 1, 7, body_stream(b"current"))
+            .await
+            .expect("stage current part");
+        assert!(
+            write_back
+                .complete_multipart_upload(&stale, &upload, vec![part.clone()])
+                .await
+                .is_err()
+        );
+        assert!(write_back.abort_multipart_upload(&stale, &upload).await.is_err());
+        write_back
+            .complete_multipart_upload(&current, &upload, vec![part])
+            .await
+            .expect("stale cleanup preserves replacement upload");
+        assert_eq!(raw_object_bytes(&store, &bucket, "object").await, b"current");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn write_back_integrity_failure_leaves_nothing_behind() {
@@ -543,7 +627,15 @@ mod tests {
         head.etag = Some(md5_hex(b"a different body"));
 
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "wrong.bin", head), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "wrong.bin",
+                    head,
+                ),
+                body_stream(&body),
+            )
             .await
             .expect_err("an ETag mismatch must fail the write-back");
         assert_eq!(err, WriteBackError::Integrity);
@@ -559,8 +651,18 @@ mod tests {
             let (store, bucket) = write_back_test_bucket("odm-wb-race", versioned).await;
             let source = b"old source bytes";
             let client = b"new client bytes";
-            let req = request(&bucket, "race", source_head(source));
-            let client_req = request(&bucket, "race", source_head(client));
+            let req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(source),
+            );
+            let client_req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(client),
+            );
             let mut client_ctx = write_back_context(&client_req, true).await;
             client_ctx.if_absent = false;
             let client_after = PutObjectCommitBarrier::install(&bucket, "race", PutObjectCommitPause::AfterNamespace);
@@ -595,13 +697,27 @@ mod tests {
     async fn write_back_multipart_completion_preserves_a_client_put_after_staging() {
         let (store, bucket) = write_back_test_bucket("odm-mpu-race", false).await;
         let write_back = OnDemandMigrationWriteBack::new();
-        let req = request(&bucket, "race", source_head(b"source"));
+        let req = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "race",
+            source_head(b"source"),
+        );
         let upload_id = write_back.create_multipart_upload(&req).await.expect("create");
         let part = write_back
             .upload_part(&req, &upload_id, 1, 6, body_stream(b"source"))
             .await
             .expect("stage");
-        let mut client_ctx = write_back_context(&request(&bucket, "race", source_head(b"client")), true).await;
+        let mut client_ctx = write_back_context(
+            &request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "race",
+                source_head(b"client"),
+            ),
+            true,
+        )
+        .await;
         client_ctx.if_absent = false;
         let committed = DefaultObjectUsecase::from_global()
             .internal_put_object(client_ctx, body_stream(b"client"))
@@ -613,7 +729,7 @@ mod tests {
             "{result:?}"
         );
         write_back
-            .abort_multipart_upload(&bucket, "race", &upload_id)
+            .abort_multipart_upload(&req, &upload_id)
             .await
             .expect("abort rejected upload");
         let stored = stored_object(&store, &bucket, "race").await;
@@ -628,7 +744,12 @@ mod tests {
         for multipart in [false, true] {
             let (store, bucket) = write_back_test_bucket("odm-wb-tombstone", true).await;
             let write_back = OnDemandMigrationWriteBack::new();
-            let mut req = request(&bucket, "deleted", source_head(b"source"));
+            let mut req = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "deleted",
+                source_head(b"source"),
+            );
             let staged = if multipart {
                 let id = write_back.create_multipart_upload(&req).await.expect("create");
                 let part = write_back
@@ -654,10 +775,7 @@ mod tests {
             assert!(marker.delete_marker);
             let rejected = if let Some((id, part)) = staged {
                 let result = write_back.complete_multipart_upload(&req, &id, vec![part]).await;
-                write_back
-                    .abort_multipart_upload(&bucket, "deleted", &id)
-                    .await
-                    .expect("abort");
+                write_back.abort_multipart_upload(&req, &id).await.expect("abort");
                 result
             } else {
                 write_back.put_object(&req, body_stream(b"source")).await
@@ -691,7 +809,15 @@ mod tests {
             Err(io::Error::new(io::ErrorKind::BrokenPipe, "tee primary dropped before EOF")),
         ]);
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "torn.bin", head.clone()), torn)
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "torn.bin",
+                    head.clone(),
+                ),
+                torn,
+            )
             .await
             .expect_err("a broken stream must fail the write-back");
         assert_ne!(err, WriteBackError::Integrity, "{err}");
@@ -700,7 +826,15 @@ mod tests {
         // A clean EOF short of the advertised size is just as fatal.
         let short = stream(vec![Ok(Bytes::copy_from_slice(&body[..64 * 1024]))]);
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "short.bin", head), short)
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "short.bin",
+                    head,
+                ),
+                short,
+            )
             .await
             .expect_err("a short body must fail the write-back");
         assert!(matches!(err, WriteBackError::Local(_) | WriteBackError::Integrity), "{err}");
@@ -716,7 +850,12 @@ mod tests {
         let mut head = source_head(&body);
         head.etag = Some(format!("{}-2", md5_hex(&body)));
         head.is_multipart_etag = true;
-        let request = request(&bucket, "big/object.bin", head.clone());
+        let request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "big/object.bin",
+            head.clone(),
+        );
         let write_back = OnDemandMigrationWriteBack::new();
 
         let upload_id = write_back.create_multipart_upload(&request).await.expect("create");
@@ -754,10 +893,7 @@ mod tests {
             .upload_part(&request, &aborted, 1, 4096, body_stream(&body[..4096]))
             .await
             .expect("stage part");
-        write_back
-            .abort_multipart_upload(&bucket, "big/object.bin", &aborted)
-            .await
-            .expect("abort");
+        write_back.abort_multipart_upload(&request, &aborted).await.expect("abort");
         let uploads = store
             .list_multipart_uploads(&bucket, "big/object.bin", None, None, None, 100)
             .await
@@ -810,7 +946,12 @@ mod tests {
 
             let body = b"plaintext that must be encrypted at rest".to_vec();
             let head = source_head(&body);
-            let request = request(&bucket, "secret.txt", head.clone());
+            let request = request(
+                &bucket,
+                store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                "secret.txt",
+                head.clone(),
+            );
             // The source ETag is not forced onto an encrypted object; the
             // local ETag is whatever the SSE write path computes.
             assert_eq!(write_back_context(&request, true).await.preserve_etag, None);
@@ -848,7 +989,15 @@ mod tests {
         let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("odm-wb-quota", 64).await;
         let body = vec![0x71; 4096];
         let err = OnDemandMigrationWriteBack::new()
-            .put_object(&request(&bucket, "over.bin", source_head(&body)), body_stream(&body))
+            .put_object(
+                &request(
+                    &bucket,
+                    store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+                    "over.bin",
+                    source_head(&body),
+                ),
+                body_stream(&body),
+            )
             .await
             .expect_err("a full quota must reject the write-back");
         assert!(matches!(err, WriteBackError::Quota(_)), "{err}");
@@ -913,7 +1062,12 @@ mod tests {
 
         let body = b"replicate me".to_vec();
         let head = source_head(&body);
-        let request = request(&bucket, "replicated.txt", head);
+        let request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "replicated.txt",
+            head,
+        );
         let ctx = write_back_context(&request, true).await;
         assert!(ctx.emit_events, "policy.emit_events reaches the creation event");
         assert_eq!(ctx.principal_id, ON_DEMAND_MIGRATION_PRINCIPAL_ID);
@@ -961,7 +1115,12 @@ mod tests {
         head.user_metadata
             .insert(forged_replica_key.to_string(), ReplicationStatusType::Replica.as_str().to_string());
 
-        let mut write_request = request(&bucket, "unadmitted.txt", head);
+        let mut write_request = request(
+            &bucket,
+            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
+            "unadmitted.txt",
+            head,
+        );
         write_request.tags = Some(HashMap::from([("replicate".to_string(), "no".to_string())]));
         OnDemandMigrationWriteBack::new()
             .put_object(&write_request, body_stream(&body))
@@ -1059,7 +1218,7 @@ mod tests {
 
     #[test]
     fn provenance_and_tags_are_stable() {
-        let mut request = request("b", "k", source_head(b"x"));
+        let mut request = request("b", Uuid::nil(), "k", source_head(b"x"));
         let metadata = provenance_metadata(&request);
         assert_eq!(metadata.len(), 10, "five keys under two prefixes");
         assert_provenance(&metadata, &request.head);
@@ -1080,7 +1239,7 @@ mod tests {
     #[tokio::test]
     async fn write_back_context_applies_the_etag_and_event_policy() {
         let body = b"context".to_vec();
-        let mut request = request("no-such-bucket", "k", source_head(&body));
+        let mut request = request("no-such-bucket", Uuid::nil(), "k", source_head(&body));
         let ctx = write_back_context(&request, true).await;
         assert_eq!(ctx.expected_md5_hex, Some(md5_hex(&body)));
         assert_eq!(ctx.preserve_etag, Some(md5_hex(&body)));
diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs
index 443605791..f1873beea 100644
--- a/rustfs/src/app/object/put.rs
+++ b/rustfs/src/app/object/put.rs
@@ -953,6 +953,7 @@ pub(super) enum PutObjectOrigin<'a> {
         principal_id: &'static str,
         emit_events: bool,
         preserve_delete_marker: bool,
+        expected_bucket_incarnation_id: Option<Uuid>,
     },
 }
 
@@ -967,7 +968,13 @@ impl PutObjectOrigin<'_> {
     fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> {
         match self {
             Self::S3 { req, .. } => apply_bucket_generation_guard(req, bucket, opts),
-            Self::Internal { .. } => Ok(()),
+            Self::Internal {
+                expected_bucket_incarnation_id,
+                ..
+            } => {
+                opts.expected_bucket_incarnation_id = *expected_bucket_incarnation_id;
+                Ok(())
+            }
         }
     }
 
diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs
index da7af7c86..4e5ae8cc4 100644
--- a/rustfs/src/app/storage_api.rs
+++ b/rustfs/src/app/storage_api.rs
@@ -27,19 +27,20 @@ pub(crate) fn EndpointServerPools(
 /// S3 wire types for app-layer modules, funneled here so new files stay off
 /// the direct s3s surface (s3s footprint ratchet, `scripts/check_s3s_footprint.sh`).
 pub(crate) mod s3 {
+    #[cfg(test)]
+    pub(crate) use s3s::S3Response;
     #[cfg(test)]
     pub(crate) use s3s::dto::ListObjectsInput;
     #[cfg(test)]
     pub(crate) use s3s::dto::{
-        BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
-        ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
-        ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
+        BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, GetObjectInput,
+        HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output, ReplicationConfiguration, ReplicationRule,
+        ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
+        ServerSideEncryptionRule, Tag, VersioningConfiguration,
     };
     #[cfg(test)]
     pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};
-    pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
-    #[cfg(test)]
-    pub(crate) use s3s::{S3Request, S3Response};
+    pub(crate) use s3s::{S3Error, S3ErrorCode, S3Request, S3Result};
 }
 
 pub(crate) mod admin {
@@ -266,8 +267,9 @@ pub(crate) mod access {
     pub(crate) use crate::storage::storage_api::access_consumer::{
         PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
         bucket_config_mutation_incarnation, has_bypass_governance_header, load_bucket_generation_from_store,
-        log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, recursive_force_delete_is_authorized,
-        replication_request_authorized, req_info_mut, req_info_ref,
+        log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
+        prepare_odm_read_generation, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut,
+        req_info_ref,
     };
 }
 
diff --git a/rustfs/src/on_demand_migration/backfill.rs b/rustfs/src/on_demand_migration/backfill.rs
index f2d14b61d..28bb85219 100644
--- a/rustfs/src/on_demand_migration/backfill.rs
+++ b/rustfs/src/on_demand_migration/backfill.rs
@@ -370,6 +370,8 @@ pub type PullReport = Option<super::pull::QueuedPullReport>;
 /// mock in unit tests. Production: [`BucketBackfillContext`].
 #[async_trait]
 pub trait BackfillContext: Send + Sync {
+    /// The bucket incarnation captured by this context.
+    fn incarnation_id(&self) -> Uuid;
     /// One source page in the local key namespace.
     async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError>;
     /// Whether the breaker admits source traffic right now.
@@ -411,6 +413,10 @@ impl BucketBackfillContext {
 
 #[async_trait]
 impl BackfillContext for BucketBackfillContext {
+    fn incarnation_id(&self) -> Uuid {
+        self.state.incarnation_id()
+    }
+
     async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
         let client = self.state.client().map_err(|err| SourceError::Unsupported(err.to_string()))?;
         let started = Instant::now();
@@ -661,8 +667,36 @@ pub async fn read_checkpoint(api: &Arc<ECStore>, bucket: &str) -> Result<Option<
 async fn write_checkpoint(
     api: &Arc<ECStore>,
     bucket: &str,
+    incarnation_id: Uuid,
     checkpoint: &BackfillCheckpoint,
     expected_etag: Option<&str>,
+) -> Result<String, BackfillError> {
+    let api = Arc::clone(api);
+    let bucket = bucket.to_string();
+    let checkpoint = checkpoint.clone();
+    let expected_etag = expected_etag.map(str::to_string);
+    // The storage commit owns detached work. Keep its user-bucket fence alive
+    // even when a caller aborts its waiter before the erasure tail has drained.
+    tokio::spawn(async move {
+        let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
+        let mut opts = ObjectOptions::default();
+        fence.attach_to_object_options(&mut opts);
+        let result = write_checkpoint_while_fenced(&api, &bucket, &checkpoint, expected_etag.as_deref(), opts).await;
+        drop(fence);
+        result
+    })
+    .await
+    .map_err(|err| StorageError::other(format!("backfill checkpoint task failed: {err}")))?
+}
+
+/// The caller holds the destination bucket's lifecycle fence through the CAS
+/// write and its read-back, including the drained erasure write tail.
+async fn write_checkpoint_while_fenced(
+    api: &Arc<ECStore>,
+    bucket: &str,
+    checkpoint: &BackfillCheckpoint,
+    expected_etag: Option<&str>,
+    mut opts: ObjectOptions,
 ) -> Result<String, BackfillError> {
     let data = checkpoint.to_json()?;
     let preconditions = match expected_etag {
@@ -675,12 +709,9 @@ async fn write_checkpoint(
             ..Default::default()
         },
     };
-    let opts = ObjectOptions {
-        max_parity: true,
-        write_completion: WriteCompletion::TailDrained,
-        http_preconditions: Some(preconditions),
-        ..Default::default()
-    };
+    opts.max_parity = true;
+    opts.write_completion = WriteCompletion::TailDrained;
+    opts.http_preconditions = Some(preconditions);
     match save_config_with_opts(Arc::clone(api), &checkpoint_path(bucket), data, &opts).await {
         Ok(()) => {}
         Err(StorageError::PreconditionFailed) => return Err(BackfillError::Conflict(bucket.to_string())),
@@ -851,7 +882,14 @@ impl BackfillRunner {
             });
         }
         let checkpoint = BackfillCheckpoint::new(&request, config_updated_at, &self.node, now);
-        let etag = write_checkpoint(&self.api, bucket, &checkpoint, stored.as_ref().map(|s| s.etag.as_str())).await?;
+        let etag = write_checkpoint(
+            &self.api,
+            bucket,
+            context.incarnation_id(),
+            &checkpoint,
+            stored.as_ref().map(|s| s.etag.as_str()),
+        )
+        .await?;
         info!(
             event = EVENT_ODM_BACKFILL_STATE,
             component = LOG_COMPONENT_ECSTORE,
@@ -886,30 +924,42 @@ impl BackfillRunner {
             }
             return Ok(handle.snapshot.lock().clone());
         }
-        let _lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
-        let Some(stored) = read_checkpoint(&self.api, bucket).await? else {
-            return Err(BackfillError::NotFound(bucket.to_string()));
-        };
-        if !stored.checkpoint.state.is_active() {
-            return Ok(stored.checkpoint);
-        }
-        let mut checkpoint = stored.checkpoint;
-        let now = OffsetDateTime::now_utc();
-        checkpoint.state = BackfillState::Cancelled;
-        checkpoint.updated_at = now;
-        write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
-        info!(
-            event = EVENT_ODM_BACKFILL_STATE,
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
-            state = checkpoint.state.as_str(),
-            result = "cancelled",
-            bucket = %bucket,
-            job_id = %checkpoint.job_id,
-            owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
-            "On-demand migration backfill job cancelled remotely"
-        );
-        Ok(checkpoint)
+        let incarnation_id = self.api.bucket_incarnation_id_from_disk(bucket).await?;
+        let lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
+        let api = Arc::clone(&self.api);
+        let bucket = bucket.to_string();
+        tokio::spawn(async move {
+            let _lock = lock;
+            let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
+            let mut opts = ObjectOptions::default();
+            fence.attach_to_object_options(&mut opts);
+            let Some(stored) = read_checkpoint(&api, &bucket).await? else {
+                return Err(BackfillError::NotFound(bucket.to_string()));
+            };
+            if !stored.checkpoint.state.is_active() {
+                return Ok(stored.checkpoint);
+            }
+            let mut checkpoint = stored.checkpoint;
+            let now = OffsetDateTime::now_utc();
+            checkpoint.state = BackfillState::Cancelled;
+            checkpoint.updated_at = now;
+            write_checkpoint_while_fenced(&api, &bucket, &checkpoint, Some(&stored.etag), opts).await?;
+            info!(
+                event = EVENT_ODM_BACKFILL_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
+                state = checkpoint.state.as_str(),
+                result = "cancelled",
+                bucket = %bucket,
+                job_id = %checkpoint.job_id,
+                owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
+                "On-demand migration backfill job cancelled remotely"
+            );
+            drop(fence);
+            Ok(checkpoint)
+        })
+        .await
+        .map_err(|err| StorageError::other(format!("backfill cancellation task failed: {err}")))?
     }
 
     /// Latest checkpoint: the in-memory progress of a local job, else the
@@ -1002,7 +1052,7 @@ impl BackfillRunner {
             checkpoint.state = BackfillState::Cancelled;
             checkpoint.updated_at = now;
             checkpoint.record_failure("config_changed", None, now);
-            write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
+            write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
             info!(
                 event = EVENT_ODM_BACKFILL_STATE,
                 component = LOG_COMPONENT_ECSTORE,
@@ -1021,7 +1071,7 @@ impl BackfillRunner {
             node: self.node.clone(),
             lease_until: now + BACKFILL_LEASE,
         });
-        let etag = write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
+        let etag = write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
         warn!(
             event = EVENT_ODM_BACKFILL_LEASE_TAKEOVER,
             component = LOG_COMPONENT_ECSTORE,
@@ -1460,7 +1510,8 @@ impl Job {
                 lease_until: now + BACKFILL_LEASE,
             });
         }
-        let etag = write_checkpoint(&self.api, &self.bucket, &self.checkpoint, Some(&self.etag)).await?;
+        let etag =
+            write_checkpoint(&self.api, &self.bucket, self.context.incarnation_id(), &self.checkpoint, Some(&self.etag)).await?;
         self.etag = etag;
         self.keys_since_save = 0;
         self.last_save = Instant::now();
@@ -1500,7 +1551,10 @@ pub async fn run_backfill_recovery_loop(runner: Arc<BackfillRunner>, cancel: Can
 
 #[cfg(test)]
 mod tests {
-    use super::super::storage_api::test_support::isolated_store_over_temp_disks;
+    use super::super::storage_api::test_support::{
+        BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations as _, PutObjectCommitBarrier, PutObjectCommitPause,
+        isolated_store_over_temp_disks,
+    };
     use super::*;
     use crate::on_demand_migration::source_client::SourceObject;
     use crate::on_demand_migration::sys::PullError;
@@ -1626,6 +1680,7 @@ mod tests {
 
     /// Scripted source + local store + queue with a controllable report path.
     struct MockContext {
+        incarnation_id: Mutex<Option<Uuid>>,
         objects: Vec<SourceObject>,
         page_size: usize,
         local: Mutex<HashMap<String, LocalBackfillObject>>,
@@ -1654,6 +1709,7 @@ mod tests {
                 })
                 .collect();
             Arc::new(Self {
+                incarnation_id: Mutex::new(None),
                 objects,
                 page_size,
                 local: Mutex::new(HashMap::new()),
@@ -1688,6 +1744,10 @@ mod tests {
 
     #[async_trait]
     impl BackfillContext for MockContext {
+        fn incarnation_id(&self) -> Uuid {
+            self.incarnation_id.lock().expect("test bucket initialized")
+        }
+
         async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
             if let Some(err) = self.list_error.lock().take() {
                 return Err(err);
@@ -1780,12 +1840,17 @@ mod tests {
         context: Arc<MockContext>,
     ) -> (Vec<tempfile::TempDir>, Arc<ECStore>, Arc<BackfillRunner>) {
         let (dirs, store) = isolated_store_over_temp_disks().await;
-        // The isolated store has no bucket metadata system; the checkpoint
-        // only needs the bucket's directory under the metadata volume.
-        for dir in &dirs {
-            std::fs::create_dir_all(dir.path().join(RUSTFS_META_BUCKET).join(BUCKET_META_PREFIX).join(bucket))
-                .expect("test bucket metadata directory");
-        }
+        super::super::storage_api::test_support::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        store
+            .make_bucket(bucket, &Default::default())
+            .await
+            .expect("create test bucket");
+        *context.incarnation_id.lock() = Some(
+            store
+                .bucket_incarnation_id_from_disk(bucket)
+                .await
+                .expect("test bucket identity"),
+        );
         let runner = runner_on(node, bucket, context, Arc::clone(&store));
         (dirs, store, runner)
     }
@@ -1795,6 +1860,129 @@ mod tests {
         BackfillRunner::new(store, node, Arc::new(contexts))
     }
 
+    #[tokio::test]
+    async fn cancelled_checkpoint_waiter_keeps_bucket_fenced_until_commit_finishes() {
+        for (suffix, pause) in [
+            ("before", PutObjectCommitPause::BeforeQuotaRename),
+            ("after", PutObjectCommitPause::AfterRenameQuorum),
+        ] {
+            let bucket = format!("backfill-cancel-tail-{suffix}");
+            let context = MockContext::new(0, 1);
+            let (_dirs, store, _runner) = runner_with("node-a", &bucket, Arc::clone(&context)).await;
+            let original_incarnation = context.incarnation_id();
+            let checkpoint = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
+            let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, &checkpoint_path(&bucket), pause);
+            let writer_api = Arc::clone(&store);
+            let writer_bucket = bucket.clone();
+            let waiter = tokio::spawn(async move {
+                write_checkpoint(&writer_api, &writer_bucket, original_incarnation, &checkpoint, None).await
+            });
+            barrier.wait_until_paused().await;
+            waiter.abort();
+            assert!(waiter.await.expect_err("caller aborted").is_cancelled());
+
+            let lifecycle_lock = store
+                .new_ns_lock(&bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)
+                .await
+                .expect("lifecycle lock");
+            {
+                let mut probe = Box::pin(lifecycle_lock.get_write_lock(Duration::from_secs(1)));
+                assert!(
+                    futures::poll!(probe.as_mut()).is_pending(),
+                    "lifecycle writer must first try to acquire the lock"
+                );
+                assert!(
+                    tokio::time::timeout(Duration::from_millis(100), probe.as_mut())
+                        .await
+                        .is_err(),
+                    "the checkpoint owner must retain the user bucket lifecycle read lock after caller cancellation"
+                );
+            }
+
+            let delete_api = Arc::clone(&store);
+            let delete_bucket = bucket.clone();
+            let mut deletion = tokio::spawn(async move { delete_api.delete_bucket(&delete_bucket, &Default::default()).await });
+            assert!(
+                tokio::time::timeout(Duration::from_millis(100), &mut deletion).await.is_err(),
+                "DeleteBucket must wait for the checkpoint owner after its caller aborts"
+            );
+            barrier.release();
+            tokio::time::timeout(Duration::from_secs(10), deletion)
+                .await
+                .expect("commit must drain and release its lifecycle guard")
+                .expect("delete task")
+                .expect("delete original bucket");
+            store
+                .make_bucket(&bucket, &Default::default())
+                .await
+                .expect("recreate bucket");
+            assert_ne!(
+                original_incarnation,
+                store.bucket_incarnation_id_from_disk(&bucket).await.expect("new identity")
+            );
+            assert!(
+                read_checkpoint(&store, &bucket)
+                    .await
+                    .expect("read recreated bucket")
+                    .is_none(),
+                "no old checkpoint may outlive bucket deletion"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn stale_checkpoint_writer_cannot_resurrect_or_overwrite_a_recreated_bucket() {
+        let bucket = "backfill-incarnation";
+        let context = MockContext::new(0, 1);
+        let (_dirs, store, _runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
+        let old_incarnation = context.incarnation_id();
+        let old = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
+        let old_etag = write_checkpoint(&store, bucket, old_incarnation, &old, None)
+            .await
+            .expect("old checkpoint");
+
+        store
+            .delete_bucket(bucket, &Default::default())
+            .await
+            .expect("delete original bucket");
+        store.make_bucket(bucket, &Default::default()).await.expect("recreate bucket");
+        let current_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
+        assert_ne!(old_incarnation, current_incarnation);
+        assert!(
+            read_checkpoint(&store, bucket)
+                .await
+                .expect("read after recreation")
+                .is_none()
+        );
+        for expected_etag in [None, Some(old_etag.as_str())] {
+            let error = write_checkpoint(&store, bucket, old_incarnation, &old, expected_etag)
+                .await
+                .expect_err("stale writer rejected");
+            assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
+        }
+        assert!(
+            read_checkpoint(&store, bucket)
+                .await
+                .expect("stale writer left no checkpoint")
+                .is_none()
+        );
+
+        let current = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-b", ts(1_700_000_002));
+        let current_etag = write_checkpoint(&store, bucket, current_incarnation, &current, None)
+            .await
+            .expect("current checkpoint");
+        let error = write_checkpoint(&store, bucket, old_incarnation, &old, Some(&current_etag))
+            .await
+            .expect_err("old identity cannot overwrite a matching ETag");
+        assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
+        let stored = read_checkpoint(&store, bucket)
+            .await
+            .expect("read current checkpoint")
+            .expect("current checkpoint remains");
+        assert_eq!(stored.etag, current_etag);
+        assert_eq!(stored.checkpoint, current);
+    }
+
     #[tokio::test]
     async fn full_backfill_lists_pages_and_counts_every_key() {
         let bucket = "backfill-full";
@@ -2134,7 +2322,7 @@ mod tests {
             node: "node-a".to_string(),
             lease_until: now - Duration::from_secs(120),
         });
-        let etag = write_checkpoint(&store, bucket, &crashed, None)
+        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
             .await
             .expect("seed checkpoint");
 
@@ -2145,7 +2333,7 @@ mod tests {
             lease_until: now + Duration::from_secs(60),
         });
         live.updated_at = now;
-        let etag = write_checkpoint(&store, bucket, &live, Some(&etag))
+        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &live, Some(&etag))
             .await
             .expect("live lease");
         assert_eq!(runner.recover_once().await.taken_over, 0, "unexpired lease must not be taken over");
@@ -2162,7 +2350,7 @@ mod tests {
             lease_until: now - Duration::from_secs(1),
         });
         expired.updated_at = now + Duration::from_millis(1);
-        write_checkpoint(&store, bucket, &expired, Some(&etag))
+        write_checkpoint(&store, bucket, context.incarnation_id(), &expired, Some(&etag))
             .await
             .expect("expire lease");
         let stats = runner.recover_once().await;
@@ -2201,7 +2389,7 @@ mod tests {
         crashed.continuation_token = Some("2".to_string());
         crashed.failed = 1;
         crashed.record_failure("local_write", Some("k/00002"), crashed_at);
-        write_checkpoint(&store, bucket, &crashed, None)
+        write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
             .await
             .expect("seed failed page with an expired lease");
 
@@ -2257,7 +2445,9 @@ mod tests {
 
         // Same node name, unexpired lease: only a restart can produce this.
         let own = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", now);
-        let etag = write_checkpoint(&store, bucket, &own, None).await.expect("seed");
+        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &own, None)
+            .await
+            .expect("seed");
         assert_eq!(runner.recover_once().await.taken_over, 1, "own-node running job is reclaimed at once");
         runner.wait_until_idle(bucket).await;
         let cp = read_checkpoint(&store, bucket)
@@ -2275,7 +2465,7 @@ mod tests {
             node: "node-z".to_string(),
             lease_until: now - Duration::from_secs(1),
         });
-        write_checkpoint(&store, bucket, &stale, Some(&stored.etag))
+        write_checkpoint(&store, bucket, context.incarnation_id(), &stale, Some(&stored.etag))
             .await
             .expect("seed stale");
         let stats = runner.recover_once().await;
diff --git a/rustfs/src/on_demand_migration/pull.rs b/rustfs/src/on_demand_migration/pull.rs
index 9d52af009..9e7525642 100644
--- a/rustfs/src/on_demand_migration/pull.rs
+++ b/rustfs/src/on_demand_migration/pull.rs
@@ -243,6 +243,8 @@ impl PullSource for SourceClient {
 #[derive(Clone, Debug)]
 pub struct WriteBackRequest {
     pub bucket: String,
+    /// Identity captured with the source configuration, retained through cleanup.
+    pub bucket_incarnation_id: uuid::Uuid,
     pub key: String,
     /// Source HEAD/GET of the whole object.
     pub head: SourceHead,
@@ -263,6 +265,7 @@ impl WriteBackRequest {
         let config = state.config();
         Self {
             bucket: state.bucket().to_string(),
+            bucket_incarnation_id: state.incarnation_id(),
             key: key.to_string(),
             head,
             source_label: format!("{}:{}", config.source.provider.as_str(), config.source.bucket),
@@ -357,7 +360,7 @@ pub trait OdmWriteBack: Send + Sync {
         parts: Vec<WriteBackPart>,
     ) -> Result<WriteBackOutcome, WriteBackError>;
 
-    async fn abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), WriteBackError>;
+    async fn abort_multipart_upload(&self, request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError>;
 }
 
 /// Why the pump stopped feeding the write-back before EOF.
@@ -662,9 +665,7 @@ async fn write_multipart(
         Err(err) => Err(err),
     };
     if completed.is_err()
-        && let Err(abort_err) = write_back
-            .abort_multipart_upload(&request.bucket, &request.key, &upload_id)
-            .await
+        && let Err(abort_err) = write_back.abort_multipart_upload(request, &upload_id).await
     {
         debug!(
             event = EVENT_ODM_PULL_FAILED,
@@ -1340,7 +1341,7 @@ mod tests {
             })
         }
 
-        async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, upload_id: &str) -> Result<(), WriteBackError> {
+        async fn abort_multipart_upload(&self, _request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError> {
             self.aborted.lock().push(upload_id.to_string());
             Ok(())
         }
diff --git a/rustfs/src/on_demand_migration/sys.rs b/rustfs/src/on_demand_migration/sys.rs
index d529594d4..8fbde7fa4 100644
--- a/rustfs/src/on_demand_migration/sys.rs
+++ b/rustfs/src/on_demand_migration/sys.rs
@@ -23,9 +23,8 @@
 //! path (initial load, admin update, peer reload, refresh loop, lazy load).
 //!
 //! Change detection compares the config by value (`PartialEq`) rather than
-//! by `updated_at`: the hook does not carry the timestamp, fetching it would
-//! re-enter the metadata system from inside its own publish path, and a
-//! byte-identical config never needs a new client anyway.
+//! by `updated_at`. The bucket incarnation is part of this comparison:
+//! recreating a bucket must cancel old work even with identical configuration.
 //!
 //! Client construction is async (TLS material may be read from disk), so
 //! the hook does not build inline: `publish` removes state synchronously and
@@ -270,6 +269,7 @@ impl Drop for InflightEntryGuard<'_> {
 /// config change (counters excepted), removed when the config goes away.
 pub struct BucketOdmState {
     bucket: String,
+    incarnation_id: uuid::Uuid,
     config: OnDemandMigrationConfig,
     applied_at: OffsetDateTime,
     endpoint_host: String,
@@ -307,6 +307,7 @@ impl BucketOdmState {
     async fn build(
         bucket: &str,
         config: &OnDemandMigrationConfig,
+        incarnation_id: uuid::Uuid,
         stats: Arc<OdmStats>,
         write_back: Option<Arc<dyn OdmWriteBack>>,
     ) -> Arc<Self> {
@@ -323,6 +324,7 @@ impl BucketOdmState {
         let policy = &config.policy;
         Arc::new(Self {
             bucket: bucket.to_string(),
+            incarnation_id,
             endpoint_host: endpoint_host(&config.source),
             config: config.clone(),
             applied_at: OffsetDateTime::now_utc(),
@@ -340,10 +342,18 @@ impl BucketOdmState {
         })
     }
 
+    pub fn filter_incarnation(self: Arc<Self>, incarnation_id: uuid::Uuid) -> Option<Arc<Self>> {
+        (self.incarnation_id == incarnation_id && !self.is_cancelled()).then_some(self)
+    }
+
     pub fn bucket(&self) -> &str {
         &self.bucket
     }
 
+    pub fn incarnation_id(&self) -> uuid::Uuid {
+        self.incarnation_id
+    }
+
     pub fn config(&self) -> &OnDemandMigrationConfig {
         &self.config
     }
@@ -742,16 +752,17 @@ impl OnDemandMigrationSys {
         BUCKET_CONFIG_PUBLISH_HOOK
             .set(Box::new(move |bucket, config_file, stored| {
                 if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
-                    self.publish_stored(bucket, stored.map(|(bytes, _)| bytes));
+                    self.publish_stored(bucket, stored.map(|(bytes, _, incarnation)| (bytes, incarnation)));
                 }
             }))
             .is_ok()
     }
 
     /// Corrupt persisted bytes withdraw state synchronously, just like deletion.
-    fn publish_stored(&'static self, bucket: &str, stored: Option<&[u8]>) {
-        match stored.map(OnDemandMigrationConfig::from_json).transpose() {
-            Ok(config) => self.publish(bucket, config.as_ref()),
+    fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
+        let incarnation_id = stored.map(|(_, id)| id).unwrap_or_default();
+        match stored.map(|(bytes, _)| OnDemandMigrationConfig::from_json(bytes)).transpose() {
+            Ok(config) => self.publish_for_incarnation(bucket, incarnation_id, config.as_ref()),
             Err(err) => {
                 warn!(
                     event = EVENT_ODM_BUCKET_STATE_APPLIED,
@@ -762,7 +773,7 @@ impl OnDemandMigrationSys {
                     error = %err,
                     "Failed to parse on-demand migration config"
                 );
-                self.publish(bucket, None);
+                self.publish_for_incarnation(bucket, incarnation_id, None);
             }
         }
     }
@@ -770,14 +781,19 @@ impl OnDemandMigrationSys {
     /// Hook entry point: removals apply immediately, installs are spawned
     /// (client construction is async). Requires a Tokio runtime for the
     /// install path; without one the config is logged and skipped.
-    pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
-        let config = self.desired(config);
+    pub fn publish_for_incarnation(
+        &'static self,
+        bucket: &str,
+        incarnation_id: uuid::Uuid,
+        config: Option<&OnDemandMigrationConfig>,
+    ) {
+        let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
         let generation = self.reserve_generation(bucket, config.is_some());
         let Some(config) = config else {
             self.remove_with_generation(bucket, generation);
             return;
         };
-        if self.is_unchanged(bucket, config, generation) {
+        if self.is_unchanged(bucket, incarnation_id, config, generation) {
             return;
         }
         let Ok(handle) = tokio::runtime::Handle::try_current() else {
@@ -796,32 +812,49 @@ impl OnDemandMigrationSys {
         let bucket = bucket.to_string();
         let config = config.clone();
         handle.spawn(async move {
-            self.apply_with_generation(&bucket, Some(&config), generation).await;
+            self.apply_with_generation(&bucket, incarnation_id, Some(&config), generation)
+                .await;
         });
     }
 
     /// Installs, rebuilds, or removes the bucket state for `config`.
     /// Idempotent: the same config on an installed bucket is a no-op.
+    #[cfg(test)]
     pub async fn apply(&self, bucket: &str, config: Option<&OnDemandMigrationConfig>) -> ApplyOutcome {
-        let config = self.desired(config);
+        self.apply_for_incarnation(bucket, uuid::Uuid::from_u128(1), config).await
+    }
+
+    #[cfg(test)]
+    pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
+        self.publish_for_incarnation(bucket, uuid::Uuid::from_u128(1), config);
+    }
+
+    pub async fn apply_for_incarnation(
+        &self,
+        bucket: &str,
+        incarnation_id: uuid::Uuid,
+        config: Option<&OnDemandMigrationConfig>,
+    ) -> ApplyOutcome {
+        let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
         let generation = self.reserve_generation(bucket, config.is_some());
-        self.apply_with_generation(bucket, config, generation).await
+        self.apply_with_generation(bucket, incarnation_id, config, generation).await
     }
 
     async fn apply_with_generation(
         &self,
         bucket: &str,
+        incarnation_id: uuid::Uuid,
         config: Option<&OnDemandMigrationConfig>,
         generation: u64,
     ) -> ApplyOutcome {
         let Some(config) = self.desired(config) else {
             return self.remove_with_generation(bucket, generation);
         };
-        if self.is_unchanged(bucket, config, generation) {
+        if self.is_unchanged(bucket, incarnation_id, config, generation) {
             return ApplyOutcome::Unchanged;
         }
         let stats = self.state(bucket).map(|state| Arc::clone(&state.stats)).unwrap_or_default();
-        let state = BucketOdmState::build(bucket, config, stats, self.write_back()).await;
+        let state = BucketOdmState::build(bucket, config, incarnation_id, stats, self.write_back()).await;
 
         let (outcome, previous) = {
             let mut buckets = self.buckets.write();
@@ -871,6 +904,7 @@ impl OnDemandMigrationSys {
 
     /// One-shot lookup: module switch, bucket state, prefix filter,
     /// client availability, negative cache, breaker, in that order.
+    #[cfg(test)]
     pub fn resolve(&self, bucket: &str, key: &str) -> Option<OdmLookup> {
         if !self.is_module_enabled() {
             return None;
@@ -878,6 +912,13 @@ impl OnDemandMigrationSys {
         self.state(bucket)?.resolve_key(key)
     }
 
+    pub fn resolve_for_incarnation(&self, bucket: &str, key: &str, incarnation_id: uuid::Uuid) -> Option<OdmLookup> {
+        if !self.is_module_enabled() {
+            return None;
+        }
+        self.state(bucket)?.filter_incarnation(incarnation_id)?.resolve_key(key)
+    }
+
     pub fn state(&self, bucket: &str) -> Option<Arc<BucketOdmState>> {
         self.buckets.read().get(bucket).and_then(|slot| slot.state.clone())
     }
@@ -925,15 +966,26 @@ impl OnDemandMigrationSys {
 
     /// Claims `generation` for the bucket when the installed state already
     /// matches `config` and has a usable client.
-    fn is_unchanged(&self, bucket: &str, config: &OnDemandMigrationConfig, generation: u64) -> bool {
+    fn is_unchanged(&self, bucket: &str, incarnation_id: uuid::Uuid, config: &OnDemandMigrationConfig, generation: u64) -> bool {
         let mut buckets = self.buckets.write();
         let Some(slot) = buckets.get_mut(bucket) else {
             return false;
         };
+        if slot.generation > generation {
+            return false;
+        }
+        if slot
+            .state
+            .as_ref()
+            .is_some_and(|state| state.incarnation_id != incarnation_id)
+            && let Some(previous) = slot.state.take()
+        {
+            previous.cancel.cancel();
+        }
         let unchanged = slot
             .state
             .as_ref()
-            .is_some_and(|state| state.client.is_ok() && state.config == *config);
+            .is_some_and(|state| state.client.is_ok() && state.incarnation_id == incarnation_id && state.config == *config);
         if unchanged && slot.generation < generation {
             slot.generation = generation;
         }
@@ -1366,13 +1418,88 @@ mod tests {
         assert!(state.is_cancelled());
     }
 
+    #[tokio::test]
+    async fn identical_config_on_recreated_bucket_cancels_old_state() {
+        let sys = enabled_sys();
+        let cfg = config(None);
+        let old_id = uuid::Uuid::new_v4();
+        let new_id = uuid::Uuid::new_v4();
+        sys.apply_for_incarnation("recreated", old_id, Some(&cfg)).await;
+        let old = sys.state("recreated").expect("old state installed");
+        assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_none());
+        sys.apply_for_incarnation("recreated", new_id, Some(&cfg)).await;
+        let replacement = sys.state("recreated").expect("replacement state installed");
+        assert!(old.is_cancelled());
+        assert!(!Arc::ptr_eq(&old, &replacement));
+        assert_eq!(replacement.incarnation_id(), new_id);
+        assert!(sys.resolve_for_incarnation("recreated", "key", old_id).is_none());
+        assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_some());
+    }
+
+    #[tokio::test]
+    async fn changed_delete_marker_policy_withdraws_the_captured_lookup() {
+        let sys = enabled_sys();
+        let incarnation = uuid::Uuid::new_v4();
+        let mut cfg = config(None);
+        cfg.policy.respect_local_delete_marker = false;
+        sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
+        let captured = sys.state("policy-snapshot").expect("policy A installed");
+        assert!(!captured.config().policy.respect_local_delete_marker);
+
+        cfg.policy.respect_local_delete_marker = true;
+        sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
+        let replacement = sys.state("policy-snapshot").expect("policy B installed");
+        assert!(replacement.config().policy.respect_local_delete_marker);
+        assert!(captured.is_cancelled());
+        assert!(
+            captured
+                .filter_incarnation(incarnation)
+                .and_then(|state| state.resolve_key("key"))
+                .is_none(),
+            "a request that evaluated policy A cannot continue through policy B"
+        );
+        assert!(
+            replacement
+                .clone()
+                .filter_incarnation(incarnation)
+                .and_then(|state| state.resolve_key("key"))
+                .is_some()
+        );
+        assert_eq!(
+            replacement
+                .stats()
+                .snapshot(replacement.breaker().state())
+                .source_latency
+                .count,
+            0
+        );
+    }
+
+    #[tokio::test]
+    async fn missing_incarnation_cannot_install_or_retain_source_state() {
+        let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
+        let cfg = config(None);
+        assert_eq!(
+            sys.apply_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg)).await,
+            ApplyOutcome::NotDesired
+        );
+        sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
+        assert!(sys.state("missing").is_none());
+
+        sys.apply_for_incarnation("missing", uuid::Uuid::new_v4(), Some(&cfg)).await;
+        let state = sys.state("missing").expect("valid identity installed");
+        sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
+        assert!(sys.state("missing").is_none());
+        assert!(state.is_cancelled());
+    }
+
     #[tokio::test]
     async fn corrupt_stored_config_withdraws_runtime_state() {
         let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
         let cfg = config(None);
         assert_eq!(sys.apply("corrupt", Some(&cfg)).await, ApplyOutcome::Installed);
         let state = sys.state("corrupt").expect("state installed");
-        sys.publish_stored("corrupt", Some(b"not-json"));
+        sys.publish_stored("corrupt", Some((b"not-json", uuid::Uuid::from_u128(1))));
         assert!(sys.state("corrupt").is_none(), "corruption cannot keep an older source active");
         assert!(state.is_cancelled(), "corruption cancels in-flight work");
     }
@@ -1395,7 +1522,11 @@ mod tests {
         let older = sys.reserve_generation("b", true);
         let newer = sys.reserve_generation("b", false);
         assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::NotDesired);
-        assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);
+        assert_eq!(
+            sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
+                .await,
+            ApplyOutcome::Superseded
+        );
         assert!(sys.state("b").is_none(), "removal must supersede an in-flight first install");
 
         assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
@@ -1404,7 +1535,11 @@ mod tests {
         let newer = sys.reserve_generation("b", false);
         assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::Removed);
         assert!(installed.is_cancelled());
-        assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);
+        assert_eq!(
+            sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
+                .await,
+            ApplyOutcome::Superseded
+        );
         assert!(sys.state("b").is_none(), "the stale install is discarded");
     }
 
diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs
index 213df046e..faa98ad72 100644
--- a/rustfs/src/storage/access.rs
+++ b/rustfs/src/storage/access.rs
@@ -288,6 +288,68 @@ async fn load_bucket_generation<T>(fs: &FS, req: &S3Request<T>, bucket: &str) ->
     load_bucket_generation_from_store(store.as_ref(), req, bucket).await
 }
 
+/// A read may consult only the source admitted before its authorization.
+/// Capture failures are deferred until a local miss actually needs that source.
+#[derive(Clone, Debug)]
+enum OdmReadGenerationGuard {
+    Unavailable,
+    Ready(BucketGenerationGuard),
+    Failed { code: S3ErrorCode, message: String },
+}
+
+impl OdmReadGenerationGuard {
+    fn from_result(result: S3Result<BucketGenerationGuard>) -> Self {
+        match result {
+            Ok(guard) => Self::Ready(guard),
+            Err(err) => Self::Failed {
+                code: err.code().clone(),
+                message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(),
+            },
+        }
+    }
+}
+
+fn odm_read_source_configured(bucket: &str) -> bool {
+    let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
+    sys.is_module_enabled() && sys.state(bucket).is_some()
+}
+
+async fn capture_odm_read_generation<T>(fs: &FS, req: &S3Request<T>, bucket: &str) -> OdmReadGenerationGuard {
+    if !odm_read_source_configured(bucket) {
+        return OdmReadGenerationGuard::Unavailable;
+    }
+    OdmReadGenerationGuard::from_result(load_bucket_generation(fs, req, bucket).await)
+}
+
+/// Direct usecase callers have no access middleware; capture at their entry.
+/// A server request without the access marker must never bind to a later source.
+pub(crate) async fn prepare_odm_read_generation<T>(
+    store: &crate::storage::storage_api::ECStore,
+    req: &mut S3Request<T>,
+    bucket: &str,
+) {
+    if req.extensions.get::<OdmReadGenerationGuard>().is_some() {
+        return;
+    }
+    let guard = if req.extensions.get::<std::sync::Arc<ServerContextSlot>>().is_some() || !odm_read_source_configured(bucket) {
+        OdmReadGenerationGuard::Unavailable
+    } else {
+        OdmReadGenerationGuard::from_result(load_bucket_generation_from_store(store, req, bucket).await)
+    };
+    req.extensions.insert(guard);
+}
+
+pub(crate) fn odm_read_generation<T>(req: &S3Request<T>, bucket: &str) -> S3Result<Option<uuid::Uuid>> {
+    match req.extensions.get::<OdmReadGenerationGuard>() {
+        Some(OdmReadGenerationGuard::Ready(guard)) if guard.bucket == bucket => Ok(Some(guard.incarnation_id)),
+        Some(OdmReadGenerationGuard::Ready(_)) => {
+            Err(s3_error!(InternalError, "source generation guard does not match request bucket"))
+        }
+        Some(OdmReadGenerationGuard::Failed { code, message }) => Err(S3Error::with_message(code.clone(), message.clone())),
+        Some(OdmReadGenerationGuard::Unavailable) | None => Ok(None),
+    }
+}
+
 async fn load_copy_source_bucket_generation(fs: &FS, bucket: &str) -> S3Result<CopySourceBucketGenerationGuard> {
     let store = fs
         .server_ctx()
@@ -2365,6 +2427,8 @@ impl S3Access for FS {
     ///
     /// This method returns `Ok(())` by default.
     async fn get_object(&self, req: &mut S3Request<GetObjectInput>) -> S3Result<()> {
+        let bucket = req.input.bucket.clone();
+        let source_generation = capture_odm_read_generation(self, req, &bucket).await;
         let req_info = ext_req_info_mut(&mut req.extensions)?;
         req_info.bucket = Some(req.input.bucket.clone());
         req_info.object = Some(req.input.key.clone());
@@ -2372,7 +2436,9 @@ impl S3Access for FS {
 
         // GHSA-3ppv: a versioned read (?versionId=...) must authorize against
         // s3:GetObjectVersion, not s3:GetObject.
-        authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await
+        authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await?;
+        req.extensions.insert(source_generation);
+        Ok(())
     }
 
     /// Checks whether the GetObjectAcl request has accesses to the resources.
@@ -2484,6 +2550,8 @@ impl S3Access for FS {
     ///
     /// This method returns `Ok(())` by default.
     async fn head_object(&self, req: &mut S3Request<HeadObjectInput>) -> S3Result<()> {
+        let bucket = req.input.bucket.clone();
+        let source_generation = capture_odm_read_generation(self, req, &bucket).await;
         let req_info = ext_req_info_mut(&mut req.extensions)?;
         req_info.bucket = Some(req.input.bucket.clone());
         req_info.object = Some(req.input.key.clone());
@@ -2496,10 +2564,13 @@ impl S3Access for FS {
         if get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_CHECK).as_deref() == Some("true") {
             authorize_request(req, Action::S3Action(S3Action::ReplicateObjectAction)).await?;
             req_info_mut(req)?.replication_request_authorized = true;
+            req.extensions.insert(source_generation);
             return Ok(());
         }
 
-        authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await
+        authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?;
+        req.extensions.insert(source_generation);
+        Ok(())
     }
 
     /// Checks whether the ListBucketAnalyticsConfigurations request has accesses to the resources.
@@ -2588,10 +2659,14 @@ impl S3Access for FS {
     ///
     /// This method returns `Ok(())` by default.
     async fn list_objects_v2(&self, req: &mut S3Request<ListObjectsV2Input>) -> S3Result<()> {
+        let bucket = req.input.bucket.clone();
+        let source_generation = capture_odm_read_generation(self, req, &bucket).await;
         let req_info = ext_req_info_mut(&mut req.extensions)?;
         req_info.bucket = Some(req.input.bucket.clone());
 
-        authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await
+        authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await?;
+        req.extensions.insert(source_generation);
+        Ok(())
     }
 
     /// Checks whether the ListParts request has accesses to the resources.
@@ -3858,6 +3933,285 @@ mod tests {
         assert_eq!(req_info.object.as_deref(), Some("test-key"));
     }
 
+    #[test]
+    #[serial]
+    fn odm_read_capture_preserves_access_and_parameter_error_order() {
+        crate::app::gating_test_env::run_large_stack_test("odm-access-order", || async {
+            let context = crate::app::gating_test_env::shared_gating_ambient().await;
+            let server_ctx = ServerContextSlot::new();
+            assert!(server_ctx.install(Arc::clone(&context)));
+            let fs = FS::with_server_ctx(server_ctx);
+            let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
+            let enabled_before = sys.is_module_enabled();
+            let bucket = format!("odm-access-missing-{}", uuid::Uuid::new_v4());
+            for enabled in [false, true] {
+                sys.set_module_enabled(enabled);
+                let mut get = build_request(
+                    GetObjectInput {
+                        bucket: bucket.clone(),
+                        key: "key".into(),
+                        part_number: Some(0),
+                        ..Default::default()
+                    },
+                    Method::GET,
+                );
+                get.extensions.insert(ReqInfo {
+                    cred: Some(rustfs_credentials::Credentials::default()),
+                    is_owner: true,
+                    ..Default::default()
+                });
+                get.extensions.insert(fs.server_ctx().clone());
+                let mut head = get.clone().map_input(|_| HeadObjectInput {
+                    bucket: bucket.clone(),
+                    key: "key".into(),
+                    part_number: Some(1),
+                    range: Some(s3s::dto::Range::Int { first: 0, last: Some(1) }),
+                    ..Default::default()
+                });
+                head.method = Method::HEAD;
+                let mut list = get.clone().map_input(|_| ListObjectsV2Input {
+                    bucket: bucket.clone(),
+                    max_keys: Some(-1),
+                    ..Default::default()
+                });
+                fs.get_object(&mut get)
+                    .await
+                    .expect("ordinary GET access must not require bucket identity");
+                fs.head_object(&mut head)
+                    .await
+                    .expect("ordinary HEAD access must not require bucket identity");
+                fs.list_objects_v2(&mut list)
+                    .await
+                    .expect("ordinary LIST access must not require bucket identity");
+                assert!(matches!(
+                    get.extensions.get::<super::OdmReadGenerationGuard>(),
+                    Some(super::OdmReadGenerationGuard::Unavailable)
+                ));
+                assert!(matches!(
+                    head.extensions.get::<super::OdmReadGenerationGuard>(),
+                    Some(super::OdmReadGenerationGuard::Unavailable)
+                ));
+                assert!(matches!(
+                    list.extensions.get::<super::OdmReadGenerationGuard>(),
+                    Some(super::OdmReadGenerationGuard::Unavailable)
+                ));
+                let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+                assert_eq!(
+                    usecase.execute_get_object(get).await.expect_err("bad GET part number").code(),
+                    &S3ErrorCode::InvalidArgument
+                );
+                assert_eq!(
+                    usecase
+                        .execute_head_object(head)
+                        .await
+                        .expect_err("range and part number conflict")
+                        .code(),
+                    &S3ErrorCode::InvalidArgument
+                );
+                assert_eq!(
+                    crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
+                        .execute_list_objects_v2(list)
+                        .await
+                        .expect_err("negative max keys")
+                        .code(),
+                    &S3ErrorCode::InvalidArgument
+                );
+            }
+            sys.set_module_enabled(enabled_before);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn odm_read_capture_failure_is_deferred_until_source_miss() {
+        crate::app::gating_test_env::run_large_stack_test("odm-access-failure", || async {
+            let context = crate::app::gating_test_env::shared_gating_ambient().await;
+            let store = context.object_store();
+            let server_ctx = ServerContextSlot::new();
+            assert!(server_ctx.install(Arc::clone(&context)));
+            let fs = FS::with_server_ctx(server_ctx);
+            let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
+            let enabled_before = sys.is_module_enabled();
+            sys.set_module_enabled(true);
+            let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
+            let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
+            config.policy.list_through = true;
+            sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
+            let mut get = build_request(
+                GetObjectInput {
+                    bucket: bucket.clone(),
+                    key: "local".into(),
+                    ..Default::default()
+                },
+                Method::GET,
+            );
+            get.extensions.insert(ReqInfo {
+                cred: Some(rustfs_credentials::Credentials::default()),
+                is_owner: true,
+                ..Default::default()
+            });
+            get.extensions.insert(fs.server_ctx().clone());
+            let mut head = get.clone().map_input(|_| HeadObjectInput {
+                bucket: bucket.clone(),
+                key: "local".into(),
+                ..Default::default()
+            });
+            head.method = Method::HEAD;
+            let mut list = get.clone().map_input(|_| ListObjectsV2Input {
+                bucket: bucket.clone(),
+                ..Default::default()
+            });
+            fs.list_objects_v2(&mut list)
+                .await
+                .expect("capture failure must not preempt LIST authorization");
+            fs.get_object(&mut get)
+                .await
+                .expect("capture failure must not preempt GET authorization");
+            fs.head_object(&mut head)
+                .await
+                .expect("capture failure must not preempt HEAD authorization");
+            assert!(matches!(
+                get.extensions.get::<super::OdmReadGenerationGuard>(),
+                Some(super::OdmReadGenerationGuard::Failed { .. })
+            ));
+            assert!(matches!(
+                head.extensions.get::<super::OdmReadGenerationGuard>(),
+                Some(super::OdmReadGenerationGuard::Failed { .. })
+            ));
+            store
+                .make_bucket(
+                    &bucket,
+                    &MakeBucketOptions {
+                        versioning_enabled: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create bucket after capture");
+            store
+                .put_object(
+                    &bucket,
+                    "local",
+                    &mut crate::storage::PutObjReader::from_vec(b"local".to_vec()),
+                    &crate::storage::ObjectOptions::default(),
+                )
+                .await
+                .expect("create local hit");
+            // Publishing a usable source later must not repair a failed capture.
+            let incarnation = store.bucket_incarnation_id(&bucket).await.expect("created identity");
+            sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
+            let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
+            usecase
+                .execute_get_object(get.clone())
+                .await
+                .expect("a local GET hit ignores source capture failure");
+            usecase
+                .execute_head_object(head.clone())
+                .await
+                .expect("a local HEAD hit ignores source capture failure");
+            get.input.key = "missing".into();
+            head.input.key = "missing".into();
+            assert_eq!(
+                usecase
+                    .execute_get_object(get.clone())
+                    .await
+                    .expect_err("failed capture cannot rebind on GET miss")
+                    .code(),
+                &S3ErrorCode::NoSuchBucket
+            );
+            assert_eq!(
+                usecase
+                    .execute_head_object(head.clone())
+                    .await
+                    .expect_err("failed capture cannot rebind on HEAD miss")
+                    .code(),
+                &S3ErrorCode::NoSuchBucket
+            );
+            assert_eq!(
+                crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
+                    .execute_list_objects_v2(list.clone())
+                    .await
+                    .expect_err("failed capture cannot rebind on LIST")
+                    .code(),
+                &S3ErrorCode::NoSuchBucket
+            );
+            config.filter.prefix = Some("remote/".into());
+            sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
+            assert_eq!(
+                usecase
+                    .execute_get_object(get.clone())
+                    .await
+                    .expect_err("filtered GET remains local")
+                    .code(),
+                &S3ErrorCode::NoSuchKey
+            );
+            assert_eq!(
+                usecase
+                    .execute_head_object(head.clone())
+                    .await
+                    .expect_err("filtered HEAD remains local")
+                    .code(),
+                &S3ErrorCode::NoSuchKey
+            );
+            list.input.prefix = Some("local/".into());
+            let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
+                .execute_list_objects_v2(list.clone())
+                .await
+                .expect("disjoint prefix needs no source identity");
+            assert_eq!(listing.output.key_count, Some(0));
+            list.input.prefix = Some("remote/".into());
+            list.input.max_keys = Some(0);
+            let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
+                .execute_list_objects_v2(list)
+                .await
+                .expect("an empty page needs no source identity");
+            assert_eq!(listing.output.key_count, Some(0));
+
+            config.filter.prefix = None;
+            config.policy.head = crate::on_demand_migration::HeadPolicy::LocalOnly;
+            sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
+            assert_eq!(
+                usecase
+                    .execute_head_object(head.clone())
+                    .await
+                    .expect_err("local-only HEAD needs no source identity")
+                    .code(),
+                &S3ErrorCode::NoSuchKey
+            );
+            config.policy.head = crate::on_demand_migration::HeadPolicy::Proxy;
+            sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
+            store
+                .delete_object(
+                    &bucket,
+                    "missing",
+                    crate::storage::ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create local delete marker");
+            assert_eq!(
+                usecase
+                    .execute_get_object(get)
+                    .await
+                    .expect_err("GET respects the delete marker before capture failure")
+                    .code(),
+                &S3ErrorCode::NoSuchKey
+            );
+            assert_eq!(
+                usecase
+                    .execute_head_object(head)
+                    .await
+                    .expect_err("HEAD respects the delete marker before capture failure")
+                    .code(),
+                &S3ErrorCode::NoSuchKey
+            );
+            sys.remove(&bucket);
+            sys.set_module_enabled(enabled_before);
+        });
+    }
+
     #[tokio::test]
     #[serial]
     async fn put_object_access_captures_authorized_bucket_incarnation() {
diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs
index fe9a55641..e35cf812d 100644
--- a/rustfs/src/storage/storage_api.rs
+++ b/rustfs/src/storage/storage_api.rs
@@ -119,8 +119,9 @@ pub(crate) mod access_consumer {
     pub(crate) use super::super::access::{
         PostObjectRequestMarker, ReqInfo, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
         authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header,
-        load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization,
-        recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref,
+        load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
+        prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_is_authorized,
+        replication_request_authorized, req_info_mut, req_info_ref,
     };
 }
 
diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs
index 60de2c850..9abdbd5fd 100644
--- a/rustfs/src/storage_api.rs
+++ b/rustfs/src/storage_api.rs
@@ -430,10 +430,12 @@ pub(crate) mod on_demand_migration {
 
     #[cfg(test)]
     pub(crate) mod test_support {
+        pub(crate) use crate::storage::storage_api::contract::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations};
         pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::{
             BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata,
         };
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::BucketMetadataSys;
         pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{BucketMetadataSys, init_bucket_metadata_sys};
+        pub(crate) use crate::storage::storage_api::ecstore_set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
     }
 }

From ea9aa53fd8e0224481c70737c3df892d83552b95 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 02:09:44 +0800
Subject: [PATCH 17/20] docs(odm): record upgrade limits in release notes
 (#7233)

---
 CHANGELOG.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ebf882167..4f6601c07 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,7 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Read paths: an object at or below `policy.inline_max_bytes` (16 MiB by default) is teed to the client and to the local store in a single source read; a larger object or a Range read streams through and a background pull stores the whole object. A HEAD miss is proxied to the source and stores nothing (`policy.head = local_only` disables it). Every source-backed response carries `x-rustfs-on-demand-migration: source`
   - Protections: a per-source circuit breaker, a per-key negative cache, singleflight per key, a concurrency limit and a bounded pull queue shared by the inline and background paths, an optional bandwidth limit, an anti-loop request marker, and the shared outbound-endpoint (SSRF) policy
   - Metrics under `rustfs_on_demand_migration_*` (`requests_total`, `pulled_bytes_total`, `pulled_objects_total`, `pull_failures_total`, `inflight_pulls`, `queue_depth`, `source_latency_seconds_*`, `breaker_state`), mirrored per node by the admin status route
-  - Limitations: listings show only local objects (the source is not merged into `ListObjectsV2`); PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
+  - Listings: `ListObjects` v1 remains local with ordinary key markers. `ListObjectsV2` can merge source objects when `policy.list_through = true`; this is off by default
+  - Upgrade and rollback: finish upgrading every node before enabling ODM. An rc.5 node that writes bucket configuration drops the ODM fields from metadata; neither a later restart nor moving the service out of ECStore recovers them. Before rollback, disable ODM and securely retain the original full configuration and credentials. After every node returns to a compatible version, restore and validate that configuration. Redacted exports cannot replace the credential backup; source-only objects are unavailable through RustFS while ODM is disabled. See the upgrade and rollback section of `docs/operations/on-demand-migration.md`
+  - Optional Google dependencies: default and `full` server builds retain native GCS support. `cargo build -p rustfs --no-default-features --features ftps,webdav` excludes Google SDKs while preserving configuration decoding and redaction; native GCS ODM and tier operations require the `gcs` feature. Do not use that build with existing GCS-tiered data
+  - Limitations: PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
 - **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
   - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
   - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window

From 112f70914dbb99736fdfc9fc2e7659968fb784ef Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 02:24:04 +0800
Subject: [PATCH 18/20] fix(build): scope migration helpers to their features
 (#7234)

* test(odm): keep listing header import test scoped

* fix(build): gate GCS-only migration HTTP helpers
---
 rustfs/src/app/bucket_list_through.rs         | 2 +-
 rustfs/src/on_demand_migration/native_http.rs | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index 86497e36f..d4004c5bc 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -38,7 +38,6 @@ use crate::on_demand_migration::{
     SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
 };
 use futures::StreamExt;
-use http::HeaderMap;
 use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header};
 use std::sync::Arc;
 use std::time::Instant;
@@ -476,6 +475,7 @@ mod tests {
         FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
         SourceCredentials, TlsConfig,
     };
+    use http::HeaderMap;
     use std::time::Duration;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
diff --git a/rustfs/src/on_demand_migration/native_http.rs b/rustfs/src/on_demand_migration/native_http.rs
index ae084d084..9e1507ce1 100644
--- a/rustfs/src/on_demand_migration/native_http.rs
+++ b/rustfs/src/on_demand_migration/native_http.rs
@@ -133,6 +133,7 @@ impl NativeHttp {
         self.send_classified(request, error_code_header, false).await
     }
 
+    #[cfg(feature = "gcs")]
     pub(super) async fn send_object(
         &self,
         request: reqwest::Request,
@@ -210,6 +211,7 @@ pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) ->
 /// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
 /// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
 /// an MD5.
+#[cfg(any(test, feature = "gcs"))]
 pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
     let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
     (raw.len() == 16).then(|| faster_hex::hex_string(&raw))

From eb1b17802c6e06f8af692d0f81057baf32689659 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 02:34:56 +0800
Subject: [PATCH 19/20] test(odm): provide the source region in access fixture
 (#7235)

---
 rustfs/src/storage/access.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs
index faa98ad72..223a73383 100644
--- a/rustfs/src/storage/access.rs
+++ b/rustfs/src/storage/access.rs
@@ -4034,7 +4034,7 @@ mod tests {
             let enabled_before = sys.is_module_enabled();
             sys.set_module_enabled(true);
             let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
-            let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
+            let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","region":"us-east-1","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
             config.policy.list_through = true;
             sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
             let mut get = build_request(

From e1608fbd9ca934d157b5de46c80b4393f2dd3dd6 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 03:09:39 +0800
Subject: [PATCH 20/20] test(odm): exercise overflow and invalid cursors
 reliably (#7236)

---
 .../on_demand_migration/concurrency_test.rs   | 43 ++++++++++++++++---
 .../on_demand_migration/list_through_test.rs  | 17 +++-----
 2 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/crates/e2e_test/src/on_demand_migration/concurrency_test.rs b/crates/e2e_test/src/on_demand_migration/concurrency_test.rs
index 60d75a099..9335713b4 100644
--- a/crates/e2e_test/src/on_demand_migration/concurrency_test.rs
+++ b/crates/e2e_test/src/on_demand_migration/concurrency_test.rs
@@ -20,9 +20,10 @@
 //! journal (`count_requests`) carries the assertion in every one of them.
 
 use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
-use crate::fake_s3_target::Operation;
+use crate::fake_s3_target::{FaultAction, Operation};
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use bytes::Bytes;
+use futures::{StreamExt, TryStreamExt};
 use std::time::Duration;
 
 type TestResult = Result<(), BoxError>;
@@ -145,14 +146,38 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
     .await?;
 
     let body = payload(128 * 1024);
+    let blocker = "queue/blocker.bin";
+    env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
+    // The one-chunk range completes immediately; its full background pull
+    // occupies the only slot while the remaining requests fill the queue.
+    env.source.inject_for_key(
+        Operation::GetObject,
+        blocker,
+        FaultAction::SlowSendBody {
+            chunk_bytes: 1024,
+            delay: Duration::from_millis(100),
+        },
+        2,
+    );
+    let response = env
+        .raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
+        .await?;
+    assert_eq!(response.status, 206);
+    assert_eq!(response.body, body.slice(0..1024));
+    env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
+
     let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
     let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
     env.seed_source(SOURCE_BUCKET, &seeds);
 
-    let responses: Vec<RawResponse> = futures::future::try_join_all(
+    // Bound source connections below the fixture's limit while still
+    // submitting all 100 requests to the eight-slot background queue.
+    let responses: Vec<RawResponse> = futures::stream::iter(
         keys.iter()
             .map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
     )
+    .buffered(16)
+    .try_collect()
     .await?;
     for (key, response) in keys.iter().zip(&responses) {
         assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
@@ -168,6 +193,15 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         .wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
         .await?;
     assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
+    let queue_full = usize::try_from(queue_full)?;
+    assert!(queue_full <= REQUESTS);
+    env.wait_for_status_counter(
+        bucket,
+        "/counters/pulled_objects_total/background",
+        u64::try_from(REQUESTS + 1 - queue_full)?,
+        SETTLE,
+    )
+    .await?;
 
     let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
     assert!(
@@ -175,9 +209,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         "every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
     );
     let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
-    assert!(
-        dropped > 0,
-        "the overflowed keys are the ones with no backfill GET, but every key got one"
-    );
+    assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
     Ok(())
 }
diff --git a/crates/e2e_test/src/on_demand_migration/list_through_test.rs b/crates/e2e_test/src/on_demand_migration/list_through_test.rs
index a912a8915..707934fe3 100644
--- a/crates/e2e_test/src/on_demand_migration/list_through_test.rs
+++ b/crates/e2e_test/src/on_demand_migration/list_through_test.rs
@@ -265,16 +265,13 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
     let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
     assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
 
-    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
-    let rejected = env
-        .raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
-        .await?;
-    assert_eq!(
-        rejected.status,
-        400,
-        "a bumped token version is a client error: {}",
-        String::from_utf8_lossy(&rejected.body)
-    );
+    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
+    assert_ne!(tampered, token, "the test must change the token version");
+    let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
+    let rejected = env.raw_list_objects_v2(bucket, &query).await?;
+    let error_body = String::from_utf8_lossy(&rejected.body);
+    assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
+    assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
     Ok(())
 }