chore: merge main PR evidence guidance

.github/workflows/rustfs-heal-test.yml

@@ -54,14 +54,25 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 480
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -117,7 +128,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -127,7 +138,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -137,13 +148,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -152,8 +160,9 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
           import re
           import sys
 
@@ -165,6 +174,7 @@ jobs:
 
           steps = {}
           order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
           version = None
           version_node = None
           verdict = None
@@ -178,14 +188,15 @@ jobs:
                           n, desc, status = m.group(1), m.group(2), m.group(3)
                           if n not in steps:
                               order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
                           continue
                       m = ver_re.match(line)
                       if m:
                           version, version_node = m.group(1), m.group(2)
                           continue
                       m = result_re.match(line)
-                      if m:
+                      if m and verdict != 'FAIL':
                           verdict, verdict_detail = m.group(1), m.group(2)
           except FileNotFoundError:
               pass
@@ -205,30 +216,43 @@ jobs:
                   out.write(f'| {n} | {desc} | {status} |\n')
               if not order:
                   out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
           PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${STEPS_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -255,11 +279,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -287,14 +310,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -310,14 +335,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-heal-test*.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}

.github/workflows/rustfs-kms-test.yml

@@ -49,10 +49,28 @@ env:
 jobs:
   kms-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -109,9 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -141,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -156,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -255,11 +231,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -287,14 +262,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -310,14 +287,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-kms.log
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-performance-test.yml

@@ -76,22 +76,33 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
   performance-test:
     runs-on: pf-testing
-    # Requirement: a failing benchmark must not fail the workflow;
-    # failures are filed to rustfs/backlog.
-    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -123,7 +134,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -133,7 +144,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -143,7 +154,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -156,17 +167,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -186,7 +195,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -194,7 +202,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -202,6 +210,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -211,8 +221,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          } > "${REPORT_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -231,11 +241,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -263,14 +272,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -286,20 +297,17 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-perf-test*.log
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-pool-expand-test.yml

@@ -76,9 +76,6 @@ jobs:
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:

.github/workflows/rustfs-replication-test.yml

@@ -62,12 +62,28 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -116,9 +132,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -141,10 +154,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -166,80 +176,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -266,11 +240,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -298,14 +271,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -321,14 +296,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-s3-compat-test.yml

@@ -37,10 +37,28 @@ env:
 jobs:
   s3-compat-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -88,9 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -107,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -132,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -235,11 +208,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -267,14 +239,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -290,14 +264,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-s3-compat.log
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-storage-test.yml

@@ -46,10 +46,28 @@ env:
 jobs:
   storage-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -97,9 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -122,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -147,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -250,11 +223,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -282,14 +254,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -305,14 +279,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-tier-test.yml

@@ -61,9 +61,6 @@ env:
 jobs:
   tier-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:

.github/workflows/rustfs-upgrade-test.yml

@@ -79,10 +79,28 @@ env:
 jobs:
   upgrade-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,9 +160,7 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
-        continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
@@ -202,10 +218,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -226,103 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
-          rows = []
-          index = {}
-          topo_rows = []
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${MATRIX_TABLE}"
+              echo ""
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -349,11 +306,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -381,14 +337,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -404,14 +362,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)

crates/ecstore/src/api/mod.rs

@@ -69,13 +69,6 @@ pub mod bucket {
             };
         }
 
-        pub mod recovery_control {
-            pub use crate::bucket::lifecycle::recovery_control::{
-                IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
-                inspect_recovery_control, list_recovery_controls,
-            };
-        }
-
         pub mod transition_transaction {
             pub use crate::bucket::lifecycle::transition_transaction::{
                 TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -22,7 +22,7 @@ use super::{
     bucket_lifecycle_ops::{
         ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
     },
-    manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
+    manual_transition_job, tier_delete_journal, transition_transaction,
 };
 use crate::error::{Error, Result};
 use crate::services::tier::tier_probe_intent;
@@ -41,7 +41,6 @@ pub(crate) enum DurableIlmRecordKind {
     ManualTransitionScope,
     ManualTransitionTask,
     ManualTransitionWorkerResult,
-    RecoveryControl,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -106,14 +105,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
-pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
-    name: "recovery-control",
-    prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
-    max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
-    kind: DurableIlmRecordKind::RecoveryControl,
-};
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
     TIER_DELETE_JOURNAL_V6_NAMESPACE,
     TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
@@ -123,7 +116,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
     MANUAL_TRANSITION_TASK_NAMESPACE,
     MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
-    RECOVERY_CONTROL_NAMESPACE,
 ];
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -249,18 +241,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
     ManualTransitionWorkerResult {
         content_sha256: String,
     },
-    RecoveryControl {
-        content_sha256: String,
-        identity_sha256: String,
-        source_generation_sha256: String,
-        first_seen_at_unix_nanos: i64,
-        revision: u64,
-        classification: recovery_control::IlmRecoveryClassification,
-        attempt_count: u64,
-        consecutive_failure_count: u32,
-        #[serde(default, skip_serializing_if = "Option::is_none")]
-        owner_fence_sha256: Option<String>,
-    },
 }
 
 impl DurableIlmRecordCheckpoint {
@@ -274,8 +254,7 @@ impl DurableIlmRecordCheckpoint {
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
             | Self::ManualTransitionTask { content_sha256 }
-            | Self::ManualTransitionWorkerResult { content_sha256 }
-            | Self::RecoveryControl { content_sha256, .. } => content_sha256,
+            | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
         }
     }
 
@@ -549,51 +528,6 @@ impl DurableIlmRecordCheckpoint {
                     ..
                 },
             ) => previous_identity == next_identity && next_updated_at > previous_updated_at,
-            (
-                Self::RecoveryControl {
-                    identity_sha256: previous_identity,
-                    source_generation_sha256: previous_generation,
-                    first_seen_at_unix_nanos: previous_first_seen,
-                    revision: previous_revision,
-                    classification: previous_classification,
-                    attempt_count: previous_attempts,
-                    consecutive_failure_count: previous_failures,
-                    owner_fence_sha256: previous_owner,
-                    ..
-                },
-                Self::RecoveryControl {
-                    identity_sha256: next_identity,
-                    source_generation_sha256: next_generation,
-                    first_seen_at_unix_nanos: next_first_seen,
-                    revision: next_revision,
-                    classification: next_classification,
-                    attempt_count: next_attempts,
-                    consecutive_failure_count: next_failures,
-                    owner_fence_sha256: next_owner,
-                    ..
-                },
-            ) => {
-                let adjacent = previous_identity == next_identity
-                    && previous_first_seen == next_first_seen
-                    && previous_revision.checked_add(1) == Some(*next_revision);
-                let claim = next_owner.is_some()
-                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && previous_attempts.checked_add(1) == Some(*next_attempts)
-                    && previous_failures == next_failures;
-                let source_refresh = previous_owner.is_some()
-                    && previous_owner == next_owner
-                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && previous_attempts == next_attempts
-                    && previous_failures == next_failures
-                    && previous_generation != next_generation;
-                let completion = previous_owner.is_some()
-                    && next_owner.is_none()
-                    && previous_generation == next_generation
-                    && previous_attempts == next_attempts;
-                adjacent && (claim || source_refresh || completion)
-            }
             _ => false,
         };
 
@@ -619,14 +553,6 @@ impl DurableIlmRecordCheckpoint {
         {
             return false;
         }
-        if let Self::RecoveryControl { classification, .. } = terminal
-            && !matches!(
-                classification,
-                recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
-            )
-        {
-            return false;
-        }
         if self == terminal || self.validate_successor(terminal).is_ok() {
             return true;
         }
@@ -726,32 +652,6 @@ impl DurableIlmRecordCheckpoint {
                         .is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
                     && (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
             }
-            (
-                Self::RecoveryControl {
-                    identity_sha256: previous_identity,
-                    source_generation_sha256: previous_generation,
-                    first_seen_at_unix_nanos: previous_first_seen,
-                    revision: previous_revision,
-                    attempt_count: previous_attempts,
-                    ..
-                },
-                Self::RecoveryControl {
-                    identity_sha256: terminal_identity,
-                    source_generation_sha256: terminal_generation,
-                    first_seen_at_unix_nanos: terminal_first_seen,
-                    revision: terminal_revision,
-                    attempt_count: terminal_attempts,
-                    classification:
-                        recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
-                    ..
-                },
-            ) => {
-                previous_identity == terminal_identity
-                    && (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
-                    && previous_first_seen == terminal_first_seen
-                    && terminal_revision > previous_revision
-                    && terminal_attempts >= previous_attempts
-            }
             _ => false,
         }
     }
@@ -1319,35 +1219,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 },
             )
         }
-        DurableIlmRecordKind::RecoveryControl => {
-            let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
-                .map_err(|err| Error::other(err.to_string()))?;
-            let control =
-                recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
-            let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
-                .map_err(|err| Error::other(err.to_string()))?;
-            if canonical != path || control.identity.protocol != protocol {
-                return Err(Error::other("ILM recovery control path is not canonical"));
-            }
-            let identity_sha256 = checkpoint_hash(&control.identity)?;
-            let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
-            let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
-            (
-                "control_id",
-                control_id,
-                DurableIlmRecordCheckpoint::RecoveryControl {
-                    content_sha256,
-                    identity_sha256,
-                    source_generation_sha256,
-                    first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
-                    revision: control.revision,
-                    classification: control.classification,
-                    attempt_count: control.attempt_count,
-                    consecutive_failure_count: control.consecutive_failure_count,
-                    owner_fence_sha256,
-                },
-            )
-        }
         DurableIlmRecordKind::ManualTransitionJob => {
             let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
                 .map_err(|err| Error::other(err.to_string()))?;
@@ -1541,94 +1412,6 @@ mod tests {
             .checkpoint
     }
 
-    fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
-        let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
-        let generation = recovery_control::IlmRecoverySourceGeneration::new(
-            transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
-            "source-etag",
-            "a".repeat(64),
-            vec![recovery_control::IlmRecoverySourceCopy {
-                authority: "pool-0/set-0".to_string(),
-                canonical_path: source_path.to_string(),
-                etag: "source-etag".to_string(),
-                encoded_len: 128,
-                content_sha256: "a".repeat(64),
-            }],
-        )
-        .expect("source generation should build");
-        recovery_control::IlmRecoveryControl::new(
-            recovery_control::IlmRecoveryControlIdentity {
-                protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
-                canonical_source_path: source_path.to_string(),
-                stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
-                record_class: "transition_transaction_v1".to_string(),
-            },
-            generation,
-            recovery_control::IlmRecoveryClassification::Retrying,
-            1_000_000_000,
-            recovery_control::IlmRecoveryErrorCode::None,
-        )
-        .expect("recovery control should build")
-    }
-
-    fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
-        let control_id = control.identity.source_operation_digest().expect("control id should derive");
-        let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
-            .expect("control path should build");
-        let encoded = control.encode().expect("control should encode");
-        let namespace = classify_durable_ilm_record(&path)
-            .expect("recovery control namespace should classify")
-            .expect("recovery control should be durable");
-        assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
-        validate_durable_ilm_record(&path, &encoded)
-            .expect("recovery control should validate")
-            .checkpoint
-    }
-
-    #[test]
-    fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
-        let initial_control = recovery_control_fixture();
-        let initial = recovery_control_checkpoint(&initial_control);
-
-        let mut claimed_control = initial_control;
-        let mut advanced_generation = claimed_control.observed_source_generation.clone();
-        advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
-        claimed_control
-            .claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
-            .expect("control should claim");
-        let claimed = recovery_control_checkpoint(&claimed_control);
-        initial.validate_successor(&claimed).expect("claim should advance receipt");
-
-        let mut retry_control = claimed_control;
-        retry_control
-            .record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
-            .expect("retry should persist");
-        let retry = recovery_control_checkpoint(&retry_control);
-        claimed.validate_successor(&retry).expect("retry should advance receipt");
-
-        let ready_at = retry_control
-            .next_attempt_at_unix_nanos
-            .expect("retry deadline should persist");
-        let mut terminal_control = retry_control;
-        terminal_control
-            .claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
-            .expect("retry should claim");
-        let reclaimed = recovery_control_checkpoint(&terminal_control);
-        retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
-        terminal_control
-            .finish_attempt(
-                recovery_control::IlmRecoveryClassification::Terminal,
-                recovery_control::IlmRecoveryErrorCode::None,
-            )
-            .expect("control should terminate");
-        let terminal = recovery_control_checkpoint(&terminal_control);
-        reclaimed
-            .validate_successor(&terminal)
-            .expect("terminal state should advance receipt");
-        assert!(initial.is_predecessor_of_terminal(&terminal));
-        assert!(!initial.is_predecessor_of_terminal(&retry));
-    }
-
     #[test]
     fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
         let initial_intent = tier_probe_intent_fixture();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -24,7 +24,6 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
 mod object_handlers_common;
 mod object_lock_boundary;
 pub use self::core as lifecycle;
-pub mod recovery_control;
 mod replication_sink;
 pub mod rule;
 mod runtime_boundary;

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,11 +23,6 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
-use crate::bucket::lifecycle::recovery_control::{
-    IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
-    ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
-    save_recovery_control_if_absent, save_recovery_control_if_current,
-};
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
@@ -49,7 +44,6 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
 pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
 const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
-const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
 pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
@@ -743,8 +737,6 @@ pub enum TransitionTransactionRecoveryOutcome {
     RemoteCandidateDeleted,
     RecordDeleted,
     Retained,
-    RetainedAmbiguous(IlmRecoveryErrorCode),
-    OperatorRequired(IlmRecoveryErrorCode),
 }
 
 #[cfg(test)]
@@ -825,80 +817,6 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
     }
 }
 
-#[cfg(test)]
-#[derive(Default)]
-struct TransitionRecoveryTerminalBarrierState {
-    transaction_id: Uuid,
-    arrived: tokio::sync::Notify,
-    release: tokio::sync::Notify,
-}
-
-#[cfg(test)]
-pub(crate) struct TransitionRecoveryTerminalBarrier {
-    state: Arc<TransitionRecoveryTerminalBarrierState>,
-}
-
-#[cfg(test)]
-static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
-    std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
-> = std::sync::OnceLock::new();
-
-#[cfg(test)]
-impl TransitionRecoveryTerminalBarrier {
-    pub(crate) fn install(transaction_id: Uuid) -> Self {
-        let state = Arc::new(TransitionRecoveryTerminalBarrierState {
-            transaction_id,
-            ..Default::default()
-        });
-        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("transition recovery terminal barrier mutex should not poison");
-        assert!(
-            slot.is_none(),
-            "transition recovery terminal barrier must be installed by one test at a time"
-        );
-        *slot = Some(Arc::clone(&state));
-        drop(slot);
-        Self { state }
-    }
-
-    pub(crate) async fn wait_until_paused(&self) {
-        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
-            .await
-            .expect("transition recovery should persist terminal control before source cleanup");
-    }
-}
-
-#[cfg(test)]
-impl Drop for TransitionRecoveryTerminalBarrier {
-    fn drop(&mut self) {
-        self.state.release.notify_one();
-        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("transition recovery terminal barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
-            *slot = None;
-        }
-    }
-}
-
-#[cfg(test)]
-async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
-    let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("transition recovery terminal barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| barrier.transaction_id == transaction_id)
-        .cloned();
-    if let Some(barrier) = barrier {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum TransitionOperatorProbe {
@@ -1102,35 +1020,17 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
     let suffix = object
         .strip_prefix(&prefix)
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
-    let mut parts = suffix.split('/');
-    let shard_a = parts
+    let file_name = suffix
+        .rsplit('/')
         .next()
         .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    let shard_b = parts
-        .next()
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    let file_name = parts
-        .next()
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    if parts.next().is_some() {
-        return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
-    }
     let transaction_key = file_name
         .strip_suffix(".json")
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
-    if transaction_key.len() != 32
-        || !transaction_key
-            .bytes()
-            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
-        || shard_a != &transaction_key[..2]
-        || shard_b != &transaction_key[2..4]
-    {
+    if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
         return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
     }
-    Uuid::parse_str(transaction_key)
-        .ok()
-        .filter(|transaction_id| !transaction_id.is_nil())
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
+    Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
 }
 
 pub async fn process_transition_transaction_record(
@@ -1155,27 +1055,6 @@ async fn process_transition_transaction_record_at(
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     let record_name =
         transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
-    let now_unix_nanos =
-        i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
-    let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
-    let recovery_control_id = recovery_control_identity
-        .source_operation_digest()
-        .map_err(|err| Error::other(err.to_string()))?;
-    let control_record_name =
-        recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .map_err(|err| Error::other(err.to_string()))?;
-    let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
-        Some(
-            api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
-                .await?,
-        )
-    } else {
-        None
-    };
-    let _control_guard = match &control_lock {
-        Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
-        None => None,
-    };
     // The synthetic key avoids nesting the recovery lock with the config
     // object's own I/O lock. Holding it across the bounded source proof and
     // remote DELETE elects one destructive recovery worker across nodes.
@@ -1194,382 +1073,55 @@ async fn process_transition_transaction_record_at(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
 
-    let mut recovery_control = if transition_state_needs_recovery_control(&current, now_unix_nanos) {
-        if cleanup_terminal_transition_recovery_control(
-            api.clone(),
-            &current,
-            &record_name,
-            &recovery_control_identity,
-            &recovery_control_id,
-        )
-        .await?
-        {
-            return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
-        }
-        match claim_transition_recovery_control(
-            api.clone(),
-            &current,
-            &record_name,
-            recovery_control_identity,
-            &recovery_control_id,
-            now_unix_nanos,
-        )
-        .await?
-        {
-            Some(control) => Some(control),
-            None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
-        }
-    } else {
-        None
-    };
-
-    let recovery = match current.state {
+    match current.state {
         TransitionTransactionState::Uploaded => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
-                Ok(TransitionTransactionRecoveryOutcome::Retained)
-            } else {
-                let mut cleanup = current.clone();
-                cleanup
-                    .mark_cleanup_pending(
-                        current.fence(),
-                        TransitionCleanupProof {
-                            transaction_id: current.transaction_id,
-                            write_id: current.write_id,
-                            remote_object: current.remote_object.clone(),
-                            remote_version: current.remote_version.clone(),
-                            backend_fingerprint: current.backend_fingerprint,
-                            decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
-                        },
-                    )
-                    .map_err(transition_transaction_store_error)?;
-                #[cfg(test)]
-                pause_before_transition_recovery_claim(current.transaction_id).await;
-                match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
-                    Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
-                    Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
-                        Ok(TransitionTransactionRecoveryOutcome::Retained)
-                    }
-                    Err(err) => Err(err),
-                }
+            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
+                return Ok(TransitionTransactionRecoveryOutcome::Retained);
+            }
+            let mut cleanup = current.clone();
+            cleanup
+                .mark_cleanup_pending(
+                    current.fence(),
+                    TransitionCleanupProof {
+                        transaction_id: current.transaction_id,
+                        write_id: current.write_id,
+                        remote_object: current.remote_object.clone(),
+                        remote_version: current.remote_version.clone(),
+                        backend_fingerprint: current.backend_fingerprint,
+                        decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
+                    },
+                )
+                .map_err(transition_transaction_store_error)?;
+            #[cfg(test)]
+            pause_before_transition_recovery_claim(current.transaction_id).await;
+            match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
+                Ok(()) => recover_cleanup_pending(api, &cleanup).await,
+                Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                Err(err) => Err(err),
             }
         }
-        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), &current).await,
+        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, &current).await,
         TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), &current).await {
-            Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
-            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-                IlmRecoveryErrorCode::LocalCommitAmbiguous,
-            )),
-            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-                IlmRecoveryErrorCode::LocalCommitAmbiguous,
-            )),
+            Ok(true) => {
+                delete_transition_transaction_record(api, &current).await?;
+                Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+            }
+            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
             Err(err) => Err(err),
         },
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api.clone(), &current)
-                .await
-                .map(|()| TransitionTransactionRecoveryOutcome::RecordDeleted)
+            delete_transition_transaction_record(api, &current).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
+            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
                 Ok(TransitionTransactionRecoveryOutcome::Retained)
             } else {
-                recover_unknown_upload_outcome(api.clone(), &current).await
+                recover_unknown_upload_outcome(api, &current).await
             }
         }
-        TransitionTransactionState::UploadStarted => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
-                Ok(TransitionTransactionRecoveryOutcome::Retained)
-            } else {
-                Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-                    IlmRecoveryErrorCode::RemoteVersionUnknown,
-                ))
-            }
-        }
-    };
-
-    if let Some(mut control) = recovery_control.take() {
-        let source_to_delete = if matches!(
-            recovery,
-            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
-                | TransitionTransactionRecoveryOutcome::RecordDeleted)
-        ) {
-            let refreshed =
-                refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
-            control = refreshed.0;
-            refreshed.1
-        } else {
-            None
-        };
-        persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
-        if let Some(source) = source_to_delete {
-            #[cfg(test)]
-            pause_after_transition_recovery_terminal(source.transaction_id).await;
-            delete_transition_transaction_record(api, &source).await?;
-        }
-    }
-    recovery
-}
-
-fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
-    IlmRecoveryControlIdentity {
-        protocol: IlmRecoveryProtocol::TransitionTransaction,
-        canonical_source_path: record_name.to_string(),
-        stable_operation_identity: transaction.transaction_id.to_string(),
-        record_class: "transition_transaction_v1".to_string(),
-    }
-}
-
-#[cfg(test)]
-pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
-    let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
-    transition_recovery_control_identity(transaction, &record_name)
-        .source_operation_digest()
-        .map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
-}
-
-fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
-    now_unix_nanos >= transaction.not_after_unix_nanos
-        && !matches!(
-            transaction.state,
-            TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
-        )
-}
-
-async fn cleanup_terminal_transition_recovery_control(
-    api: Arc<ECStore>,
-    transaction: &TransitionTransaction,
-    record_name: &str,
-    identity: &IlmRecoveryControlIdentity,
-    control_id: &str,
-) -> EcstoreResult<bool> {
-    let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
-        Ok(observed) => observed,
-        Err(Error::ConfigNotFound) => return Ok(false),
-        Err(err) => return Err(err),
-    };
-    if observed.control.classification != IlmRecoveryClassification::Terminal {
-        return Ok(false);
-    }
-    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
-    let exact_source = source.is_consistent()
-        && source.generation == observed.control.observed_source_generation
-        && source.canonical_data.as_deref().is_some_and(|data| {
-            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
-        });
-    if observed.control.identity != *identity || !exact_source {
-        return Ok(false);
-    }
-    delete_transition_transaction_record(api, transaction).await?;
-    Ok(true)
-}
-
-async fn claim_transition_recovery_control(
-    api: Arc<ECStore>,
-    transaction: &TransitionTransaction,
-    record_name: &str,
-    identity: IlmRecoveryControlIdentity,
-    control_id: &str,
-    now_unix_nanos: i64,
-) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
-    let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
-        Ok(control) => Some(control),
-        Err(Error::ConfigNotFound) => None,
-        Err(err) => return Err(err),
-    };
-    if existing
-        .as_ref()
-        .is_some_and(|observed| observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos))
-    {
-        return Ok(None);
-    }
-
-    let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
-        Ok(source) => source,
-        Err(err) => {
-            if let Some(observed) = existing {
-                persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
-                return Ok(None);
-            }
-            return Err(err);
-        }
-    };
-    let source_matches = source.is_consistent()
-        && source.canonical_data.as_deref().is_some_and(|data| {
-            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
-        });
-    let source_error = if source_matches {
-        IlmRecoveryErrorCode::None
-    } else if source.canonical_data.is_some() {
-        IlmRecoveryErrorCode::SourceGenerationChanged
-    } else {
-        IlmRecoveryErrorCode::SourceDivergent
-    };
-
-    let mut observed = match existing {
-        Some(control) => control,
-        None => {
-            let candidate = IlmRecoveryControl::new(
-                identity.clone(),
-                source.generation.clone(),
-                if source_matches {
-                    IlmRecoveryClassification::Retrying
-                } else {
-                    IlmRecoveryClassification::Corrupt
-                },
-                now_unix_nanos,
-                source_error,
-            )
-            .map_err(|err| Error::other(err.to_string()))?;
-            match save_recovery_control_if_absent(api.clone(), &candidate).await {
-                Ok(()) | Err(Error::PreconditionFailed) => {}
-                Err(err) => return Err(err),
-            }
-            load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
-        }
-    };
-    if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
-        return Ok(None);
-    }
-
-    let mut claimed = observed.control.clone();
-    claimed
-        .claim_for_source_generation(
-            api.id.to_string(),
-            Uuid::new_v4(),
-            now_unix_nanos,
-            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
-            source.generation,
-        )
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
-    observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
-    if observed.control != claimed {
-        return Err(Error::PreconditionFailed);
-    }
-    if !source_matches {
-        let mut corrupt = observed.control.clone();
-        corrupt
-            .finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
-            .map_err(|err| Error::other(err.to_string()))?;
-        save_recovery_control_if_current(api, &observed, &corrupt).await?;
-        return Ok(None);
-    }
-    Ok(Some(observed))
-}
-
-async fn persist_transition_recovery_source_failure(
-    api: Arc<ECStore>,
-    observed: ObservedIlmRecoveryControl,
-    now_unix_nanos: i64,
-) -> EcstoreResult<()> {
-    let mut claimed = observed.control.clone();
-    claimed
-        .claim(
-            api.id.to_string(),
-            Uuid::new_v4(),
-            now_unix_nanos,
-            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
-        )
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
-    let claimed = load_recovery_control(
-        api.clone(),
-        IlmRecoveryProtocol::TransitionTransaction,
-        &claimed
-            .identity
-            .source_operation_digest()
-            .map_err(|err| Error::other(err.to_string()))?,
-    )
-    .await?;
-    let mut failed = claimed.control.clone();
-    failed
-        .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api, &claimed, &failed).await
-}
-
-async fn refresh_transition_recovery_control_source(
-    api: Arc<ECStore>,
-    mut observed: ObservedIlmRecoveryControl,
-    record_name: &str,
-    transaction_id: Uuid,
-) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
-    let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
-        Ok(transaction) => transaction,
-        Err(Error::ConfigNotFound) => return Ok((observed, None)),
-        Err(err) => return Err(err),
-    };
-    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
-    let exact_source = source.is_consistent()
-        && source
-            .canonical_data
-            .as_deref()
-            .is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
-    if !exact_source {
-        return Err(Error::PreconditionFailed);
-    }
-    if observed.control.observed_source_generation != source.generation {
-        let mut refreshed = observed.control.clone();
-        refreshed
-            .refresh_owned_source_generation(source.generation)
-            .map_err(|err| Error::other(err.to_string()))?;
-        save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
-        observed = load_recovery_control(
-            api,
-            IlmRecoveryProtocol::TransitionTransaction,
-            &refreshed
-                .identity
-                .source_operation_digest()
-                .map_err(|err| Error::other(err.to_string()))?,
-        )
-        .await?;
-        if observed.control != refreshed {
-            return Err(Error::PreconditionFailed);
-        }
-    }
-    Ok((observed, Some(transaction)))
-}
-
-async fn persist_transition_recovery_result(
-    api: Arc<ECStore>,
-    observed: ObservedIlmRecoveryControl,
-    recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
-    now_unix_nanos: i64,
-) -> EcstoreResult<()> {
-    let mut next = observed.control.clone();
-    match recovery {
-        Ok(
-            TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
-        ) => next
-            .finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::Retained) => next
-            .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
-            .finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
-            .finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Err(err) => next
-            .record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
-            .map_err(|err| Error::other(err.to_string()))?,
-    }
-    save_recovery_control_if_current(api, &observed, &next).await
-}
-
-fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
-    match err {
-        Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
-        Error::ConfigNotFound
-        | Error::FileNotFound
-        | Error::FileVersionNotFound
-        | Error::ObjectNotFound(_, _)
-        | Error::VersionNotFound(_, _, _)
-        | Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
-        Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
-        _ => IlmRecoveryErrorCode::Unknown,
+        TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
     }
 }
 
@@ -1582,7 +1134,10 @@ async fn recover_cleanup_pending(
     transaction: &TransitionTransaction,
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     match local_commit_matches_transaction(api.clone(), transaction).await {
-        Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
+        Ok(true) => {
+            delete_transition_transaction_record(api, transaction).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        }
         Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) => Err(err),
@@ -1602,6 +1157,7 @@ async fn delete_unreferenced_transition_candidate(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
     delete_transition_remote_candidate(api.clone(), &current).await?;
+    delete_transition_transaction_record(api, &current).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 
@@ -1622,26 +1178,24 @@ async fn recover_unknown_upload_outcome(
         .await
         .map_err(Error::other)?
     {
-        TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
+        TransitionCandidateProbe::Missing => {
+            delete_transition_transaction_record(api, transaction).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        }
         TransitionCandidateProbe::UnversionedPresent => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
         }
         TransitionCandidateProbe::VersionedPresent(version_id)
             if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
         {
-            Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-                IlmRecoveryErrorCode::RemoteVersionUnknown,
-            ))
+            Ok(TransitionTransactionRecoveryOutcome::Retained)
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
         }
-        TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-            IlmRecoveryErrorCode::RemoteProbeAmbiguous,
-        )),
-        TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-            IlmRecoveryErrorCode::RemoteProbeUnsupported,
-        )),
+        TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
+            Ok(TransitionTransactionRecoveryOutcome::Retained)
+        }
     }
 }
 
@@ -1769,11 +1323,6 @@ async fn recover_transition_transaction_records_with_now(
             false,
         )
         .await?;
-    if list.is_truncated && list.next_continuation_token.is_none() {
-        return Err(Error::other(
-            "transition transaction recovery returned a truncated page without a continuation marker",
-        ));
-    }
 
     let mut stats = TransitionTransactionRecoveryStats {
         scanned: 0,
@@ -1832,11 +1381,7 @@ async fn recover_transition_transaction_records_with_now(
             ) => {
                 stats.recovered += 1;
             }
-            Ok(
-                TransitionTransactionRecoveryOutcome::Retained
-                | TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
-                | TransitionTransactionRecoveryOutcome::OperatorRequired(_),
-            ) => {
+            Ok(TransitionTransactionRecoveryOutcome::Retained) => {
                 stats.retained += 1;
                 debug!(
                     event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
@@ -2423,19 +1968,5 @@ mod tests {
             transition_transaction_record_object_name(Uuid::nil()),
             Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
         ));
-        assert_eq!(
-            transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
-            transaction_id
-        );
-        for malformed in [
-            object.to_ascii_uppercase(),
-            object.replace("/aa/aa/", "/ff/aa/"),
-            object.replace("/aa/aa/", "/aa/aa/extra/"),
-        ] {
-            assert!(matches!(
-                transition_transaction_id_from_record_object_name(&malformed),
-                Err(TransitionTransactionError::Corrupt(_))
-            ));
-        }
     }
 }

crates/ecstore/src/store/init.rs

@@ -814,7 +814,6 @@ mod tests {
                 manual_transition_scope_record_object_name, manual_transition_task_object_name,
                 manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
             },
-            recovery_control::{IlmRecoveryClassification, IlmRecoveryErrorCode, IlmRecoveryProtocol, load_recovery_control},
             tier_delete_journal::{
                 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
                 TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
@@ -834,13 +833,12 @@ mod tests {
             },
             transition_transaction::{
                 TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
-                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
-                TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
-                TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
-                finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
-                load_transition_transaction_record, recover_transition_transaction_records,
-                recover_transition_transaction_records_at, save_transition_transaction_record,
-                save_transition_transaction_record_if_current, transition_recovery_control_id,
+                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity,
+                TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                recover_transition_transaction_records, recover_transition_transaction_records_at,
+                save_transition_transaction_record, save_transition_transaction_record_if_current,
                 transition_transaction_record_object_name,
             },
             validate_durable_ilm_record,
@@ -18856,7 +18854,6 @@ mod tests {
             ),
         ];
         let mut expected_removes = Vec::new();
-        let mut recovery_control_ids = Vec::new();
         for (case, put_version, remote_version, source_mode) in cases {
             let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
                 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
@@ -18894,8 +18891,6 @@ mod tests {
             save_transition_transaction_record(store.clone(), &transaction)
                 .await
                 .expect("transaction record should persist");
-            recovery_control_ids
-                .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
             expected_removes.push((transaction.remote_object, put_version));
         }
 
@@ -18911,14 +18906,6 @@ mod tests {
         assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(backend.object_count().await, 0);
-        for control_id in recovery_control_ids {
-            let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
-                .await
-                .expect("completed recovery control should remain inspectable");
-            assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
-            assert_eq!(control.control.attempt_count, 1);
-            assert!(control.control.owner.is_none());
-        }
 
         let replay = recover_transition_transaction_records(store, 100, None)
             .await
@@ -18931,94 +18918,6 @@ mod tests {
         );
     }
 
-    #[cfg(feature = "test-util")]
-    #[tokio::test]
-    #[serial_test::serial(storage_class_env)]
-    async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
-        let temp_dir = tempfile::tempdir().expect("create temp store dir");
-        let (ctx, store, _shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
-                .await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-
-        let tier_name = "TXTERMINALCRASH";
-        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
-        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
-            .await
-            .expect("tier lease should resolve")
-            .backend_identity();
-        let remote_version = uuid::Uuid::new_v4().to_string();
-        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
-            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
-            transaction_id: uuid::Uuid::new_v4(),
-            owner_epoch: uuid::Uuid::new_v4(),
-            write_id: uuid::Uuid::new_v4(),
-            source: TransitionSourceIdentity {
-                bucket: "source-bucket".to_string(),
-                object: "source-object".to_string(),
-                version_id: Some(uuid::Uuid::new_v4()),
-                data_dir: uuid::Uuid::new_v4(),
-                mod_time_unix_nanos: 1_770_000_000_000_000_000,
-                size: 42,
-                etag: "source-etag".to_string(),
-                version_mode: TransitionSourceVersionMode::Versioned,
-            },
-            tier_name: tier_name.to_string(),
-            backend_fingerprint: backend_identity,
-            not_after_unix_nanos: 1,
-        })
-        .expect("transaction should build");
-        transaction
-            .advance(
-                transaction.fence(),
-                TransitionTransactionState::Uploaded,
-                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
-            )
-            .expect("transaction should enter uploaded state");
-        backend.set_put_remote_version(Some(remote_version)).await;
-        let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
-        backend
-            .put(
-                &transaction.remote_object,
-                ReaderImpl::Body(candidate.clone()),
-                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
-            )
-            .await
-            .expect("mock backend should accept candidate");
-        save_transition_transaction_record(store.clone(), &transaction)
-            .await
-            .expect("transaction record should persist");
-        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
-
-        let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
-        let recovery_store = store.clone();
-        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
-        barrier.wait_until_paused().await;
-        let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
-            .await
-            .expect("terminal control should persist before source cleanup");
-        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
-        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
-        assert_eq!(backend.object_count().await, 0);
-        assert_eq!(backend.exact_remove_count(), 1);
-
-        recovery.abort();
-        assert!(
-            recovery
-                .await
-                .expect_err("recovery should be cancelled at the crash boundary")
-                .is_cancelled()
-        );
-        drop(barrier);
-
-        let replay = recover_transition_transaction_records(store.clone(), 100, None)
-            .await
-            .expect("terminal control should resume source cleanup without another remote delete");
-        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
-        assert_eq!(transition_transaction_record_count(store).await, 0);
-        assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
-    }
-
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19076,8 +18975,6 @@ mod tests {
         save_transition_transaction_record(store.clone(), &uploaded)
             .await
             .expect("transaction record should persist");
-        let recovery_control_id =
-            transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
 
         let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
         let recovery_store = store.clone();
@@ -19099,17 +18996,11 @@ mod tests {
             .expect("recovery should treat the lost CAS as a retained transaction");
         assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
         assert_eq!(
-            load_transition_transaction_record(store.clone(), uploaded.transaction_id)
+            load_transition_transaction_record(store, uploaded.transaction_id)
                 .await
                 .expect("newer transaction revision must remain"),
             active
         );
-        let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("lost source CAS should retain a retryable recovery control");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
-        assert_eq!(control.control.consecutive_failure_count, 1);
-        assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
         assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
         assert_eq!(backend.remove_count().await, 0);
     }
@@ -19411,15 +19302,27 @@ mod tests {
             not_after_unix_nanos: 1_780_000_000_000_000_000,
         })
         .expect("transaction should build");
-        transaction
+        let uploaded_fence = transaction
             .advance(
                 transaction.fence(),
                 TransitionTransactionState::Uploaded,
-                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
+                Some(TransitionRemoteVersion::versioned(remote_version)),
             )
             .expect("transaction should enter uploaded state");
+        transaction
+            .mark_cleanup_pending(
+                uploaded_fence,
+                TransitionCleanupProof {
+                    transaction_id: transaction.transaction_id,
+                    write_id: transaction.write_id,
+                    remote_object: transaction.remote_object.clone(),
+                    remote_version: transaction.remote_version.clone(),
+                    backend_fingerprint: transaction.backend_fingerprint,
+                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
+                },
+            )
+            .expect("transaction should enter cleanup pending state");
         let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
-        backend.set_put_remote_version(Some(remote_version)).await;
         backend
             .put(
                 &transaction.remote_object,
@@ -19431,8 +19334,6 @@ mod tests {
         save_transition_transaction_record(store.clone(), &transaction)
             .await
             .expect("transaction record should persist");
-        let recovery_control_id =
-            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
 
         backend.set_remove_failure(true);
         let stats = recover_transition_transaction_records(store.clone(), 100, None)
@@ -19448,42 +19349,6 @@ mod tests {
         assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
         assert_eq!(backend.exact_remove_count(), 1);
         assert_eq!(backend.object_count().await, 1);
-        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("failed recovery control should persist");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
-        assert_eq!(control.control.attempt_count, 1);
-        assert_eq!(control.control.consecutive_failure_count, 1);
-        assert!(
-            control
-                .control
-                .next_attempt_at_unix_nanos
-                .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
-        );
-
-        backend.set_remove_failure(false);
-        let replay = recover_transition_transaction_records(store.clone(), 100, None)
-            .await
-            .expect("recovery before the persisted deadline should be skipped");
-        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
-        assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
-
-        let retry_at = control
-            .control
-            .next_attempt_at_unix_nanos
-            .expect("retry deadline should persist");
-        let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
-            .await
-            .expect("recovery at the persisted deadline should retry the advanced source generation");
-        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
-        assert_eq!(backend.exact_remove_count(), 2);
-        assert_eq!(backend.object_count().await, 0);
-        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
-        let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("completed retry control should remain inspectable");
-        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
-        assert_eq!(terminal.control.attempt_count, 2);
     }
 
     #[cfg(feature = "test-util")]
@@ -19784,10 +19649,6 @@ mod tests {
         local_commit_started
             .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
             .expect("transaction should enter local commit state");
-        let upload_started_control_id =
-            transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
-        let local_commit_control_id =
-            transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
 
         backend.set_put_remote_version(Some(remote_version)).await;
         for transaction in [&upload_started, &local_commit_started] {
@@ -19818,19 +19679,6 @@ mod tests {
         assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
         assert_eq!(backend.remove_count().await, 0);
         assert_eq!(backend.exact_remove_count(), 0);
-        let upload_started_control =
-            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
-                .await
-                .expect("upload-started control should persist");
-        assert_eq!(
-            upload_started_control.control.classification,
-            IlmRecoveryClassification::RetainedAmbiguous
-        );
-        let local_commit_control =
-            load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
-                .await
-                .expect("local-commit control should persist");
-        assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
     }
 
     #[cfg(feature = "test-util")]
@@ -20181,12 +20029,6 @@ mod tests {
             "an unsupported provider probe must retain the unknown upload"
         );
         assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
-        let recovery_control_id =
-            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
-        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("unsupported probe control should persist");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
         assert!(
             backend.contains(&transaction.remote_object).await,
             "unsupported recovery must not delete the candidate"

docs/testing/ci-gates.md

@@ -91,6 +91,12 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 
 Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline.
 
+## Packaged functional acceptance
+
+`rustfs-functional-chain.yml` dispatches the packaged-build suites in `rustfs-*-test.yml` on the shared lab runners. A failing suite step or job must fail its workflow. Report collection, cleanup, and dispatch of the next suite can still run with `always()`; continuing diagnostics does not make the failed suite successful.
+
+Workflow status preserves errors that the test scripts report. It does not establish complete execution or a common package identity across the chain: inspect the current run's case results, package identity, and test-script revision as well. A script that returns zero after a failed tool invocation needs its own result check.
+
 ## Release validation
 
 Post-merge and tag-driven; not a substitute for a PR gate.

rustfs/src/admin/handlers/ilm_transition.rs

@@ -18,20 +18,18 @@ use crate::admin::runtime_sources::object_store_from_extensions;
 use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket;
 use crate::admin::storage_api::error::StorageError;
 use crate::admin::storage_api::lifecycle::{
-    IlmRecoveryClassification, IlmRecoveryProtocol, ManualTransitionCancelCheck, ManualTransitionJobRecord,
-    ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, ManualTransitionRunOptions,
-    ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
-    TransitionOperatorDeleteResult, TransitionOperatorError, claim_manual_transition_scope_admission,
-    delete_manual_transition_scope_admission_if_current, delete_transition_candidate_for_operator,
-    enqueue_transition_for_existing_objects_scoped, finalize_missing_transition_transaction_for_operator,
-    inspect_recovery_control, inspect_transition_transaction_for_operator, list_recovery_controls,
+    ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
+    ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
+    ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
+    claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
+    delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
+    finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
     load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired,
     manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
     persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned,
     request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record,
 };
 use crate::admin::storage_api::runtime::ECStore;
-use crate::admin::storage_api::s3::{S3ErrorCode as AdminS3ErrorCode, error as admin_s3_error};
 use crate::admin::utils::json_response;
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
 use http::HeaderMap;
@@ -232,48 +230,9 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
         format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
         AdminOperation(&TransitionReconcileApplyHandler {}),
     )?;
-    r.insert(
-        Method::GET,
-        format!("{ADMIN_PREFIX}/v3/ilm/recovery/records").as_str(),
-        AdminOperation(&IlmRecoveryControlListHandler {}),
-    )?;
-    r.insert(
-        Method::GET,
-        format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(),
-        AdminOperation(&IlmRecoveryControlInspectHandler {}),
-    )?;
     Ok(())
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-struct IlmRecoveryControlListQuery {
-    protocol: IlmRecoveryProtocol,
-    #[serde(default)]
-    classification: Option<IlmRecoveryClassification>,
-    #[serde(default = "default_recovery_control_list_limit")]
-    limit: usize,
-    #[serde(default)]
-    marker: Option<String>,
-}
-
-const fn default_recovery_control_list_limit() -> usize {
-    100
-}
-
-fn parse_recovery_control_list_query(query: Option<&str>) -> S3Result<IlmRecoveryControlListQuery> {
-    let query = query.ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "protocol is required"))?;
-    let parsed: IlmRecoveryControlListQuery = serde_urlencoded::from_bytes(query.as_bytes())
-        .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control query"))?;
-    if !(1..=1_000).contains(&parsed.limit) {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "limit must be between 1 and 1000"));
-    }
-    if parsed.marker.as_ref().is_some_and(|marker| marker.is_empty()) {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "marker must not be empty"));
-    }
-    Ok(parsed)
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum ManualTransitionRunMode {
     EnqueueOnly,
@@ -464,26 +423,6 @@ fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uu
         .map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
 }
 
-fn recovery_control_id_from_params(params: &Params<'_, '_>) -> S3Result<String> {
-    let control_id = params.get("control_id").unwrap_or("");
-    if control_id.len() != 64
-        || !control_id
-            .bytes()
-            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
-    {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control id"));
-    }
-    Ok(control_id.to_string())
-}
-
-fn map_recovery_control_error(err: StorageError) -> S3Error {
-    if err == StorageError::ConfigNotFound {
-        admin_s3_error(AdminS3ErrorCode::NoSuchKey, "ILM recovery control not found")
-    } else {
-        admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery control request failed")
-    }
-}
-
 fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
     match err {
         TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
@@ -1092,40 +1031,6 @@ impl Operation for TransitionReconcileInspectHandler {
     }
 }
 
-pub struct IlmRecoveryControlListHandler {}
-
-#[async_trait::async_trait]
-impl Operation for IlmRecoveryControlListHandler {
-    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
-        let query = parse_recovery_control_list_query(req.uri.query())?;
-        let Some(store) = object_store_from_extensions(&req.extensions) else {
-            return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized"));
-        };
-        let page = list_recovery_controls(store, query.protocol, query.classification, query.limit, query.marker)
-            .await
-            .map_err(map_recovery_control_error)?;
-        json_response(StatusCode::OK, &page)
-    }
-}
-
-pub struct IlmRecoveryControlInspectHandler {}
-
-#[async_trait::async_trait]
-impl Operation for IlmRecoveryControlInspectHandler {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
-        let control_id = recovery_control_id_from_params(&params)?;
-        let Some(store) = object_store_from_extensions(&req.extensions) else {
-            return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized"));
-        };
-        let control = inspect_recovery_control(store, &control_id)
-            .await
-            .map_err(map_recovery_control_error)?;
-        json_response(StatusCode::OK, &control)
-    }
-}
-
 pub struct TransitionReconcileApplyHandler {}
 
 #[async_trait::async_trait]
@@ -1199,49 +1104,6 @@ mod tests {
         f(&matched.params)
     }
 
-    fn with_recovery_control_params<T>(path: &str, f: impl FnOnce(&Params<'_, '_>) -> T) -> T {
-        let mut router = Router::new();
-        router
-            .insert("/rustfs/admin/v3/ilm/recovery/records/{control_id}", ())
-            .expect("route should insert");
-        let matched = router.at(path).expect("route should match");
-        f(&matched.params)
-    }
-
-    #[test]
-    fn recovery_control_query_is_bounded_and_strict() {
-        let query = parse_recovery_control_list_query(Some("protocol=transition_transaction"))
-            .expect("minimal recovery query should parse");
-        assert_eq!(query.protocol, IlmRecoveryProtocol::TransitionTransaction);
-        assert_eq!(query.classification, None);
-        assert_eq!(query.limit, 100);
-
-        let filtered = parse_recovery_control_list_query(Some(
-            "protocol=tier_delete_journal&classification=retained_ambiguous&limit=1000&marker=opaque",
-        ))
-        .expect("bounded filtered query should parse");
-        assert_eq!(filtered.protocol, IlmRecoveryProtocol::TierDeleteJournal);
-        assert_eq!(filtered.classification, Some(IlmRecoveryClassification::RetainedAmbiguous));
-        assert_eq!(filtered.limit, 1000);
-        assert!(parse_recovery_control_list_query(None).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=0")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=1001")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=unknown")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&extra=true")).is_err());
-    }
-
-    #[test]
-    fn recovery_control_id_is_canonical_lowercase_sha256() {
-        let id = "ab".repeat(32);
-        with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{id}"), |params| {
-            assert_eq!(recovery_control_id_from_params(params).expect("control id should parse"), id);
-        });
-        let uppercase = "AB".repeat(32);
-        with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{uppercase}"), |params| {
-            assert!(recovery_control_id_from_params(params).is_err())
-        });
-    }
-
     fn manual_transition_job_request(method: Method, path: &'static str) -> S3Request<Body> {
         S3Request {
             input: Body::empty(),

rustfs/src/admin/storage_api.rs

@@ -232,9 +232,6 @@ pub(crate) mod lifecycle {
     pub(crate) type ManualTransitionRunOptions =
         super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
     pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
-    pub(crate) use super::ecstore_bucket::lifecycle::recovery_control::{
-        IlmRecoveryClassification, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls,
-    };
     pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
         TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
         finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,

scripts/functional_case_report.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Preserve every functional case execution and its suite context in reports."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import re
+
+
+def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
+    ansi = re.compile(r"\x1b\[[0-9;]*m")
+    start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
+    done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
+    context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
+    topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
+    rows = []
+    pending = {}
+    topo_rows = []
+    context = "context not recorded"
+    complete = True
+    try:
+        with log_file.open(encoding="utf-8", errors="replace") as log:
+            for raw in log:
+                line = ansi.sub("", raw).strip()
+                if match := context_re.match(line):
+                    context = match[1]
+                    pending.clear()
+                elif match := topo_re.match(line):
+                    topo_rows.append(match.groups())
+                elif match := start_re.match(line):
+                    case_id, name = match.groups()
+                    pending[case_id] = len(rows)
+                    rows.append([case_id, f"{name} ({context})", "RUNNING"])
+                elif match := done_re.match(line):
+                    status, case_id = match.groups()
+                    index = pending.pop(case_id, None)
+                    if index is None:
+                        complete = False
+                        rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
+                    else:
+                        rows[index][2] = status
+    except FileNotFoundError:
+        pass
+
+    counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
+    with case_file.open("w", encoding="utf-8") as out:
+        out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
+        for status, count in counts.items():
+            out.write(f"- {status}: {count}\n")
+        out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
+        for row in rows:
+            out.write("| " + " | ".join(value.replace("|", "|") for value in row) + " |\n")
+        if not rows:
+            out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
+
+    valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
+    if matrix_file is not None:
+        with matrix_file.open("w", encoding="utf-8") as out:
+            out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
+            out.write("| --- | --- | --- | --- | --- |\n")
+            for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                result = "PASS" if nfail == "0" else "FAIL"
+                out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
+            if not topo_rows:
+                out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
+        valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
+    return valid
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("log_file", type=Path)
+    parser.add_argument("case_file", type=Path)
+    parser.add_argument("matrix_file", type=Path, nargs="?")
+    args = parser.parse_args()
+    raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)

scripts/test_security_workflow.py

@@ -1,16 +1,18 @@
 #!/usr/bin/env python3
-"""Run the security workflow's evidence and result steps without remote VMs."""
+"""Exercise functional workflow failures and security evidence without remote VMs."""
 
 from __future__ import annotations
 
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import unittest
 from pathlib import Path
 
 from check_test_wiring import yaml_block
+from functional_case_report import generate_report
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -18,16 +20,61 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
 CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
 
 
-class SecurityWorkflowTests(unittest.TestCase):
+def named_steps(job: list[str]) -> dict[str, list[str]]:
+    starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
+    return {
+        job[start].split(": ", 1)[1].strip('"'): job[start:end]
+        for start, end in zip(starts, starts[1:] + [len(job)])
+    }
+
+
+def shell_body(lines: list[str]) -> str:
+    start = lines.index("        run: |") + 1
+    shell_lines = []
+    for line in lines[start:]:
+        if line.strip() and not line.startswith("          "):
+            break
+        shell_lines.append(line[10:])
+    if not shell_lines:
+        raise ValueError("missing literal shell body")
+    return "\n".join(shell_lines)
+
+
+class WorkflowSteps:
+    def render(self, value: str) -> str:
+        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+        result = {}
+        for line in yaml_block(lines, "env", indent) or []:
+            if line.strip() and not line.lstrip().startswith("#"):
+                key, value = line.strip().split(": ", 1)
+                result[key] = self.render(value.strip("'\""))
+        return result
+
+    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+        lines = self.steps[name]
+        result = subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+        )
+        for line in lines:
+            if line.startswith("        id: "):
+                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+        if Path(self.env["GITHUB_ENV"]).exists():
+            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+                key, value = line.split("=", 1)
+                self.env[key] = value
+                self.context[f"env.{key}"] = value
+        return result
+
+
+class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
         self.assertIsNotNone(self.job)
-        starts = [i for i, line in enumerate(self.job) if line.startswith("      - name: ")]
-        self.steps = {
-            self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
-            for start, end in zip(starts, starts[1:] + [len(self.job)])
-        }
+        self.steps = named_steps(self.job)
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
         self.directory = Path(self.temp.name)
@@ -70,40 +117,6 @@ class SecurityWorkflowTests(unittest.TestCase):
             'exit "$FAKE_EXIT"\n'
         )
 
-    def render(self, value: str) -> str:
-        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
-    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
-        result = {}
-        for line in yaml_block(lines, "env", indent) or []:
-            if line.strip() and not line.lstrip().startswith("#"):
-                key, value = line.strip().split(": ", 1)
-                result[key] = self.render(value.strip("'\""))
-        return result
-
-    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
-        lines = self.steps[name]
-        start = lines.index("        run: |") + 1
-        shell_lines = []
-        for line in lines[start:]:
-            if line.strip() and not line.startswith("          "):
-                break
-            shell_lines.append(line[10:])
-        self.assertTrue(shell_lines, f"missing literal shell body: {name}")
-        result = subprocess.run(
-            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))],
-            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
-        )
-        for line in lines:
-            if line.startswith("        id: "):
-                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
-        if Path(self.env["GITHUB_ENV"]).exists():
-            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
-                key, value = line.split("=", 1)
-                self.env[key] = value
-                self.context[f"env.{key}"] = value
-        return result
-
     def test_workflow_wiring(self) -> None:
         names = list(self.steps)
         self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -193,5 +206,391 @@ class SecurityWorkflowTests(unittest.TestCase):
         self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
 
+class FunctionalWorkflowTests(unittest.TestCase):
+    JOBS = {
+        "kms": "kms-test", "storage": "storage-test", "s3-compat": "s3-compat-test",
+        "upgrade": "upgrade-test", "replication": "replication-test", "heal": "heal-test",
+        "tier": "tier-test", "pool-expand": "pool-expansion-test", "performance": "performance-test",
+    }
+    DIRECT_TESTS = {
+        "kms": "Run KMS suite", "storage": "Run storage engine suite",
+        "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite",
+        "replication": "Run replication suite",
+    }
+
+    def test_failure_and_always_step_wiring(self) -> None:
+        for suite, job_id in self.JOBS.items():
+            with self.subTest(suite=suite):
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+                job = yaml_block(source.splitlines(), job_id, 2)
+                self.assertIsNotNone(job)
+                self.assertNotRegex("\n".join(job), r'''(?m)^    ["']?continue-on-error["']?\s*:''')
+                steps = named_steps(job)
+                if suite in self.DIRECT_TESTS:
+                    test = steps[self.DIRECT_TESTS[suite]]
+                    self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
+                    self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
+                cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
+                condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
+                self.assertIn(condition, (
+                    "if: always()",
+                    "if: ${{ always() && inputs.cleanup_after != 'false' }}",
+                    "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
+                ))
+                if suite != "performance":
+                    handoff = steps["Chain complete"] if suite == "replication" else next(
+                        value for name, value in steps.items() if name.startswith("Continue functional chain")
+                    )
+                    self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
+
+    def test_failed_suite_preserves_exit_and_cleanup_and_dispatch_execute(self) -> None:
+        for suite, test_name in self.DIRECT_TESTS.items():
+            with self.subTest(suite=suite), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                (root / "auto-testing").mkdir()
+                script = root / f"auto-testing/rustfs-{suite}-test.sh"
+                script.write_text('#!/bin/sh\nprintf "partial suite diagnostics\\n"\nexit 17\n')
+                script.chmod(0o755)
+                fake_bin = root / "bin"
+                fake_bin.mkdir()
+                for command, marker in (("ssh", "cleanup"), ("gh", "dispatch")):
+                    fake = fake_bin / command
+                    fake.write_text(f'#!/bin/sh\nprintf "{marker}\\n" >> "$EXECUTED"\n')
+                    fake.chmod(0o755)
+                env = {
+                    **os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}",
+                    "EXECUTED": str(root / "executed"), "RUSTFS_NODES": "fixture-node",
+                    "RUSTFS_SSH_USER": "fixture-user", "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb",
+                    "GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "repository_dispatch", "GITHUB_RUN_ID": "314159",
+                }
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+                steps = named_steps(yaml_block(source.splitlines(), self.JOBS[suite], 2))
+                context = {"github.event_name": "repository_dispatch", "steps.test.outcome": "failure"}
+                for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", source):
+                    if expression.startswith("inputs.") and re.fullmatch(r"inputs\.\w+", expression):
+                        context[expression] = ""
+                def execute(name):
+                    lines = steps[name]
+                    rendered = re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: context[match[1]], shell_body(lines))
+                    return subprocess.run(
+                        ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered],
+                        cwd=root, env={**env, "LOG_FILE": str(root / "suite.log")}, capture_output=True, text=True,
+                    )
+                failed = execute(test_name)
+                self.assertEqual(failed.returncode, 17, failed.stderr)
+                self.assertIn("partial suite diagnostics", failed.stdout)
+                cleanup = execute("Cleanup environment (after)")
+                self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
+                handoff_name = "Chain complete" if suite == "replication" else next(
+                    name for name in steps if name.startswith("Continue functional chain")
+                )
+                handoff = execute(handoff_name)
+                self.assertEqual(handoff.returncode, 0, handoff.stderr)
+                markers = (root / "executed").read_text().splitlines()
+                self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
+
+
+class FunctionalCaseReportTests(unittest.TestCase):
+    def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            log = root / "suite.log"
+            if text is not None:
+                log.write_text(text)
+            valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
+            return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
+
+    def test_repeated_case_executions_preserve_failure_and_context(self):
+        # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
+        log = subprocess.check_output(["bash", "-c", r'''
+log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
+log '== topology: single-single  kms-backend: local =='
+printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
+log '== topology: single-multi  kms-backend: vault-kv2 =='
+printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
+printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
+'''], text=True)
+        valid, cases, _ = self.report(log)
+        self.assertFalse(valid)
+        self.assertEqual(cases.count("| KMS-101 |"), 3)
+        self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
+        self.assertIn("roundtrip (topology: single-single  kms-backend: local) | FAIL |", cases)
+        self.assertIn("roundtrip (topology: single-multi  kms-backend: vault-kv2) | PASS |", cases)
+        self.assertNotIn("\\n", cases)
+
+    def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
+        for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
+            with self.subTest(log=text):
+                valid, cases, _ = self.report(text)
+                self.assertFalse(valid)
+                self.assertIn("## Case Summary", cases)
+        valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
+        self.assertTrue(valid)
+        self.assertIn("suite: bucket replication", cases)
+        self.assertIn("- UNSUPPORTED: 1\n", cases)
+
+    def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
+        case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
+        for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
+                                 ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
+            with self.subTest(matrix=suffix):
+                valid, _, matrix = self.report(case + suffix, matrix=True)
+                self.assertEqual(valid, expected)
+                self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
+                self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
+
+    def test_s3_case_identifiers_include_digits(self):
+        valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
+        self.assertTrue(valid)
+        self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
+
+
+class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
+    SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
+
+    def prepare(self, suite: str) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+        self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
+        self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
+        self.context.update({
+            "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
+            "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
+            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+            "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
+        })
+        self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
+        self.env = {
+            **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
+            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
+            "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
+        }
+        for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
+            self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
+        (self.directory / "scripts").mkdir()
+        (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
+        fake_bin = self.directory / "bin"
+        fake_bin.mkdir()
+        (fake_bin / "python3").symlink_to(sys.executable)
+        for command, body in (
+            ("ssh", 'printf "fixture-version\\n"\n'),
+            ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
+                   '  while [ "$#" -gt 0 ]; do\n'
+                   '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                   '    shift\n'
+                   '  done\n'
+                   'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
+        ):
+            script = fake_bin / command
+            script.write_text("#!/bin/sh\n" + body)
+            script.chmod(0o755)
+        self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
+
+    def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertNotIn("/tmp/rustfs-", self.source)
+                names = list(self.steps)
+                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+                if suite in FunctionalWorkflowTests.DIRECT_TESTS:
+                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+                for name, lines in self.steps.items():
+                    if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
+                    if any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/", lines)
+                        self.assertIn("          if-no-files-found: error", lines)
+                self.artifacts.mkdir()
+                for filename in ("report.md", "suite.log"):
+                    (self.artifacts / filename).write_text("OLD RUN EVIDENCE")
+                self.env.update(REPORT_FILE=str(self.artifacts / "report.md"), LOG_FILE=str(self.artifacts / "suite.log"))
+                initialized = self.run_step("Initialize functional evidence")
+                self.assertNotEqual(initialized.returncode, 0)
+                self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+                issue = self.run_step("File failure issue in rustfs/backlog")
+                self.assertEqual(issue.returncode, 0, issue.stderr)
+                body = Path(self.env["CAPTURE_BODY"]).read_text()
+                self.assertNotIn("OLD RUN EVIDENCE", body)
+                self.assertIn("no report or log file was produced", body)
+                self.assertEqual((self.artifacts / "report.md").read_text(), "OLD RUN EVIDENCE")
+
+    def test_reports_use_only_current_complete_suite_evidence(self):
+        for suite in self.SUITES[:-1]:
+            good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
+            partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
+            if suite == "s3-compat":
+                good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
+            if suite == "upgrade":
+                good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
+            if suite == "heal":
+                good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
+                partial = "[HEAL-STEP] 1 fixture PASS\n"
+            for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
+                                 ("success", None), ("skipped", None), ("cancelled", good)):
+                with self.subTest(suite=suite, outcome=outcome, log=log):
+                    self.prepare(suite)
+                    stale = self.directory / "old-suite.log"
+                    stale.write_text("OLD RUN EVIDENCE\n" + good)
+                    self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
+                    self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                    self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
+                    self.assertEqual(self.env["TMPDIR"], str(self.artifacts))
+                    if log is not None:
+                        Path(self.env["LOG_FILE"]).write_text(log)
+                    self.context["steps.test.outcome"] = outcome
+                    report = self.run_step("Generate report")
+                    success = outcome == "success" and log == good
+                    self.assertEqual(report.returncode == 0, success, report.stderr)
+                    contents = Path(self.env["REPORT_FILE"]).read_text()
+                    self.assertNotIn("OLD RUN EVIDENCE", contents)
+                    self.assertEqual("| PASS |" in contents, success)
+                    for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
+                                  f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
+                        self.assertIn(value, contents)
+                    self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
+                    evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
+                    if log in (good, partial):
+                        self.assertIn("| PASS |", evidence)
+                    self.assertNotIn("OLD RUN EVIDENCE", evidence)
+
+    def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                scripts = self.directory / "auto-testing"
+                scripts.mkdir()
+                filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
+                script = scripts / filename
+                script.write_text(
+                    '#!/bin/bash\nset -euo pipefail\nlog=""\n'
+                    'while [ "$#" -gt 0 ]; do\n'
+                    '  if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
+                    '  shift\n'
+                    'done\n'
+                    '[ "$log" = "$LOG_FILE" ] || exit 31\n'
+                    'printf "CURRENT SUITE LOG\\n" > "$log"\n'
+                    'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
+                    'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
+                    'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
+                    '  mkdir -p "$RUSTFS_RESULT_DIR"\n'
+                    '  printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
+                    'fi\n'
+                )
+                script.chmod(0o755)
+                name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
+                    "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
+                )
+                result = self.run_step(name)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
+                self.assertEqual(len(list(self.artifacts.glob("fixture.*/trace.log"))), 1)
+                if suite == "performance":
+                    self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
+
+    def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
+        self.prepare("heal")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        script = self.directory / "auto-testing/rustfs_heal_test.sh"
+        script.parent.mkdir()
+        # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
+        script.write_text(r'''#!/bin/bash
+set -euo pipefail
+SELECTED_STEPS=()
+PREFLIGHT=0
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
+    --log-file) LOG_FILE="$2"; shift ;;
+    --preflight) PREFLIGHT=1 ;;
+  esac
+  shift
+done
+if [ "$PREFLIGHT" -eq 1 ]; then
+  printf '\n' >> "$INVOKED_STEPS"
+  exit 0
+fi
+printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
+emit_step_result() {
+  local n="$1" desc="$2" status="$3"
+  printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
+}
+{
+  for step in "${SELECTED_STEPS[@]}"; do
+    emit_step_result "$step" "fixture step $step" PASS
+  done
+  want_all=1
+  for s in 1 2 3 4 5 6 7; do
+    [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
+  done
+  if [ "${want_all}" -eq 1 ]; then
+    printf '[HEAL-RESULT] PASS all steps passed\n'
+  fi
+} >> "$LOG_FILE"
+''')
+        script.chmod(0o755)
+        self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
+        for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
+            result = self.run_step(name)
+            self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
+        log = Path(self.env["LOG_FILE"]).read_text()
+        self.assertNotIn("[HEAL-RESULT]", log)
+        self.assertEqual(log.count("[HEAL-STEP]"), 7)
+        report = self.run_step("Generate report")
+        self.assertEqual(report.returncode, 0, report.stderr)
+        failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
+                       for step in range(1, 8)]
+        failed_logs += [
+            log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
+            log + "[HEAL-STEP] 3 later step failure FAIL\n",
+            log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
+            log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
+        ]
+        for failed_log in failed_logs:
+            with self.subTest(log=failed_log):
+                Path(self.env["LOG_FILE"]).write_text(failed_log)
+                report = self.run_step("Generate report")
+                self.assertNotEqual(report.returncode, 0, report.stderr)
+                contents = Path(self.env["REPORT_FILE"]).read_text()
+                self.assertIn("Test Step Outcome: failure", contents)
+                self.assertNotIn("| PASS |", contents)
+                if "original failure" in failed_log:
+                    self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
+                if "later step failure" in failed_log:
+                    self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
+
+    def test_performance_results_version_and_report_are_bound_to_the_run(self):
+        self.prepare("performance")
+        initialized = self.run_step("Initialize functional evidence")
+        self.assertEqual(initialized.returncode, 0, initialized.stderr)
+        self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
+        self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
+        version = self.run_step("Collect RustFS version info")
+        self.assertEqual(version.returncode, 0, version.stderr)
+        self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
+        old_summary = self.directory / "old-results/summary.md"
+        old_summary.parent.mkdir()
+        old_summary.write_text("OLD RUN EVIDENCE")
+        upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
+        self.assertNotEqual(self.run_step(upload).returncode, 0)
+        self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
+        results = Path(self.env["RUSTFS_RESULT_DIR"])
+        results.mkdir()
+        (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
+        report = self.run_step(upload)
+        self.assertEqual(report.returncode, 0, report.stderr)
+        contents = Path(self.env["REPORT_FILE"]).read_text()
+        for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
+                      "CURRENT PERFORMANCE RESULTS", "fixture-version"):
+            self.assertIn(value, contents)
+        self.assertNotIn("OLD RUN EVIDENCE", contents)
+
+
 if __name__ == "__main__":
     unittest.main()