fix(ci): isolate functional evidence and preserve every result

.github/workflows/rustfs-heal-test.yml

@@ -59,6 +59,20 @@ jobs:
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -114,7 +128,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -124,7 +138,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -134,13 +148,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -149,8 +160,9 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
           import re
           import sys
 
@@ -162,6 +174,7 @@ jobs:
 
           steps = {}
           order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
           version = None
           version_node = None
           verdict = None
@@ -175,14 +188,15 @@ jobs:
                           n, desc, status = m.group(1), m.group(2), m.group(3)
                           if n not in steps:
                               order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
                           continue
                       m = ver_re.match(line)
                       if m:
                           version, version_node = m.group(1), m.group(2)
                           continue
                       m = result_re.match(line)
-                      if m:
+                      if m and verdict != 'FAIL':
                           verdict, verdict_detail = m.group(1), m.group(2)
           except FileNotFoundError:
               pass
@@ -202,30 +216,43 @@ jobs:
                   out.write(f'| {n} | {desc} | {status} |\n')
               if not order:
                   out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
           PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${STEPS_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -252,11 +279,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -284,14 +310,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -307,14 +335,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-heal-test*.log
+          if-no-files-found: error
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}

.github/workflows/rustfs-kms-test.yml

@@ -52,6 +52,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -108,8 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -139,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -154,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          CASE_RESULT=success
-          import re
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          import sys
+          RESULT=failure
-
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-          log_file, out_file = sys.argv[1], sys.argv[2]
+            RESULT=success
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          fi
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${CASE_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -253,11 +231,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -285,14 +262,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -308,14 +287,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-kms.log
+          if-no-files-found: error
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-performance-test.yml

@@ -76,8 +76,6 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
@@ -89,6 +87,22 @@ jobs:
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -120,7 +134,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -130,7 +144,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -140,7 +154,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -153,17 +167,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -183,7 +195,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -191,7 +202,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -199,6 +210,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -208,8 +221,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
+          } > "${REPORT_FILE}"
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -228,11 +241,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -260,14 +272,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -283,20 +297,17 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-perf-test*.log
+          if-no-files-found: error
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-replication-test.yml

@@ -65,6 +65,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -113,8 +132,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -137,10 +154,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -162,80 +176,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          CASE_RESULT=success
-          import re
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          import sys
+          RESULT=failure
-
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-          log_file, out_file = sys.argv[1], sys.argv[2]
+            RESULT=success
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          fi
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${CASE_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -262,11 +240,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -294,14 +271,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -317,14 +296,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-replication.log
+          if-no-files-found: error
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-s3-compat-test.yml

@@ -40,6 +40,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -87,8 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -105,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -130,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          CASE_RESULT=success
-          import re
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          import sys
+          RESULT=failure
-
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-          log_file, out_file = sys.argv[1], sys.argv[2]
+            RESULT=success
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          fi
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${CASE_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -233,11 +208,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -265,14 +239,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -288,14 +264,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-s3-compat.log
+          if-no-files-found: error
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-storage-test.yml

@@ -49,6 +49,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -96,8 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -120,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -145,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          CASE_RESULT=success
-          import re
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          import sys
+          RESULT=failure
-
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-          log_file, out_file = sys.argv[1], sys.argv[2]
+            RESULT=success
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          fi
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${CASE_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -248,11 +223,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -280,14 +254,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -303,14 +279,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-storage.log
+          if-no-files-found: error
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: always()

.github/workflows/rustfs-upgrade-test.yml

@@ -82,6 +82,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,7 +161,6 @@ jobs:
       - name: Run upgrade compatibility suite
         id: test
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
@@ -200,10 +218,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -224,103 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
+          CASE_RESULT=success
-          import re
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
-          import sys
+          RESULT=failure
-
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
+            RESULT=success
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          fi
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
-          rows = []
-          index = {}
-          topo_rows = []
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
-          PY
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
+            if [ "${RESULT}" = "success" ]; then
-            echo ""
+              cat "${MATRIX_TABLE}"
-            cat "${CASE_TABLE}" || true
+              echo ""
-            echo ""
+              cat "${CASE_TABLE}"
-            echo "## Log tail"
+              echo ""
-            echo '```text'
+              echo "## Log tail"
-            tail -n 200 "${LOG_FILE}" || true
+              echo '```text'
-            echo '```'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -347,11 +306,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -379,14 +337,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -402,14 +362,12 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
+          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/
-            /tmp/rustfs-upgrade-report.md
+          if-no-files-found: error
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
           retention-days: 3
 
       - name: Cleanup environment (after)

scripts/functional_case_report.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Preserve every functional case execution and its suite context in reports."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import re
+
+
+def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
+    ansi = re.compile(r"\x1b\[[0-9;]*m")
+    start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
+    done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
+    context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
+    topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
+    rows = []
+    pending = {}
+    topo_rows = []
+    context = "context not recorded"
+    complete = True
+    try:
+        with log_file.open(encoding="utf-8", errors="replace") as log:
+            for raw in log:
+                line = ansi.sub("", raw).strip()
+                if match := context_re.match(line):
+                    context = match[1]
+                    pending.clear()
+                elif match := topo_re.match(line):
+                    topo_rows.append(match.groups())
+                elif match := start_re.match(line):
+                    case_id, name = match.groups()
+                    pending[case_id] = len(rows)
+                    rows.append([case_id, f"{name} ({context})", "RUNNING"])
+                elif match := done_re.match(line):
+                    status, case_id = match.groups()
+                    index = pending.pop(case_id, None)
+                    if index is None:
+                        complete = False
+                        rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
+                    else:
+                        rows[index][2] = status
+    except FileNotFoundError:
+        pass
+
+    counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
+    with case_file.open("w", encoding="utf-8") as out:
+        out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
+        for status, count in counts.items():
+            out.write(f"- {status}: {count}\n")
+        out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
+        for row in rows:
+            out.write("| " + " | ".join(value.replace("|", "|") for value in row) + " |\n")
+        if not rows:
+            out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
+
+    valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
+    if matrix_file is not None:
+        with matrix_file.open("w", encoding="utf-8") as out:
+            out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
+            out.write("| --- | --- | --- | --- | --- |\n")
+            for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                result = "PASS" if nfail == "0" else "FAIL"
+                out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
+            if not topo_rows:
+                out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
+        valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
+    return valid
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("log_file", type=Path)
+    parser.add_argument("case_file", type=Path)
+    parser.add_argument("matrix_file", type=Path, nargs="?")
+    args = parser.parse_args()
+    raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)

scripts/test_security_workflow.py

@@ -6,11 +6,13 @@ from __future__ import annotations
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import unittest
 from pathlib import Path
 
 from check_test_wiring import yaml_block
+from functional_case_report import generate_report
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -38,7 +40,36 @@ def shell_body(lines: list[str]) -> str:
     return "\n".join(shell_lines)
 
 
-class SecurityWorkflowTests(unittest.TestCase):
+class WorkflowSteps:
+    def render(self, value: str) -> str:
+        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+        result = {}
+        for line in yaml_block(lines, "env", indent) or []:
+            if line.strip() and not line.lstrip().startswith("#"):
+                key, value = line.strip().split(": ", 1)
+                result[key] = self.render(value.strip("'\""))
+        return result
+
+    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+        lines = self.steps[name]
+        result = subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+        )
+        for line in lines:
+            if line.startswith("        id: "):
+                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+        if Path(self.env["GITHUB_ENV"]).exists():
+            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+                key, value = line.split("=", 1)
+                self.env[key] = value
+                self.context[f"env.{key}"] = value
+        return result
+
+
+class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -86,33 +117,6 @@ class SecurityWorkflowTests(unittest.TestCase):
             'exit "$FAKE_EXIT"\n'
         )
 
-    def render(self, value: str) -> str:
-        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
-    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
-        result = {}
-        for line in yaml_block(lines, "env", indent) or []:
-            if line.strip() and not line.lstrip().startswith("#"):
-                key, value = line.strip().split(": ", 1)
-                result[key] = self.render(value.strip("'\""))
-        return result
-
-    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
-        lines = self.steps[name]
-        result = subprocess.run(
-            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
-            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
-        )
-        for line in lines:
-            if line.startswith("        id: "):
-                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
-        if Path(self.env["GITHUB_ENV"]).exists():
-            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
-                key, value = line.split("=", 1)
-                self.env[key] = value
-                self.context[f"env.{key}"] = value
-        return result
-
     def test_workflow_wiring(self) -> None:
         names = list(self.steps)
         self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -225,7 +229,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 if suite in self.DIRECT_TESTS:
                     test = steps[self.DIRECT_TESTS[suite]]
                     self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
-                    self.assertIn("        if: always()", steps["Generate report"])
+                    self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
                 self.assertIn(condition, (
@@ -286,5 +290,307 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
 
 
+class FunctionalCaseReportTests(unittest.TestCase):
+    def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            log = root / "suite.log"
+            if text is not None:
+                log.write_text(text)
+            valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
+            return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
+
+    def test_repeated_case_executions_preserve_failure_and_context(self):
+        # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
+        log = subprocess.check_output(["bash", "-c", r'''
+log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
+log '== topology: single-single  kms-backend: local =='
+printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
+log '== topology: single-multi  kms-backend: vault-kv2 =='
+printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
+printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
+'''], text=True)
+        valid, cases, _ = self.report(log)
+        self.assertFalse(valid)
+        self.assertEqual(cases.count("| KMS-101 |"), 3)
+        self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
+        self.assertIn("roundtrip (topology: single-single  kms-backend: local) | FAIL |", cases)
+        self.assertIn("roundtrip (topology: single-multi  kms-backend: vault-kv2) | PASS |", cases)
+        self.assertNotIn("\\n", cases)
+
+    def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
+        for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
+            with self.subTest(log=text):
+                valid, cases, _ = self.report(text)
+                self.assertFalse(valid)
+                self.assertIn("## Case Summary", cases)
+        valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
+        self.assertTrue(valid)
+        self.assertIn("suite: bucket replication", cases)
+        self.assertIn("- UNSUPPORTED: 1\n", cases)
+
+    def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
+        case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
+        for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
+                                 ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
+            with self.subTest(matrix=suffix):
+                valid, _, matrix = self.report(case + suffix, matrix=True)
+                self.assertEqual(valid, expected)
+                self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
+                self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
+
+    def test_s3_case_identifiers_include_digits(self):
+        valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
+        self.assertTrue(valid)
+        self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
+
+
+class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
+    SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
+
+    def prepare(self, suite: str) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+        self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
+        self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
+        self.context.update({
+            "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
+            "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
+            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+            "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
+        })
+        self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
+        self.env = {
+            **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
+            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
+            "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
+        }
+        for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
+            self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
+        (self.directory / "scripts").mkdir()
+        (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
+        fake_bin = self.directory / "bin"
+        fake_bin.mkdir()
+        (fake_bin / "python3").symlink_to(sys.executable)
+        for command, body in (
+            ("ssh", 'printf "fixture-version\\n"\n'),
+            ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
+                   '  while [ "$#" -gt 0 ]; do\n'
+                   '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                   '    shift\n'
+                   '  done\n'
+                   'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
+        ):
+            script = fake_bin / command
+            script.write_text("#!/bin/sh\n" + body)
+            script.chmod(0o755)
+        self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
+
+    def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertNotIn("/tmp/rustfs-", self.source)
+                names = list(self.steps)
+                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+                if suite in FunctionalWorkflowTests.DIRECT_TESTS:
+                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+                for name, lines in self.steps.items():
+                    if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
+                    if any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("          path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/", lines)
+                        self.assertIn("          if-no-files-found: error", lines)
+                self.artifacts.mkdir()
+                for filename in ("report.md", "suite.log"):
+                    (self.artifacts / filename).write_text("OLD RUN EVIDENCE")
+                self.env.update(REPORT_FILE=str(self.artifacts / "report.md"), LOG_FILE=str(self.artifacts / "suite.log"))
+                initialized = self.run_step("Initialize functional evidence")
+                self.assertNotEqual(initialized.returncode, 0)
+                self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+                issue = self.run_step("File failure issue in rustfs/backlog")
+                self.assertEqual(issue.returncode, 0, issue.stderr)
+                body = Path(self.env["CAPTURE_BODY"]).read_text()
+                self.assertNotIn("OLD RUN EVIDENCE", body)
+                self.assertIn("no report or log file was produced", body)
+                self.assertEqual((self.artifacts / "report.md").read_text(), "OLD RUN EVIDENCE")
+
+    def test_reports_use_only_current_complete_suite_evidence(self):
+        for suite in self.SUITES[:-1]:
+            good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
+            partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
+            if suite == "s3-compat":
+                good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
+            if suite == "upgrade":
+                good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
+            if suite == "heal":
+                good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
+                partial = "[HEAL-STEP] 1 fixture PASS\n"
+            for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
+                                 ("success", None), ("skipped", None), ("cancelled", good)):
+                with self.subTest(suite=suite, outcome=outcome, log=log):
+                    self.prepare(suite)
+                    stale = self.directory / "old-suite.log"
+                    stale.write_text("OLD RUN EVIDENCE\n" + good)
+                    self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
+                    self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                    self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
+                    self.assertEqual(self.env["TMPDIR"], str(self.artifacts))
+                    if log is not None:
+                        Path(self.env["LOG_FILE"]).write_text(log)
+                    self.context["steps.test.outcome"] = outcome
+                    report = self.run_step("Generate report")
+                    success = outcome == "success" and log == good
+                    self.assertEqual(report.returncode == 0, success, report.stderr)
+                    contents = Path(self.env["REPORT_FILE"]).read_text()
+                    self.assertNotIn("OLD RUN EVIDENCE", contents)
+                    self.assertEqual("| PASS |" in contents, success)
+                    for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
+                                  f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
+                        self.assertIn(value, contents)
+                    self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
+                    evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
+                    if log in (good, partial):
+                        self.assertIn("| PASS |", evidence)
+                    self.assertNotIn("OLD RUN EVIDENCE", evidence)
+
+    def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                scripts = self.directory / "auto-testing"
+                scripts.mkdir()
+                filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
+                script = scripts / filename
+                script.write_text(
+                    '#!/bin/bash\nset -euo pipefail\nlog=""\n'
+                    'while [ "$#" -gt 0 ]; do\n'
+                    '  if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
+                    '  shift\n'
+                    'done\n'
+                    '[ "$log" = "$LOG_FILE" ] || exit 31\n'
+                    'printf "CURRENT SUITE LOG\\n" > "$log"\n'
+                    'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
+                    'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
+                    'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
+                    '  mkdir -p "$RUSTFS_RESULT_DIR"\n'
+                    '  printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
+                    'fi\n'
+                )
+                script.chmod(0o755)
+                name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
+                    "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
+                )
+                result = self.run_step(name)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
+                self.assertEqual(len(list(self.artifacts.glob("fixture.*/trace.log"))), 1)
+                if suite == "performance":
+                    self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
+
+    def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
+        self.prepare("heal")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        script = self.directory / "auto-testing/rustfs_heal_test.sh"
+        script.parent.mkdir()
+        # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
+        script.write_text(r'''#!/bin/bash
+set -euo pipefail
+SELECTED_STEPS=()
+PREFLIGHT=0
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
+    --log-file) LOG_FILE="$2"; shift ;;
+    --preflight) PREFLIGHT=1 ;;
+  esac
+  shift
+done
+if [ "$PREFLIGHT" -eq 1 ]; then
+  printf '\n' >> "$INVOKED_STEPS"
+  exit 0
+fi
+printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
+emit_step_result() {
+  local n="$1" desc="$2" status="$3"
+  printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
+}
+{
+  for step in "${SELECTED_STEPS[@]}"; do
+    emit_step_result "$step" "fixture step $step" PASS
+  done
+  want_all=1
+  for s in 1 2 3 4 5 6 7; do
+    [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
+  done
+  if [ "${want_all}" -eq 1 ]; then
+    printf '[HEAL-RESULT] PASS all steps passed\n'
+  fi
+} >> "$LOG_FILE"
+''')
+        script.chmod(0o755)
+        self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
+        for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
+            result = self.run_step(name)
+            self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
+        log = Path(self.env["LOG_FILE"]).read_text()
+        self.assertNotIn("[HEAL-RESULT]", log)
+        self.assertEqual(log.count("[HEAL-STEP]"), 7)
+        report = self.run_step("Generate report")
+        self.assertEqual(report.returncode, 0, report.stderr)
+        failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
+                       for step in range(1, 8)]
+        failed_logs += [
+            log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
+            log + "[HEAL-STEP] 3 later step failure FAIL\n",
+            log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
+            log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
+        ]
+        for failed_log in failed_logs:
+            with self.subTest(log=failed_log):
+                Path(self.env["LOG_FILE"]).write_text(failed_log)
+                report = self.run_step("Generate report")
+                self.assertNotEqual(report.returncode, 0, report.stderr)
+                contents = Path(self.env["REPORT_FILE"]).read_text()
+                self.assertIn("Test Step Outcome: failure", contents)
+                self.assertNotIn("| PASS |", contents)
+                if "original failure" in failed_log:
+                    self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
+                if "later step failure" in failed_log:
+                    self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
+
+    def test_performance_results_version_and_report_are_bound_to_the_run(self):
+        self.prepare("performance")
+        initialized = self.run_step("Initialize functional evidence")
+        self.assertEqual(initialized.returncode, 0, initialized.stderr)
+        self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
+        self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
+        version = self.run_step("Collect RustFS version info")
+        self.assertEqual(version.returncode, 0, version.stderr)
+        self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
+        old_summary = self.directory / "old-results/summary.md"
+        old_summary.parent.mkdir()
+        old_summary.write_text("OLD RUN EVIDENCE")
+        upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
+        self.assertNotEqual(self.run_step(upload).returncode, 0)
+        self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
+        results = Path(self.env["RUSTFS_RESULT_DIR"])
+        results.mkdir()
+        (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
+        report = self.run_step(upload)
+        self.assertEqual(report.returncode, 0, report.stderr)
+        contents = Path(self.env["REPORT_FILE"]).read_text()
+        for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
+                      "CURRENT PERFORMANCE RESULTS", "fixture-version"):
+            self.assertIn(value, contents)
+        self.assertNotIn("OLD RUN EVIDENCE", contents)
+
+
 if __name__ == "__main__":
     unittest.main()