From f77efe5f2accbd79fd128b38089e7cecd6e84ab5 Mon Sep 17 00:00:00 2001
From: hector <42570491+majinghe@users.noreply.github.com>
Date: Tue, 15 Sep 2026 18:20:05 +0800
Subject: [PATCH] fix(ci): retain failed functional chain reports and check
 runners (#7906)

---
 .github/workflows/functional-chain-health.yml |  3 +-
 .github/workflows/rustfs-functional-chain.yml | 28 +++++-
 .github/workflows/rustfs-heal-test.yml        |  6 +-
 .github/workflows/rustfs-kms-test.yml         |  6 +-
 .github/workflows/rustfs-performance-test.yml | 20 +++-
 .github/workflows/rustfs-pool-expand-test.yml | 23 +++--
 .github/workflows/rustfs-replication-test.yml |  6 +-
 .github/workflows/rustfs-s3-compat-test.yml   |  6 +-
 .github/workflows/rustfs-security-test.yml    |  6 +-
 .github/workflows/rustfs-storage-test.yml     |  6 +-
 .github/workflows/rustfs-table-test.yml       |  2 +
 .github/workflows/rustfs-tier-test.yml        |  6 +-
 .github/workflows/rustfs-upgrade-test.yml     |  6 +-
 scripts/check_functional_runners.py           | 27 ++++++
 scripts/functional_chain_evidence.py          | 66 ++++++++++++-
 scripts/functional_chain_health.py            | 11 ++-
 scripts/test_functional_chain.py              | 94 +++++++++++++++++++
 scripts/test_functional_chain_health.py       |  9 ++
 scripts/test_security_workflow.py             | 18 ++--
 19 files changed, 303 insertions(+), 46 deletions(-)
 create mode 100644 scripts/check_functional_runners.py

diff --git a/.github/workflows/functional-chain-health.yml b/.github/workflows/functional-chain-health.yml
index e27f12813..973327090 100644
--- a/.github/workflows/functional-chain-health.yml
+++ b/.github/workflows/functional-chain-health.yml
@@ -20,7 +20,8 @@ jobs:
       - name: Collect and publish verified chain health
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: python3 scripts/functional_chain_health.py --publish --output "${RUNNER_TEMP}/chain-health.json"
+          NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
+        run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
       - name: Retain health observation
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml
index 7e19c0cca..5a300b463 100644
--- a/.github/workflows/rustfs-functional-chain.yml
+++ b/.github/workflows/rustfs-functional-chain.yml
@@ -65,6 +65,11 @@ jobs:
           path: ${{ runner.temp }}/chain-candidate.json
           if-no-files-found: error
 
+      - name: Check shared functional runners before scheduling suites
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
+
   upgrade:
     needs: [prepare]
     if: ${{ always() && needs.prepare.result == 'success' }}
@@ -147,7 +152,7 @@ jobs:
 
   complete-chain:
     needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, performance]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result != 'skipped' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -155,12 +160,33 @@ jobs:
         with:
           persist-credentials: false
       - name: Download suite evidence
+        id: download
+        continue-on-error: true
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
           pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-evidence
           merge-multiple: true
+      - name: Summarize every suite including failures and missing evidence
+        if: always()
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py summarize
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-report.json"
+      - name: Retain chain report regardless of test verdict
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/chain-report.json
+            ${{ runner.temp }}/chain-report.md
+          if-no-files-found: error
       - name: Verify every required suite
+        if: always()
         env:
           CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
           CHAIN_NEEDS: ${{ toJSON(needs) }}
diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index c1629296c..d4ec9f05c 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -303,10 +303,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -449,7 +449,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index afc278f3d..2b62fcdad 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -249,10 +249,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -393,7 +393,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index 7cd6795f0..9acdaa6d9 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -85,7 +85,21 @@ env:
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
+  runner-check:
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check performance runner before entering its queue
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/check_functional_runners.py pf-testing
+
   performance-test:
+    needs: runner-check
     runs-on: pf-testing
     timeout-minutes: 900
     env:
@@ -260,10 +274,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -348,7 +362,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml
index c356f691f..115027b07 100644
--- a/.github/workflows/rustfs-pool-expand-test.yml
+++ b/.github/workflows/rustfs-pool-expand-test.yml
@@ -69,7 +69,7 @@ env:
   RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
   RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   # Package used by the nightly run (workflow_dispatch inputs are empty for
@@ -89,7 +89,7 @@ jobs:
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
+      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -128,6 +128,17 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Validate pool topology before destructive cleanup
+        id: topology
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
+          if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
+            echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
+            exit 1
+          fi
+
       - name: Cleanup environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
@@ -581,10 +592,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -643,7 +654,7 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -731,7 +742,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 1763d83af..cb2adad2a 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -257,10 +257,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -398,7 +398,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index f1bfd75b9..0430cb559 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -226,10 +226,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -370,7 +370,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index 39f9ce78e..44dac29b4 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -259,10 +259,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -372,7 +372,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index dc46b27ed..31e935183 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -241,10 +241,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -385,7 +385,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-table-test.yml b/.github/workflows/rustfs-table-test.yml
index 108a2d903..cdbbe529b 100644
--- a/.github/workflows/rustfs-table-test.yml
+++ b/.github/workflows/rustfs-table-test.yml
@@ -225,6 +225,8 @@ jobs:
           python3 auto-testing/scripts/issue_manager.py handle \
             --repo rustfs/backlog \
             --suite table --category table --suite-label "S3 Tables" \
+            --outcome '${{ steps.test.outcome }}' \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
             --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
             --run-id "${GITHUB_RUN_ID}" \
             --attempt "${GITHUB_RUN_ATTEMPT}" \
diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml
index 5316909a9..e43d67c56 100644
--- a/.github/workflows/rustfs-tier-test.yml
+++ b/.github/workflows/rustfs-tier-test.yml
@@ -523,10 +523,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -642,7 +642,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 95cfbf610..b94e3e706 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -324,10 +324,10 @@ jobs:
             echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
           EXISTING="$(gh issue list -R rustfs/backlog --state all \
             --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
+            --json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
           if [ -n "${EXISTING}" ]; then
             echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
             exit 0
@@ -472,7 +472,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
 
       - name: Upload chain evidence
-        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/scripts/check_functional_runners.py b/scripts/check_functional_runners.py
new file mode 100644
index 000000000..f312b4f78
--- /dev/null
+++ b/scripts/check_functional_runners.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Fail before dispatching destructive suites to an offline shared runner."""
+import sys
+
+from resolve_functional_candidate import REPOSITORY, api, require
+
+
+def check(labels):
+    runners = []
+    for page in range(1, 11):
+        batch = api(f"repos/{REPOSITORY}/actions/runners?per_page=100&page={page}")["runners"]
+        runners.extend(batch)
+        if len(batch) < 100:
+            break
+    else:
+        raise ValueError("runner inventory exceeds the inspection limit")
+    missing = [label for label in labels if not any(
+        runner.get("status") == "online" and label in {item["name"] for item in runner.get("labels", [])}
+        for runner in runners)]
+    require(not missing, "No online runner for: " + ", ".join(missing)
+            + ". Restore the runner service before retrying; no tests were dispatched by this check.")
+    print("Online functional runner labels: " + ", ".join(labels))
+
+
+if __name__ == "__main__":
+    require(len(sys.argv) > 1, "at least one runner label is required")
+    check(sys.argv[1:])
\ No newline at end of file
diff --git a/scripts/functional_chain_evidence.py b/scripts/functional_chain_evidence.py
index 21145a69d..e768ad91e 100644
--- a/scripts/functional_chain_evidence.py
+++ b/scripts/functional_chain_evidence.py
@@ -101,12 +101,67 @@ def record(chain, suite, report, output):
         result["valid"] = True
     except (OSError, ValueError, subprocess.SubprocessError) as exc:
         error = exc
+        result["error"] = str(exc)
     output.parent.mkdir(parents=True, exist_ok=False)
     output.write_text(json.dumps(result, sort_keys=True) + "\n")
+    # A failed validation may still produce fresh diagnostics. A failed write
+    # or directory collision must never authorize uploading a leftover file.
+    with open(os.environ["GITHUB_OUTPUT"], "a") as step_output:
+        step_output.write("written=true\n")
     if error:
         raise error
 
 
+def summarize(chain, directory, needs):
+    """Retain failed/missing lanes without granting complete-success evidence."""
+    lanes = []
+    for suite in SUITES:
+        lane = {"suite": suite, "result": needs.get(suite, {}).get("result", "missing"),
+                "evidence": None, "error": None}
+        try:
+            record = json.loads((directory / (suite + ".json")).read_text())
+            require(chain is not None and record.get("chain") == chain and record.get("suite") == suite,
+                    "suite evidence identity mismatch")
+            lane["evidence"] = record
+        except (OSError, ValueError, AttributeError) as error:
+            lane["error"] = str(error)
+        lanes.append(lane)
+    result = {"schema": 1, "chain": chain, "needs": needs, "lanes": lanes,
+              "complete": False, "error": None, "completed_at": datetime.now(timezone.utc).isoformat()}
+    try:
+        require(chain is not None, "candidate preparation did not complete")
+        aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"})
+        result["complete"] = True
+    except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
+        result["error"] = str(error)
+    return result
+
+
+def render_summary(result):
+    lines = ["# RustFS functional chain report", "",
+             "- All required suites passed with verified evidence: " + str(result["complete"]).lower(),
+             "- Preparation: " + result["needs"].get("prepare", {}).get("result", "missing")]
+    if result["chain"]:
+        chain = result["chain"]
+        manifest = chain["candidate"]["manifest"]
+        lines += [f"- Chain run / attempt: {chain['run_id']} / {chain['attempt']}",
+                  f"- Build run / attempt: {manifest['build_run_id']} / {manifest['build_run_attempt']}",
+                  f"- Source: {manifest.get('source_ref', 'main')} @ {manifest['source_sha']}",
+                  f"- Package SHA256: {manifest['package_sha256']}", f"- Test scripts: {chain['testing_sha']}"]
+    lines += ["", "| Suite | Job result | Evidence | PASS | FAIL | SKIP | UNSUPPORTED | RUNNING |",
+              "| --- | --- | --- | --- | --- | --- | --- | --- |"]
+    for lane in result["lanes"]:
+        record = lane["evidence"] or {}
+        counts = record.get("counts", {})
+        state = "valid" if record.get("valid") is True else ("invalid" if record else "missing")
+        values = [lane["suite"], lane["result"], state] + [str(counts.get(key, "—")) for key in
+                  ("PASS", "FAIL", "SKIP", "UNSUPPORTED", "RUNNING")]
+        lines.append("| " + " | ".join(values) + " |")
+    lines += ["", "Missing, skipped, cancelled or invalid evidence is NOT a passing test or proof of a fix.",
+              "See the suite artifacts for case results and diagnostics; this report does not replace the complete-success gate."]
+    return "\n".join(lines) + "\n"
+
+
 def aggregate(chain, directory, needs):
     require(set(needs) == set(SUITES), "aggregate is missing a required lane")
     require(all(value.get("result") == "success" for value in needs.values()), "a required suite did not succeed")
@@ -129,12 +184,21 @@ def validate_records(chain, records):
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("mode", choices=("consume", "record", "aggregate"))
+    parser.add_argument("mode", choices=("consume", "record", "aggregate", "summarize"))
     parser.add_argument("--suite", choices=SUITES)
     parser.add_argument("--report", type=Path)
     parser.add_argument("--output", type=Path)
     parser.add_argument("--directory", type=Path)
     args = parser.parse_args()
+    if args.mode == "summarize":
+        chain = current_chain() if os.environ.get("CHAIN_MANIFEST") else None
+        result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]))
+        args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
+        args.output.with_suffix(".md").write_text(render_summary(result))
+        if os.environ.get("GITHUB_STEP_SUMMARY"):
+            with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
+                summary.write(render_summary(result))
+        return
     chain = current_chain()
     if args.mode == "consume":
         consume(chain)
diff --git a/scripts/functional_chain_health.py b/scripts/functional_chain_health.py
index a52509c76..d32deaf1b 100644
--- a/scripts/functional_chain_health.py
+++ b/scripts/functional_chain_health.py
@@ -63,13 +63,15 @@ def complete_success(run):
     return result
 
 
-def collect(limit=20):
+def collect(limit=20, source_ref="main"):
+    source_ref = source_ref.removeprefix("refs/heads/")
+    require(bool(source_ref), "expected nightly source ref is empty")
     observed = datetime.now(timezone.utc)
     workflow = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}")
     runs = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
     runs.sort(key=lambda run: timestamp(run["run_started_at"]), reverse=True)
     result = {"schema": 1, "observed_at": observed.isoformat(), "workflow_state": workflow["state"],
-              "owner": "@overtrue", "scan_limit": limit, "inspection_complete": True,
+              "owner": "@overtrue", "scan_limit": limit, "inspection_complete": True, "expected_source_ref": source_ref,
               "latest_attempt": None, "last_complete_success": {}, "healthy": False}
     for index, listed in enumerate(runs):
         run = api(f"repos/{REPOSITORY}/actions/runs/{listed['id']}/attempts/{listed['run_attempt']}")
@@ -104,7 +106,7 @@ def collect(limit=20):
         complete["fresh"] = observed <= timestamp(complete["expires_at"])
     latest = result["latest_attempt"] or {}
     result["healthy"] = (result["workflow_state"] == "active" and latest.get("verification") == "complete"
-                         and latest.get("source_ref") == "main" and result["last_complete_success"].get("main", {}).get("fresh") is True)
+                         and latest.get("source_ref") == source_ref and result["last_complete_success"].get(source_ref, {}).get("fresh") is True)
     return result
 
 
@@ -134,8 +136,9 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--publish", action="store_true")
+    parser.add_argument("--source-ref", default="main", help="Expected nightly build source, not the workflow branch")
     args = parser.parse_args()
-    result = collect()
+    result = collect(source_ref=args.source_ref)
     if args.publish:
         publish(result)
     args.output.write_text(json.dumps(result, indent=2) + "\n")
diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py
index f60b44aec..23c6524fd 100644
--- a/scripts/test_functional_chain.py
+++ b/scripts/test_functional_chain.py
@@ -94,6 +94,35 @@ class EvidenceTests(unittest.TestCase):
         self.assertEqual(result["chain"], self.chain)
         self.assertEqual(len(result["suites"]), 10)
 
+    def test_failure_report_preserves_counts_without_weakening_success_gate(self):
+        path = self.directory / "s3.json"
+        record = json.loads(path.read_text())
+        record.update(valid=False, counts={**record["counts"], "FAIL": 2}, error="failed cases")
+        path.write_text(json.dumps(record))
+        needs = {**self.needs, "s3": {"result": "failure"}, "performance": {"result": "cancelled"}}
+        (self.directory / "performance.json").unlink()
+        result = evidence.summarize(self.chain, self.directory, needs)
+        self.assertFalse(result["complete"])
+        self.assertEqual(result["lanes"][1]["evidence"]["counts"]["FAIL"], 2)
+        self.assertEqual(result["lanes"][-1]["result"], "cancelled")
+        self.assertIsNone(result["lanes"][-1]["evidence"])
+        with self.assertRaises(ValueError):
+            evidence.aggregate(self.chain, self.directory, needs)
+
+    def test_preparation_failure_still_has_all_ten_lanes(self):
+        result = evidence.summarize(None, self.directory / "absent", {"prepare": {"result": "failure"}})
+        self.assertFalse(result["complete"])
+        self.assertEqual(len(result["lanes"]), 10)
+        self.assertIn("| performance | missing | missing |", evidence.render_summary(result))
+        self.assertIn("Preparation: failure", evidence.render_summary(result))
+
+    def test_malformed_or_cross_attempt_record_is_not_reused(self):
+        for payload in ("broken json", "[]", json.dumps({"suite": "s3", "chain": {"attempt": 9}})):
+            (self.directory / "s3.json").write_text(payload)
+            result = evidence.summarize(self.chain, self.directory, self.needs)
+            self.assertFalse(result["complete"])
+            self.assertIsNone(result["lanes"][1]["evidence"])
+
     def test_missing_failed_cancelled_or_skipped_lane_never_passes(self):
         for state in ("failure", "cancelled", "skipped", "pending"):
             with self.subTest(state=state), self.assertRaises(ValueError):
@@ -172,14 +201,32 @@ class EnvelopeTests(unittest.TestCase):
         report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n")
         for index, (key, status) in enumerate((("CHAIN_TEST_OUTCOME", "failure"), ("CHAIN_REPORT_OUTCOME", "failure"), ("CHAIN_JOB_STATUS", "cancelled"))):
             output = self.root / str(index) / "kms.json"
+            Path(self.env["GITHUB_OUTPUT"]).unlink(missing_ok=True)
             with mock.patch.dict(evidence.os.environ, {**self.env, key: status}), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40), self.assertRaises(ValueError):
                 evidence.record(self.chain, "kms", report, output)
             self.assertFalse(json.loads(output.read_text())["valid"])
+            self.assertIn("error", json.loads(output.read_text()))
+            self.assertEqual(Path(self.env["GITHUB_OUTPUT"]).read_text(), "written=true\n")
         output = self.root / "success" / "kms.json"
         with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40):
             evidence.record(self.chain, "kms", report, output)
         self.assertTrue(json.loads(output.read_text())["valid"])
 
+    def test_collision_or_failed_write_never_authorizes_evidence_upload(self):
+        report = self.root / "cases.md"
+        report.write_text("| Case | Status |\n| --- | --- |\n| KMS-1 | PASS |\n")
+        output = self.root / "stale" / "kms.json"
+        output.parent.mkdir()
+        output.write_text("OLD RUN EVIDENCE")
+        with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40):
+            with self.assertRaises(FileExistsError):
+                evidence.record(self.chain, "kms", report, output)
+            self.assertEqual(output.read_text(), "OLD RUN EVIDENCE")
+            self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists())
+            with mock.patch.object(Path, "write_text", side_effect=OSError("disk full")), self.assertRaises(OSError):
+                evidence.record(self.chain, "kms", report, self.root / "new" / "kms.json")
+            self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists())
+
     def test_unknown_status_cannot_hide_among_passing_cases(self):
         text = "| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n| KMS-2 | fixture | NOT RUN |\n"
         with self.assertRaises(ValueError):
@@ -198,6 +245,53 @@ class EnvelopeTests(unittest.TestCase):
         complete = "\n".join(yaml_block(lines, "complete-chain", 2))
         self.assertIn("needs: [prepare, " + ", ".join(evidence.SUITES) + "]", complete)
         self.assertIn("functional_chain_evidence.py aggregate", complete)
+        self.assertIn("functional_chain_evidence.py summarize", complete)
+        self.assertIn("functional-chain-report-", complete)
+        self.assertIn("needs.prepare.result != 'skipped'", complete)
+
+    def test_every_lane_retains_failed_evidence_and_deduplicates_its_own_attempt(self):
+        paths = list((candidate.ROOT / ".github/workflows").glob("rustfs-*-test.yml"))
+        lanes = [path for path in paths if "name: Upload chain evidence" in path.read_text()]
+        self.assertEqual(len(lanes), 10)
+        for path in lanes:
+            with self.subTest(path=path.name):
+                text = path.read_text()
+                self.assertIn("if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", text)
+                self.assertIn("attempt ${GITHUB_RUN_ATTEMPT})", text)
+                self.assertIn('select(.title == \\"${TITLE}\\")', text)
+
+    def test_table_issue_manager_receives_outcome_and_case_evidence(self):
+        text = (candidate.ROOT / ".github/workflows/rustfs-table-test.yml").read_text()
+        self.assertIn("--outcome '${{ steps.test.outcome }}'", text)
+        self.assertIn('--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"', text)
+
+    def test_pool_topology_is_checked_before_cleanup_with_four_node_defaults(self):
+        text = (candidate.ROOT / ".github/workflows/rustfs-pool-expand-test.yml").read_text()
+        self.assertIn("http://rustfs-node4:9000", text)
+        self.assertIn("vars.RUSTFS_NODES || 'vm000 vm001 vm002'", text)
+        self.assertLess(text.index("name: Validate pool topology"), text.index("name: Cleanup environment (before)"))
+        self.assertIn("steps.topology.outcome == 'success' && inputs.cleanup_after", text)
+
+
+class RunnerTests(unittest.TestCase):
+    def test_offline_or_missing_runner_fails_before_dispatch(self):
+        import check_functional_runners as runners
+        for inventory in ([], [{"status": "offline", "labels": [{"name": "pf-testing"}]}]):
+            with mock.patch.object(runners, "api", return_value={"runners": inventory}), self.assertRaisesRegex(ValueError, "pf-testing"):
+                runners.check(["pf-testing"])
+
+    def test_busy_online_runner_is_available_and_inventory_is_paginated(self):
+        import check_functional_runners as runners
+        pages = [{"runners": [{"status": "online", "labels": []}] * 100},
+                 {"runners": [{"status": "online", "busy": True, "labels": [{"name": "pf-testing"}]}]}]
+        with mock.patch.object(runners, "api", side_effect=pages) as api:
+            runners.check(["pf-testing"])
+        self.assertIn("page=2", api.call_args.args[0])
+
+    def test_unavailable_inventory_does_not_assume_online(self):
+        import check_functional_runners as runners
+        with mock.patch.object(runners, "api", side_effect=OSError("forbidden")), self.assertRaises(OSError):
+            runners.check(["pf-testing"])
 
 
 if __name__ == "__main__":
diff --git a/scripts/test_functional_chain_health.py b/scripts/test_functional_chain_health.py
index bec146cae..225bb99b4 100644
--- a/scripts/test_functional_chain_health.py
+++ b/scripts/test_functional_chain_health.py
@@ -85,6 +85,15 @@ class HealthTests(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, "changed during inspection"):
                 health.collect()
 
+    def test_health_follows_configured_build_source_not_workflow_main(self):
+        run = {**self.run, "status": "completed", "conclusion": "success"}
+        complete = self.validate()
+        for expected, healthy in (("release", True), ("refs/heads/release", True), ("main", False)):
+            responses = [{"state": "active"}, {"workflow_runs": [run]}, run, {"workflow_runs": [run]}]
+            with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
+                result = health.collect(source_ref=expected)
+            self.assertEqual(result["healthy"], healthy)
+
     def test_publication_uses_the_read_blob_sha(self):
         current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
         existing = {"sha": "reviewed-blob", "content": base64.b64encode(json.dumps(current).encode()).decode()}
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index a7cf1f991..4ab6f50ea 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -398,11 +398,17 @@ class FunctionalWorkflowTests(unittest.TestCase):
                     self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
-                self.assertIn(condition, (
-                    "if: always()",
-                    "if: ${{ always() && inputs.cleanup_after != 'false' }}",
-                    "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
-                ))
+                if suite == "pool-expand":
+                    self.assertEqual(condition, "if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}")
+                    self.assertIn("        id: topology", steps["Validate pool topology before destructive cleanup"])
+                    self.assertLess(list(steps).index("Validate pool topology before destructive cleanup"),
+                                    list(steps).index("Cleanup environment (before)"))
+                else:
+                    self.assertIn(condition, (
+                        "if: always()",
+                        "if: ${{ always() && inputs.cleanup_after != 'false' }}",
+                        "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
+                    ))
                 if suite != "performance":
                     handoff = next(
                         value for name, value in steps.items() if name.startswith("Continue functional chain")
@@ -567,7 +573,7 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
                     self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts"))
                 for name, lines in self.steps.items():
                     if name == "Upload chain evidence":
-                        self.assertIn("        if: ${{ always() && steps.chain_record.outcome == 'success' }}", lines)
+                        self.assertIn("        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", lines)
                         self.assertIn("        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}", self.steps["Record chain evidence"])
                         self.assertIn("          if-no-files-found: error", lines)
                         continue