From cc5060ac20a3087394b623fbc45cac21c43471e6 Mon Sep 17 00:00:00 2001
From: hector <42570491+majinghe@users.noreply.github.com>
Date: Sat, 5 Sep 2026 23:06:28 +0800
Subject: [PATCH 1/5] ci(functional): fix dashboard report upload argv overflow
 and security checkout clobbering (#7212)

Two fixes for the functional test chain:

1. Report upload fails with 'jq: Argument list too long' when the base64
   report is passed through '--arg content' (pool reports exceed the OS
   argv limit; last night's pool run lost its Step Results report this
   way). Write the base64 payload to a temp file and load it in jq via
   --rawfile instead. Applied uniformly to all nine suite workflows
   that share this upload step.

2. The security workflow cloned rustfs/auto-testing into the workspace
   and then ran actions/checkout at the workspace root for the OIDC
   live gate script, which wiped the auto-testing clone and killed the
   suite with 'chmod: cannot access auto-testing/rustfs-security-test.sh'.
   Check out the repository into the rustfs-repo/ subdirectory instead
   and point RUSTFS_SECURITY_OIDC_LIVE_SCRIPT there.

Co-authored-by: rustfs-ci <ci@rustfs.com>
---
 .github/workflows/rustfs-heal-test.yml        | 15 ++++++++-----
 .github/workflows/rustfs-kms-test.yml         | 15 ++++++++-----
 .github/workflows/rustfs-pool-expand-test.yml | 15 ++++++++-----
 .github/workflows/rustfs-replication-test.yml | 15 ++++++++-----
 .github/workflows/rustfs-s3-compat-test.yml   | 15 ++++++++-----
 .github/workflows/rustfs-security-test.yml    | 21 +++++++++++++------
 .github/workflows/rustfs-storage-test.yml     | 15 ++++++++-----
 .github/workflows/rustfs-tier-test.yml        | 15 ++++++++-----
 .github/workflows/rustfs-upgrade-test.yml     | 15 ++++++++-----
 9 files changed, 95 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index 666c83b4c..efbe2a09c 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -235,17 +235,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index 8647c9268..5c3a2b6b1 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -236,17 +236,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml
index 3c9836b2d..c700df7c8 100644
--- a/.github/workflows/rustfs-pool-expand-test.yml
+++ b/.github/workflows/rustfs-pool-expand-test.yml
@@ -539,17 +539,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 74c57b6d5..839d1de7f 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -245,17 +245,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index 80856194c..875db70bb 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -216,17 +216,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index 16e10a80c..ee37e7d5d 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -77,10 +77,14 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      # Checkout the repository into its own subdirectory. Checking out at
+      # the workspace root would wipe the auto-testing clone above (that is
+      # exactly how run 33934141181 lost rustfs-security-test.sh).
       - name: Checkout repository (for the OIDC live gate script)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          path: rustfs-repo
 
       - name: Initialize security evidence
         id: evidence
@@ -145,7 +149,7 @@ jobs:
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
           TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-security-test.sh
@@ -219,17 +223,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index 767f734dc..1e99dce4e 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -231,17 +231,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml
index 5d9a2c1d7..abe09f101 100644
--- a/.github/workflows/rustfs-tier-test.yml
+++ b/.github/workflows/rustfs-tier-test.yml
@@ -377,17 +377,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: Verify required tier evidence
         id: evidence_verify
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 0b4c19af1..612765874 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -330,17 +330,22 @@ jobs:
           fi
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          # Base64-encode the report into a temp file and feed it to jq via
+          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
+          rm -f "${B64_FILE}"
 
       - name: File failure issue in rustfs/backlog
         if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}

From f54323b06221519e07242dfde494c93e98a02afd Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Sat, 5 Sep 2026 23:50:43 +0800
Subject: [PATCH 2/5] chore(deps): preserve scanner and heal validation
 compatibility (#7209)

* chore(deps): refresh scanner heal batch dependency baseline

Regenerate compatible lockfile selections before the next implementation
batch. Cargo upgrade leaves direct requirements unchanged.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(ecstore): remove duplicate local rename implementation

Keep the canonical commit module after concurrent storage changes merged.
The control-write and rollback changes are already present there.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* chore(deps): refresh profiling dependencies for the next batch

Update hotpath and its macro crate to the compatible patch release before
the next dependency-ready implementation tasks.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(deps): preserve supported hotpath focus expressions

Keep the profiler runtime before its regex-lite compatibility regression.
Track the opt-in validation required to remove this constraint in backlog.

Refs rustfs/backlog#2302.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(rustfs): complete list-through source config

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
---
 Cargo.lock                            | 51 ++++++++++++++++-----------
 Cargo.toml                            |  3 +-
 rustfs/src/app/bucket_list_through.rs |  2 ++
 3 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dc72ece63..76072f5da 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1679,7 +1679,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -1698,7 +1698,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -2110,7 +2110,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "inout 0.1.4",
 ]
 
@@ -2580,7 +2580,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rand_core 0.6.4",
  "subtle",
  "zeroize",
@@ -2605,11 +2605,11 @@ dependencies = [
 
 [[package]]
 name = "crypto-common"
-version = "0.1.7"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "typenum",
 ]
 
@@ -3901,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer 0.10.4",
  "const-oid 0.9.6",
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "subtle",
 ]
 
@@ -4166,7 +4166,7 @@ dependencies = [
  "crypto-bigint 0.5.5",
  "digest 0.10.7",
  "ff 0.13.1",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "group 0.13.0",
  "hkdf 0.12.4",
  "pem-rfc7468 0.7.0",
@@ -4499,7 +4499,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
 dependencies = [
  "io-lifetimes 2.0.4",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4622,9 +4622,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.7"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
 dependencies = [
  "typenum",
  "version_check",
@@ -4637,7 +4637,7 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rustversion",
  "typenum",
 ]
@@ -5319,9 +5319,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros"
-version = "0.25.0"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
+checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5660,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
  "block-padding 0.3.3",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
 dependencies = [
  "io-lifetimes 3.0.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -7115,7 +7115,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.1",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -11400,7 +11400,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
  "base16ct 0.2.0",
  "der 0.7.10",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "pkcs8 0.10.2",
  "subtle",
  "zeroize",
@@ -12400,7 +12400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -13652,6 +13652,15 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
@@ -13824,7 +13833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
 dependencies = [
  "bitflags 2.13.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 871d8cb71..8d37bfc52 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -371,7 +371,8 @@ dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
-hotpath = { version = "0.25.0", default-features = false }
+# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
+hotpath = { version = "=0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 
diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index b8aa0d38e..a41b994f9 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -712,6 +712,8 @@ mod tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: filter_prefix.map(str::to_string),

From d5426f59ec9a1c639854f8cae4ee8b888c31535f Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:14:22 +0800
Subject: [PATCH 3/5] fix(ci): isolate functional evidence and preserve every
 result (#7201)

* fix(ci): preserve reported functional suite failures

* fix(ci): isolate functional evidence and preserve every result

* fix(ci): exclude sensitive scratch files from suite artifacts
---
 .github/workflows/rustfs-heal-test.yml        |  89 ++-
 .github/workflows/rustfs-kms-test.yml         | 128 ++--
 .github/workflows/rustfs-performance-test.yml |  68 ++-
 .github/workflows/rustfs-replication-test.yml | 128 ++--
 .github/workflows/rustfs-s3-compat-test.yml   | 131 ++--
 .github/workflows/rustfs-security-test.yml    |  11 +-
 .github/workflows/rustfs-storage-test.yml     | 131 ++--
 .github/workflows/rustfs-upgrade-test.yml     | 154 ++---
 scripts/functional_case_report.py             |  77 +++
 scripts/test/oidc_keycloak_live.sh            |   4 +-
 scripts/test_security_workflow.py             | 565 ++++++++++++++++--
 11 files changed, 968 insertions(+), 518 deletions(-)
 create mode 100644 scripts/functional_case_report.py

diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index efbe2a09c..4e6b4e190 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -59,6 +59,21 @@ jobs:
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -114,7 +129,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -124,7 +139,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
@@ -134,13 +149,10 @@ jobs:
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -149,8 +161,9 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          CASE_RESULT=success
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
           import re
           import sys
 
@@ -162,6 +175,7 @@ jobs:
 
           steps = {}
           order = []
+          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
           version = None
           version_node = None
           verdict = None
@@ -175,14 +189,15 @@ jobs:
                           n, desc, status = m.group(1), m.group(2), m.group(3)
                           if n not in steps:
                               order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
+                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+                              steps[n] = (desc, status)
                           continue
                       m = ver_re.match(line)
                       if m:
                           version, version_node = m.group(1), m.group(2)
                           continue
                       m = result_re.match(line)
-                      if m:
+                      if m and verdict != 'FAIL':
                           verdict, verdict_detail = m.group(1), m.group(2)
           except FileNotFoundError:
               pass
@@ -202,30 +217,43 @@ jobs:
                   out.write(f'| {n} | {desc} | {status} |\n')
               if not order:
                   out.write('| - | - | NOT RUN (no step result lines found) |\n')
+          complete = set(steps) == {str(n) for n in range(1, 8)}
+          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
           PY
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${STEPS_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
           SUITE: heal
         run: |
           set -euo pipefail
@@ -257,11 +285,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -289,14 +316,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -312,14 +341,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-heal-test-${{ github.run_id }}
+          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-heal-test*.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index 5c3a2b6b1..c9eb02d00 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -52,6 +52,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -108,8 +127,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -139,10 +156,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-kms.log
-          REPORT_FILE: /tmp/rustfs-kms-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -154,79 +168,43 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
           SUITE: kms
         run: |
           set -euo pipefail
@@ -258,11 +236,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -290,14 +267,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -313,14 +292,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-kms-test-${{ github.run_id }}
+          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-kms.log
-            /tmp/rustfs-kms-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index ff3e2978d..148e5ccf3 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -76,8 +76,6 @@ env:
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  # Fixed benchmark result directory so later steps can read summary.md
-  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
   # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
@@ -89,6 +87,22 @@ jobs:
     # Skipped when nightly failed.
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -120,7 +134,7 @@ jobs:
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
           chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Install RustFS package & start cluster (4x4)
         run: |
@@ -130,7 +144,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Preflight checks
         run: |
@@ -140,7 +154,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run benchmark (GET/PUT/MIXED)
         id: benchmark
@@ -153,17 +167,15 @@ jobs:
             --step 5 -y \
             --warp-duration "${{ inputs.warp_duration || '5m' }}" \
             --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file /tmp/rustfs-perf-test.log
+            --log-file "${LOG_FILE}"
 
       - name: Analyze results
         if: ${{ steps.benchmark.conclusion == 'success' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Collect RustFS version info
         if: ${{ steps.benchmark.conclusion == 'success' }}
-        env:
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -183,7 +195,6 @@ jobs:
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
-          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -191,7 +202,7 @@ jobs:
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
-          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
           REPORT_PATH="reports/${DATE}.md"
           {
@@ -199,6 +210,8 @@ jobs:
             echo ""
             echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+            echo "- **Workflow Commit**: ${GITHUB_SHA}"
             echo "- **Trigger**: ${{ github.event_name }}"
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
@@ -208,8 +221,8 @@ jobs:
             echo '```text'
             cat "${VERSION_FILE}"
             echo '```'
-          } > /tmp/rustfs-perf-report.md
-          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          } > "${REPORT_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
           if [ -n "${SHA}" ]; then
             jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -228,11 +241,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -260,14 +272,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -283,20 +297,26 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs & results
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-perf-test-${{ github.run_id }}
+          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-perf-test*.log
-            /tmp/rustfs-perf-results/**
-            /tmp/rustfs-version.txt
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+          if-no-files-found: error
 
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
       - name: Notify on failure
         if: failure()
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 839d1de7f..ae8f9dc50 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -65,6 +65,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -113,8 +132,6 @@ jobs:
 
       - name: Run replication suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -137,10 +154,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -162,80 +176,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
           SUITE: replication
         run: |
           set -euo pipefail
@@ -267,11 +245,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -299,14 +276,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -322,14 +301,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-replication-${{ github.run_id }}
+          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index 875db70bb..99828537b 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -40,6 +40,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -87,8 +106,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -105,10 +122,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-s3-compat.log
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -130,83 +144,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
           SUITE: s3
         run: |
           set -euo pipefail
@@ -238,11 +213,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -270,14 +244,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -293,14 +269,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-s3-compat-${{ github.run_id }}
+          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-s3-compat.log
-            /tmp/rustfs-s3-compat-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index ee37e7d5d..02fe7d662 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -92,7 +92,7 @@ jobs:
           set -euo pipefail
           umask 077
           SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${SECURITY_ARTIFACTS_DIR}"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
       # auto-testing is private: clone it with the dedicated PF token (not
@@ -148,7 +148,7 @@ jobs:
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
-          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
           RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
         run: |
           set -euo pipefail
@@ -172,7 +172,7 @@ jobs:
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
-          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
 
       - name: Generate report
         id: report
@@ -305,7 +305,10 @@ jobs:
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
+          path: |
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
+            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
           if-no-files-found: error
           retention-days: 3
 
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index 1e99dce4e..e16fc0058 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -49,6 +49,25 @@ jobs:
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -96,8 +115,6 @@ jobs:
 
       - name: Run storage engine suite
         id: test
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -120,10 +137,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           PACKAGE_URL='${{ inputs.package_url }}'
@@ -145,83 +159,44 @@ jobs:
               RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
             fi
           fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
           SUITE: storage
         run: |
           set -euo pipefail
@@ -253,11 +228,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -285,14 +259,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -308,14 +284,15 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-storage-${{ github.run_id }}
+          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 612765874..5a91793a0 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -82,6 +82,25 @@ jobs:
     timeout-minutes: 420
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
       # auto-testing is private: clone it with the dedicated PF token (not
       # GITHUB_TOKEN) and retry transient GitHub/network failures.
       - name: Checkout auto-testing scripts (with retry)
@@ -142,7 +161,6 @@ jobs:
       - name: Run upgrade compatibility suite
         id: test
         env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
           set -euo pipefail
@@ -200,10 +218,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
           FROM_URL='${{ inputs.from_url }}'
@@ -224,103 +239,47 @@ jobs:
           else
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
-          rows = []
-          index = {}
-          topo_rows = []
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
-          PY
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- From: ${FROM_SOURCE}"
             echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
+            if [ "${RESULT}" = "success" ]; then
+              cat "${MATRIX_TABLE}"
+              echo ""
+              cat "${CASE_TABLE}"
+              echo ""
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}"
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          [ "${RESULT}" = "success" ]
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
           SUITE: upgrade
         run: |
           set -euo pipefail
@@ -352,11 +311,10 @@ jobs:
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -384,14 +342,16 @@ jobs:
             echo ""
             echo "- Suite: \`${SUITE}\`"
             echo "- Run: ${RUN_URL}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
             echo "- Trigger: ${GITHUB_EVENT_NAME}"
             echo "- Date: $(date -u +%Y-%m-%d)"
             echo ""
             echo "## Report (errors and symptoms)"
             echo ""
-            if [ -s "${REPORT_FILE}" ]; then
+            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
               redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
+            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
               echo "(report file missing; log tail below)"
               echo ""
               tail -n 200 "${LOG_FILE}" | redact
@@ -407,14 +367,16 @@ jobs:
           echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
+          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+          if-no-files-found: error
           retention-days: 3
 
       - name: Cleanup environment (after)
diff --git a/scripts/functional_case_report.py b/scripts/functional_case_report.py
new file mode 100644
index 000000000..7036eaad5
--- /dev/null
+++ b/scripts/functional_case_report.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Preserve every functional case execution and its suite context in reports."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import re
+
+
+def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
+    ansi = re.compile(r"\x1b\[[0-9;]*m")
+    start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
+    done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
+    context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
+    topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
+    rows = []
+    pending = {}
+    topo_rows = []
+    context = "context not recorded"
+    complete = True
+    try:
+        with log_file.open(encoding="utf-8", errors="replace") as log:
+            for raw in log:
+                line = ansi.sub("", raw).strip()
+                if match := context_re.match(line):
+                    context = match[1]
+                    pending.clear()
+                elif match := topo_re.match(line):
+                    topo_rows.append(match.groups())
+                elif match := start_re.match(line):
+                    case_id, name = match.groups()
+                    pending[case_id] = len(rows)
+                    rows.append([case_id, f"{name} ({context})", "RUNNING"])
+                elif match := done_re.match(line):
+                    status, case_id = match.groups()
+                    index = pending.pop(case_id, None)
+                    if index is None:
+                        complete = False
+                        rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
+                    else:
+                        rows[index][2] = status
+    except FileNotFoundError:
+        pass
+
+    counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
+    with case_file.open("w", encoding="utf-8") as out:
+        out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
+        for status, count in counts.items():
+            out.write(f"- {status}: {count}\n")
+        out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
+        for row in rows:
+            out.write("| " + " | ".join(value.replace("|", "&#124;") for value in row) + " |\n")
+        if not rows:
+            out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
+
+    valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
+    if matrix_file is not None:
+        with matrix_file.open("w", encoding="utf-8") as out:
+            out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
+            out.write("| --- | --- | --- | --- | --- |\n")
+            for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                result = "PASS" if nfail == "0" else "FAIL"
+                out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
+            if not topo_rows:
+                out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
+        valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
+    return valid
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("log_file", type=Path)
+    parser.add_argument("case_file", type=Path)
+    parser.add_argument("matrix_file", type=Path, nargs="?")
+    args = parser.parse_args()
+    raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)
diff --git a/scripts/test/oidc_keycloak_live.sh b/scripts/test/oidc_keycloak_live.sh
index c464f6dcb..73b8eb6da 100755
--- a/scripts/test/oidc_keycloak_live.sh
+++ b/scripts/test/oidc_keycloak_live.sh
@@ -187,7 +187,7 @@ values = {}
 for element in root.iter():
     values[element.tag.rsplit("}", 1)[-1]] = element.text or ""
 for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"):
-    assert values.get(field), values
+    assert values.get(field), f"missing required STS field: {field}"
 print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken")))
 PY
 )
@@ -218,7 +218,6 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \
   --data-urlencode DurationSeconds=900 \
   --data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")"
 [[ "${TAMPERED_STATUS}" == 403 ]] || {
-  cat "${WORK_DIR}/sts-tampered.xml" >&2
   echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2
   exit 1
 }
@@ -235,7 +234,6 @@ BAD_STATUS="$(curl --noproxy '*' -sS \
   --data-urlencode DurationSeconds=900 \
   --data-urlencode "WebIdentityToken=${BAD_TOKEN}")"
 [[ "${BAD_STATUS}" == 403 ]] || {
-  cat "${WORK_DIR}/sts-bad.xml" >&2
   echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2
   exit 1
 }
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index ea2d75487..4b4b063dd 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -3,14 +3,18 @@
 
 from __future__ import annotations
 
+import glob
+import json
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import unittest
 from pathlib import Path
 
 from check_test_wiring import yaml_block
+from functional_case_report import generate_report
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -38,7 +42,46 @@ def shell_body(lines: list[str]) -> str:
     return "\n".join(shell_lines)
 
 
-class SecurityWorkflowTests(unittest.TestCase):
+class WorkflowSteps:
+    def uploaded_files(self) -> set[Path]:
+        upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
+        start = upload.index("          path: |") + 1
+        paths = []
+        for line in upload[start:]:
+            if not line.startswith("            "):
+                break
+            paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
+        return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
+
+    def render(self, value: str) -> str:
+        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+        result = {}
+        for line in yaml_block(lines, "env", indent) or []:
+            if line.strip() and not line.lstrip().startswith("#"):
+                key, value = line.strip().split(": ", 1)
+                result[key] = self.render(value.strip("'\""))
+        return result
+
+    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+        lines = self.steps[name]
+        result = subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+        )
+        for line in lines:
+            if line.startswith("        id: "):
+                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+        if Path(self.env["GITHUB_ENV"]).exists():
+            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+                key, value = line.split("=", 1)
+                self.env[key] = value
+                self.context[f"env.{key}"] = value
+        return result
+
+
+class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -78,6 +121,7 @@ class SecurityWorkflowTests(unittest.TestCase):
             '#!/usr/bin/env bash\nset -euo pipefail\n'
             'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
             'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
+            'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
             'case "$FAKE_REPORT" in\n'
             f'  present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
             '  empty) : > "$REPORT_FILE" ;;\n'
@@ -86,33 +130,6 @@ class SecurityWorkflowTests(unittest.TestCase):
             'exit "$FAKE_EXIT"\n'
         )
 
-    def render(self, value: str) -> str:
-        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
-    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
-        result = {}
-        for line in yaml_block(lines, "env", indent) or []:
-            if line.strip() and not line.lstrip().startswith("#"):
-                key, value = line.strip().split(": ", 1)
-                result[key] = self.render(value.strip("'\""))
-        return result
-
-    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
-        lines = self.steps[name]
-        result = subprocess.run(
-            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
-            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
-        )
-        for line in lines:
-            if line.startswith("        id: "):
-                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
-        if Path(self.env["GITHUB_ENV"]).exists():
-            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
-                key, value = line.split("=", 1)
-                self.env[key] = value
-                self.context[f"env.{key}"] = value
-        return result
-
     def test_workflow_wiring(self) -> None:
         names = list(self.steps)
         self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -128,7 +145,7 @@ class SecurityWorkflowTests(unittest.TestCase):
         for name in ("Upload functional report to dashboard", "Upload report and logs"):
             self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
         artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
-        self.assertIn("          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
+        self.assertIn("          path: |", artifact_settings)
         self.assertIn("          if-no-files-found: error", artifact_settings)
 
     def test_suite_report_and_result_matrix(self) -> None:
@@ -147,7 +164,7 @@ class SecurityWorkflowTests(unittest.TestCase):
                 if outcome != "skipped" or mode == "present":
                     suite = self.run_step("Run security suite")
                     self.assertEqual(suite.returncode, exit_code, suite.stderr)
-                    logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
+                    logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
                     self.assertEqual(len(logs), 1)
                     self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
                 self.context["steps.test.outcome"] = outcome
@@ -169,37 +186,91 @@ class SecurityWorkflowTests(unittest.TestCase):
                 summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
                 self.assertEqual(summary, contents)
                 self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
+                expected = {self.artifacts / "report.md"}
+                if outcome != "skipped" or mode == "present":
+                    expected.add(self.artifacts / "suite.log")
+                    self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n")
+                if mode in ("present", "empty"):
+                    expected.add(self.artifacts / "suite-report.md")
+                (self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
+                scratch = Path(str(self.artifacts) + "-scratch")
+                (scratch / "case.out").write_text("FAKE-SECRET-CANARY")
+                self.assertEqual(self.uploaded_files(), expected)
+
+
+    def test_oidc_negative_responses_never_print_issued_credentials(self):
+        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+        for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
+            start = source.index('[[ "${' + variable + '}" == 403 ]]')
+            end = source.index("\n", source.index("grep -q '<Code>AccessDenied</Code>'", start))
+            guard = source[start:end]
+            credential_xml = "<Credentials><AccessKeyId>FAKE-ACCESS-CANARY</AccessKeyId><SecretAccessKey>FAKE-SECRET-CANARY</SecretAccessKey><SessionToken>FAKE-SESSION-CANARY</SessionToken></Credentials>"
+            for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
+                                           ("403", "<Code>AccessDenied</Code>", 0)):
+                with self.subTest(variable=variable, status=status, expected=expected):
+                    (self.directory / filename).write_text(body)
+                    result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
+                                            env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
+                                            capture_output=True, text=True)
+                    self.assertEqual(result.returncode, expected, result.stderr)
+                    for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+                        self.assertNotIn(canary, result.stdout + result.stderr)
+                    if status == "200":
+                        self.assertIn("HTTP 403, got 200", result.stderr)
+
+    def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
+        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+        start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
+        end = source.index("\n)\n", start) + 3
+        extract = source[start:end]
+        values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
+                  "SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
+                  "SubjectFromWebIdentityToken": "alice"}
+        for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
+            with self.subTest(missing=missing):
+                xml = "<Credentials>" + "".join(f"<{key}>{value}</{key}>" for key, value in values.items() if key != missing) + "</Credentials>"
+                (self.directory / "sts-good.xml").write_text(xml)
+                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
+                                        env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
+                self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
+                for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+                    self.assertNotIn(canary, result.stdout + result.stderr)
+                if missing:
+                    self.assertIn(f"missing required STS field: {missing}", result.stderr)
 
     def test_existing_evidence_directory_is_rejected(self) -> None:
-        self.artifacts.mkdir()
-        stale = self.artifacts / "suite-report.md"
-        stale.write_text("OLD RUN REPORT")
-        self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
-        self.assertEqual(stale.read_text(), "OLD RUN REPORT")
-        self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
-        (self.artifacts / "report.md").write_text("OLD RUN REPORT")
-        self.context.update({
-            "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
-        })
-        fake_bin = self.directory / "bin"
-        fake_bin.mkdir()
-        gh = fake_bin / "gh"
-        gh.write_text(
-            '#!/usr/bin/env bash\nset -euo pipefail\n'
-            'if [ "$1 $2" = "issue create" ]; then\n'
-            '  while [ "$#" -gt 0 ]; do\n'
-            '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
-            '    shift\n'
-            '  done\n'
-            'fi\n'
-        )
-        gh.chmod(0o755)
-        body = self.directory / "issue-body.md"
-        self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
-        result = self.run_step("File failure issue in rustfs/backlog")
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertNotIn("OLD RUN REPORT", body.read_text())
-        self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
+        for suffix in ("", "-scratch"):
+            self.setUp()
+            existing = Path(str(self.artifacts) + suffix)
+            existing.mkdir()
+            stale = existing / "suite-report.md"
+            stale.write_text("OLD RUN REPORT")
+            self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
+            self.assertEqual(stale.read_text(), "OLD RUN REPORT")
+            self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+            (self.artifacts / "report.md").write_text("OLD RUN REPORT")
+            self.context.update({
+                "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
+            })
+            fake_bin = self.directory / "bin"
+            fake_bin.mkdir()
+            gh = fake_bin / "gh"
+            gh.write_text(
+                '#!/usr/bin/env bash\nset -euo pipefail\n'
+                'if [ "$1 $2" = "issue create" ]; then\n'
+                '  while [ "$#" -gt 0 ]; do\n'
+                '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                '    shift\n'
+                '  done\n'
+                'fi\n'
+            )
+            gh.chmod(0o755)
+            body = self.directory / "issue-body.md"
+            self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
+            result = self.run_step("File failure issue in rustfs/backlog")
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertNotIn("OLD RUN REPORT", body.read_text())
+            self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
 
 class FunctionalWorkflowTests(unittest.TestCase):
@@ -225,7 +296,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 if suite in self.DIRECT_TESTS:
                     test = steps[self.DIRECT_TESTS[suite]]
                     self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
-                    self.assertIn("        if: always()", steps["Generate report"])
+                    self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
                 self.assertIn(condition, (
@@ -286,5 +357,379 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
 
 
+class FunctionalCaseReportTests(unittest.TestCase):
+    def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            log = root / "suite.log"
+            if text is not None:
+                log.write_text(text)
+            valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
+            return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
+
+    def test_repeated_case_executions_preserve_failure_and_context(self):
+        # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
+        log = subprocess.check_output(["bash", "-c", r'''
+log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
+log '== topology: single-single  kms-backend: local =='
+printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
+log '== topology: single-multi  kms-backend: vault-kv2 =='
+printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
+printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
+'''], text=True)
+        valid, cases, _ = self.report(log)
+        self.assertFalse(valid)
+        self.assertEqual(cases.count("| KMS-101 |"), 3)
+        self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
+        self.assertIn("roundtrip (topology: single-single  kms-backend: local) | FAIL |", cases)
+        self.assertIn("roundtrip (topology: single-multi  kms-backend: vault-kv2) | PASS |", cases)
+        self.assertNotIn("\\n", cases)
+
+    def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
+        for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
+                     "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
+            with self.subTest(log=text):
+                valid, cases, _ = self.report(text)
+                self.assertFalse(valid)
+                self.assertIn("## Case Summary", cases)
+        valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
+        self.assertTrue(valid)
+        self.assertIn("suite: bucket replication", cases)
+        self.assertIn("- UNSUPPORTED: 1\n", cases)
+
+    def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
+        case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
+        for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
+                                 ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
+            with self.subTest(matrix=suffix):
+                valid, _, matrix = self.report(case + suffix, matrix=True)
+                self.assertEqual(valid, expected)
+                self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
+                self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
+
+    def test_s3_case_identifiers_include_digits(self):
+        valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
+        self.assertTrue(valid)
+        self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
+
+
+class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
+    SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
+
+    def prepare(self, suite: str) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+        self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
+        self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
+        self.context.update({
+            "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
+            "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
+            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+            "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
+        })
+        self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
+        self.env = {
+            **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
+            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
+            "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
+        }
+        for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
+            self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
+        (self.directory / "scripts").mkdir()
+        (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
+        fake_bin = self.directory / "bin"
+        fake_bin.mkdir()
+        (fake_bin / "python3").symlink_to(sys.executable)
+        for command, body in (
+            ("ssh", 'printf "fixture-version\\n"\n'),
+            ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
+                   '  while [ "$#" -gt 0 ]; do\n'
+                   '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+                   '    shift\n'
+                   '  done\n'
+                   'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
+        ):
+            script = fake_bin / command
+            script.write_text("#!/bin/sh\n" + body)
+            script.chmod(0o755)
+        self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
+
+    def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
+        for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")):
+            with self.subTest(suite=suite, collision=suffix or "artifact"):
+                self.prepare(suite)
+                self.assertNotIn("/tmp/rustfs-", self.source)
+                names = list(self.steps)
+                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+                if suite in FunctionalWorkflowTests.DIRECT_TESTS:
+                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+                for name, lines in self.steps.items():
+                    if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
+                    if any("uses: actions/upload-artifact@" in line for line in lines):
+                        self.assertIn("          path: |", lines)
+                        self.assertIn("          if-no-files-found: error", lines)
+                existing = Path(str(self.artifacts) + suffix)
+                existing.mkdir()
+                for filename in ("report.md", "suite.log"):
+                    (existing / filename).write_text("OLD RUN EVIDENCE")
+                self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
+                initialized = self.run_step("Initialize functional evidence")
+                self.assertNotEqual(initialized.returncode, 0)
+                self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+                issue = self.run_step("File failure issue in rustfs/backlog")
+                self.assertEqual(issue.returncode, 0, issue.stderr)
+                body = Path(self.env["CAPTURE_BODY"]).read_text()
+                self.assertNotIn("OLD RUN EVIDENCE", body)
+                self.assertIn("no report or log file was produced", body)
+                self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
+
+    def test_reports_use_only_current_complete_suite_evidence(self):
+        for suite in self.SUITES[:-1]:
+            good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
+            partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
+            if suite == "s3-compat":
+                good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
+            if suite == "upgrade":
+                good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
+            if suite == "heal":
+                good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
+                partial = "[HEAL-STEP] 1 fixture PASS\n"
+            for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
+                                 ("success", None), ("skipped", None), ("cancelled", good)):
+                with self.subTest(suite=suite, outcome=outcome, log=log):
+                    self.prepare(suite)
+                    stale = self.directory / "old-suite.log"
+                    stale.write_text("OLD RUN EVIDENCE\n" + good)
+                    self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
+                    self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                    self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
+                    self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
+                    if log is not None:
+                        Path(self.env["LOG_FILE"]).write_text(log)
+                    self.context["steps.test.outcome"] = outcome
+                    report = self.run_step("Generate report")
+                    success = outcome == "success" and log == good
+                    self.assertEqual(report.returncode == 0, success, report.stderr)
+                    contents = Path(self.env["REPORT_FILE"]).read_text()
+                    self.assertNotIn("OLD RUN EVIDENCE", contents)
+                    self.assertEqual("| PASS |" in contents, success)
+                    for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
+                                  f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
+                        self.assertIn(value, contents)
+                    self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
+                    evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
+                    if log in (good, partial):
+                        self.assertIn("| PASS |", evidence)
+                    self.assertNotIn("OLD RUN EVIDENCE", evidence)
+
+    def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                if suite == "heal":
+                    self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
+                scripts = self.directory / "auto-testing"
+                scripts.mkdir()
+                filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
+                script = scripts / filename
+                script.write_text(
+                    '#!/bin/bash\nset -euo pipefail\nlog=""\n'
+                    'while [ "$#" -gt 0 ]; do\n'
+                    '  if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
+                    '  shift\n'
+                    'done\n'
+                    '[ "$log" = "$LOG_FILE" ] || exit 31\n'
+                    'printf "CURRENT SUITE LOG\\n" > "$log"\n'
+                    'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
+                    'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
+                    'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
+                    '  mkdir -p "$RUSTFS_RESULT_DIR"\n'
+                    '  printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
+                    'fi\n'
+                )
+                script.chmod(0o755)
+                name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
+                    "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
+                )
+                result = self.run_step(name)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
+                self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1)
+                self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
+                if suite == "performance":
+                    self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
+
+    def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
+        extra = {
+            "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
+            "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"],
+            "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
+                            "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
+        }
+        for suite in self.SUITES:
+            with self.subTest(suite=suite):
+                self.prepare(suite)
+                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+                expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
+                for path in expected:
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_text("PARTIAL FAILURE DIAGNOSTIC")
+                for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
+                    directory.mkdir(exist_ok=True)
+                    (directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
+                self.assertEqual(self.uploaded_files(), expected)
+                self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
+
+    def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
+        self.prepare("kms")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        scripts = self.directory / "auto-testing"
+        scripts.mkdir()
+        # Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
+        # The Docker boundary returns synthetic credentials; setup fails before vault_stop.
+        (scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
+set -Eeuo pipefail
+TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
+trap 'rm -rf "${TEST_TMP}"' EXIT
+VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
+VAULT_CONTAINER="rustfs-vault"
+heal_run() { "$@"; }
+while [ "$#" -gt 0 ]; do
+  if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
+  shift
+done
+mkdir -p "${VAULT_DATA_DIR}"
+tmp_init="${TEST_TMP}/vault-init.json"
+tmp_err="${TEST_TMP}/vault-init.stderr"
+heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
+  vault operator init -key-shares=1 -key-threshold=1 -format=json \
+  > "${tmp_init}" 2>"${tmp_err}"
+cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
+printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
+exit 42
+''')
+        docker = self.directory / "bin/docker"
+        docker.write_text("""#!/bin/sh
+[ "$1" = exec ] || exit 99
+printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
+""")
+        docker.chmod(0o755)
+        result = self.run_step("Run KMS suite")
+        self.assertEqual(result.returncode, 42, result.stderr)
+        vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
+        self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
+        self.assertNotIn(self.artifacts, vault_init.parents)
+        self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
+        self.assertNotEqual(self.run_step("Generate report").returncode, 0)
+        self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
+        self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
+        for path in self.uploaded_files():
+            self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
+            self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
+
+    def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
+        self.prepare("heal")
+        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+        script = self.directory / "auto-testing/rustfs_heal_test.sh"
+        script.parent.mkdir()
+        # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
+        script.write_text(r'''#!/bin/bash
+set -euo pipefail
+SELECTED_STEPS=()
+PREFLIGHT=0
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
+    --log-file) LOG_FILE="$2"; shift ;;
+    --preflight) PREFLIGHT=1 ;;
+  esac
+  shift
+done
+if [ "$PREFLIGHT" -eq 1 ]; then
+  printf '\n' >> "$INVOKED_STEPS"
+  exit 0
+fi
+printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
+emit_step_result() {
+  local n="$1" desc="$2" status="$3"
+  printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
+}
+{
+  for step in "${SELECTED_STEPS[@]}"; do
+    emit_step_result "$step" "fixture step $step" PASS
+  done
+  want_all=1
+  for s in 1 2 3 4 5 6 7; do
+    [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
+  done
+  if [ "${want_all}" -eq 1 ]; then
+    printf '[HEAL-RESULT] PASS all steps passed\n'
+  fi
+} >> "$LOG_FILE"
+''')
+        script.chmod(0o755)
+        self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
+        for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
+            result = self.run_step(name)
+            self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
+        log = Path(self.env["LOG_FILE"]).read_text()
+        self.assertNotIn("[HEAL-RESULT]", log)
+        self.assertEqual(log.count("[HEAL-STEP]"), 7)
+        report = self.run_step("Generate report")
+        self.assertEqual(report.returncode, 0, report.stderr)
+        failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
+                       for step in range(1, 8)]
+        failed_logs += [
+            log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
+            log + "[HEAL-STEP] 3 later step failure FAIL\n",
+            log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
+            log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
+        ]
+        for failed_log in failed_logs:
+            with self.subTest(log=failed_log):
+                Path(self.env["LOG_FILE"]).write_text(failed_log)
+                report = self.run_step("Generate report")
+                self.assertNotEqual(report.returncode, 0, report.stderr)
+                contents = Path(self.env["REPORT_FILE"]).read_text()
+                self.assertIn("Test Step Outcome: failure", contents)
+                self.assertNotIn("| PASS |", contents)
+                if "original failure" in failed_log:
+                    self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
+                if "later step failure" in failed_log:
+                    self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
+
+    def test_performance_results_version_and_report_are_bound_to_the_run(self):
+        self.prepare("performance")
+        initialized = self.run_step("Initialize functional evidence")
+        self.assertEqual(initialized.returncode, 0, initialized.stderr)
+        self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
+        self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
+        version = self.run_step("Collect RustFS version info")
+        self.assertEqual(version.returncode, 0, version.stderr)
+        self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
+        old_summary = self.directory / "old-results/summary.md"
+        old_summary.parent.mkdir()
+        old_summary.write_text("OLD RUN EVIDENCE")
+        upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
+        self.assertNotEqual(self.run_step(upload).returncode, 0)
+        self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
+        results = Path(self.env["RUSTFS_RESULT_DIR"])
+        results.mkdir()
+        (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
+        report = self.run_step(upload)
+        self.assertEqual(report.returncode, 0, report.stderr)
+        contents = Path(self.env["REPORT_FILE"]).read_text()
+        for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
+                      "CURRENT PERFORMANCE RESULTS", "fixture-version"):
+            self.assertIn(value, contents)
+        self.assertNotIn("OLD RUN EVIDENCE", contents)
+
+
 if __name__ == "__main__":
     unittest.main()

From 1210428b6db37f7c412bb5282a1133dde0cc82ee Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:14:37 +0800
Subject: [PATCH 4/5] fix(ci): publish immutable nightly package candidates
 (#7202)

---
 .config/make/tests.mak                  |   1 +
 .github/actions/quick-checks/action.yml |   1 +
 .github/workflows/nightly-gnu.yml       |  59 ++++++-
 scripts/test_nightly_candidate.py       | 208 ++++++++++++++++++++++++
 4 files changed, 261 insertions(+), 8 deletions(-)
 create mode 100644 scripts/test_nightly_candidate.py

diff --git a/.config/make/tests.mak b/.config/make/tests.mak
index d1297e9ad..726ca04cf 100644
--- a/.config/make/tests.mak
+++ b/.config/make/tests.mak
@@ -41,6 +41,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml
index e6cc59775..18c6cb64d 100644
--- a/.github/actions/quick-checks/action.yml
+++ b/.github/actions/quick-checks/action.yml
@@ -100,6 +100,7 @@ runs:
         python3 ./scripts/check_test_wiring.py --self-test
         python3 ./scripts/check_scheduled_validation_freshness.py --self-test
         python3 ./scripts/test_security_workflow.py
+        python3 ./scripts/test_nightly_candidate.py
         python3 ./scripts/check_test_wiring.py
 
     - name: Check no planning docs committed
diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml
index 0dedbf465..1864a9715 100644
--- a/.github/workflows/nightly-gnu.yml
+++ b/.github/workflows/nightly-gnu.yml
@@ -166,8 +166,9 @@ jobs:
       # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
       # Skipped when the R2 secrets are not configured (artifact-only mode).
       - name: Upload DEB to Cloudflare R2
-        if: env.R2_ACCESS_KEY_ID != ''
+        id: publish
         env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
             exit 0
           fi
 
-          if ! command -v aws >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y -qq awscli
-          fi
-
           export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          SOURCE_SHA="$(git rev-parse HEAD)"
+          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+            echo "Checkout SHA does not match the nightly build run" >&2
+            exit 1
+          fi
+          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
+          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
+          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
+
+          # Old AWS CLI models lack conditional PutObject support. Never fall
+          # back to an overwriting upload for a candidate.
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
+            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          "${AWS_CLI}" --version
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
+            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
+          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
+            echo "Published candidate checksum does not match the built package" >&2
+            exit 1
+          fi
+
           R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
 
           echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           # Stable "latest" alias so tests can fetch the newest nightly
           # without knowing today's date.
           echo "📤 Uploading latest alias"
-          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
             --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           echo "✅ R2 upload complete"
 
+          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg source_sha "${SOURCE_SHA}" \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
+            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            > "${CANDIDATE_FILE}"
+          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload nightly candidate manifest
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error
+
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py
new file mode 100644
index 000000000..3e5a50401
--- /dev/null
+++ b/scripts/test_nightly_candidate.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Exercise the nightly publication step without AWS, network or package builds."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+from check_test_wiring import yaml_block
+
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
+
+
+class NightlyCandidateTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.package = self.root / "rustfs-nightly-2026-09-06.deb"
+        self.package.write_bytes(b"built package bytes\x00\xff")
+        for command in (["git", "init", "-q"], ["git", "add", self.package.name],
+                        ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
+            subprocess.run(command, cwd=self.root, check=True, capture_output=True)
+        self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
+        self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
+        self.output = self.root / "github-output"
+        self.store = self.root / "store"
+        self.shims = self.root / "fake-tools.sh"
+        self.shims.write_text(r'''aws() {
+  printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
+  if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
+  if [[ "$*" == *--generate-cli-skeleton* ]]; then
+    if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
+      printf '{}\n'
+    else
+      printf '{"IfNoneMatch":""}\n'
+    fi
+    return
+  fi
+  if [[ "$1 $2" == 's3api put-object' ]]; then
+    [[ "$FAKE_MODE" != upload-failure ]] || return 42
+    shift 2
+    local key="" body="" condition=""
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        --key) key="$2";;
+        --body) body="$2";;
+        --if-none-match) condition="$2";;
+      esac
+      shift 2
+    done
+    [[ -z "$condition" || "$condition" == '*' ]] || return 43
+    if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
+    mkdir -p "$(dirname "$FAKE_STORE/$key")"
+    cp "$body" "$FAKE_STORE/$key"
+  elif [[ "$1 $2" == 's3 cp' ]]; then
+    [[ "$FAKE_MODE" != alias-failure ]] || return 45
+    local destination="${4#s3://test-bucket/}"
+    [[ "$destination" != */ ]] || destination+="$(basename "$3")"
+    mkdir -p "$(dirname "$FAKE_STORE/$destination")"
+    cp "$3" "$FAKE_STORE/$destination"
+  else
+    return 46
+  fi
+}
+curl() {
+  local url="${!#}"
+  printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
+  [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
+  [[ "$FAKE_MODE" != missing-public-url ]] || return 22
+  if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
+  cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
+  [[ "$FAKE_MODE" != incomplete-download ]] || return 47
+}
+sudo() {
+  [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
+  [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
+}
+python3() {
+  [[ "$1 $2" == '-m venv' ]] || return 50
+  mkdir -p "$3/bin"
+  cat > "$3/bin/python" <<'SH'
+#!/usr/bin/env bash
+[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
+[[ "$FAKE_MODE" != install-failure ]] || exit 52
+: > "$FAKE_INSTALLED"
+SH
+  printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
+  chmod +x "$3/bin/python" "$3/bin/aws"
+}
+''')
+        self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
+                        R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
+                        RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
+                        FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
+        source = WORKFLOW.read_text()
+        job = yaml_block(source.splitlines(), "build", 2)
+        starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
+        self.steps = {
+            job[start].split(": ", 1)[1]: job[start:end]
+            for start, end in zip(starts, starts[1:] + [len(job)])
+        }
+        self.publish = self.steps["Upload DEB to Cloudflare R2"]
+        start = self.publish.index("        run: |") + 1
+        self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith("          "))
+
+    def run_publish(self, **overrides):
+        self.output.unlink(missing_ok=True)
+        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
+                              cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
+
+    def manifest(self):
+        output = self.output.read_text().strip()
+        self.assertTrue(output.startswith("candidate_file="), output)
+        return json.loads(Path(output.split("=", 1)[1]).read_text())
+
+    def test_success_binds_actual_package_checkout_and_attempt(self):
+        result = self.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.manifest()
+        self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
+                                   "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
+        for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
+            self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
+        self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
+
+    def test_missing_credentials_remain_artifact_only(self):
+        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+            with self.subTest(missing=key):
+                result = self.run_publish(**{key: ""})
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertFalse(self.output.exists())
+                self.assertFalse((self.root / "aws.log").exists())
+                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_publication_failures_never_emit_a_candidate(self):
+        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
+            with self.subTest(mode=mode):
+                result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
+                self.assertNotEqual(result.returncode, 0, result.stdout)
+                self.assertFalse(self.output.exists())
+                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
+        result = self.run_publish(FAKE_MODE="old-cli")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue((self.root / "installed").exists())
+        self.assertEqual(self.manifest()["package_sha256"], self.digest)
+        self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+    def test_failed_cli_bootstrap_cannot_publish(self):
+        for mode in ("bootstrap-failure", "install-failure", "broken-install"):
+            with self.subTest(mode=mode):
+                (self.root / "installed").unlink(missing_ok=True)
+                result = self.run_publish(FAKE_MODE=mode)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse(self.output.exists())
+                self.assertFalse(self.store.exists())
+                self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+    def test_checkout_sha_mismatch_fails_before_upload(self):
+        result = self.run_publish(GITHUB_SHA="f" * 40)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("Checkout SHA", result.stderr)
+        self.assertFalse(self.output.exists())
+        self.assertFalse((self.root / "aws.log").exists())
+
+    def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
+        urls = []
+        for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
+            result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            urls.append(self.manifest()["package_url"])
+        self.assertEqual(len(set(urls)), 3)
+        self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
+
+    def test_duplicate_key_is_not_overwritten_or_recertified(self):
+        result = self.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
+        stored = self.store / key
+        stored.write_bytes(b"preexisting conflicting object")
+        (self.root / "nightly-candidate-12345-1.json").unlink()
+        result = self.run_publish()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
+        self.assertFalse(self.output.exists())
+        self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+    def test_manifest_upload_requires_publication_output(self):
+        upload = self.steps["Upload nightly candidate manifest"]
+        self.assertIn("        id: publish", self.publish)
+        self.assertIn("          DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
+        self.assertIn("        if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
+        self.assertIn("          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
+        self.assertIn("          path: ${{ steps.publish.outputs.candidate_file }}", upload)
+        self.assertIn("          if-no-files-found: error", upload)
+        self.assertNotIn("        continue-on-error: true", self.publish)
+        self.assertNotIn("          overwrite: true", upload)
+
+
+if __name__ == "__main__":
+    unittest.main()

From a6b5da64f27c2c0c82fe8ce5592a0e9e80138655 Mon Sep 17 00:00:00 2001
From: Zhengchao An <anzhengchao@gmail.com>
Date: Sun, 6 Sep 2026 00:15:03 +0800
Subject: [PATCH 5/5] fix(ci): serialize performance on shared functional VMs
 (#7204)

---
 .github/workflows/rustfs-functional-chain.yml |  17 +--
 .github/workflows/rustfs-performance-test.yml |   9 +-
 .github/workflows/rustfs-replication-test.yml |  50 ++++++--
 scripts/test_security_workflow.py             | 112 +++++++++++++++++-
 4 files changed, 157 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml
index 6ce828c1a..10b9c67ad 100644
--- a/.github/workflows/rustfs-functional-chain.yml
+++ b/.github/workflows/rustfs-functional-chain.yml
@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index 148e5ccf3..d698b27e0 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index ae8f9dc50..0faaf7809 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -330,13 +329,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index 4b4b063dd..b96b3e3f5 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Exercise functional workflow failures and security evidence without remote VMs."""
+"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
 
 from __future__ import annotations
 
@@ -272,6 +272,110 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
             self.assertNotIn("OLD RUN REPORT", body.read_text())
             self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
+    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
+        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+            with self.subTest(suite=suite):
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+                # Workflow-level concurrency covers every job, including cleanup,
+                # regardless of trigger or the runner hosting the job.
+                self.assertEqual([
+                    line.strip() for line in yaml_block(source, "concurrency", 0)
+                    if line.strip() and not line.lstrip().startswith("#")
+                ], [
+                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
+                ])
+                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
+                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
+                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
+                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
+
+    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
+            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
+                self.setUp()
+                fake_bin = self.directory / "bin"
+                fake_bin.mkdir()
+                commands = {
+                    "gh": '''#!/usr/bin/env bash
+set -euo pipefail
+if [ "$1" = api ]; then
+  printf '%s\\n' "$*" >> "$DISPATCHES"
+  attempt=$(wc -l < "$DISPATCHES")
+  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
+elif [ "$1 $2" = 'issue create' ]; then
+  printf 'issue\\n' >> "$EXECUTED"
+  while [ "$#" -gt 0 ]; do
+    if [ "$1" = --body-file ]; then
+      cat "$2" > "$CAPTURE_BODY"
+      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
+    fi
+    shift
+  done
+  exit "$ISSUE_EXIT"
+else
+  exit 99
+fi
+''',
+                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
+                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
+                }
+                for name, contents in commands.items():
+                    command = fake_bin / name
+                    command.write_text(contents)
+                    command.chmod(0o755)
+                dispatches = self.directory / "dispatches"
+                executed = self.directory / "executed"
+                body = self.directory / "issue-body.md"
+                body_path = self.directory / "issue-body-path"
+                self.env.update(
+                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
+                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
+                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
+                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
+                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+                )
+                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
+                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
+                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
+                started = self.run_step("Dispatch first suite (upgrade)")
+                self.assertEqual(started.returncode, 0, started.stderr)
+                self.assertEqual(dispatches.read_text().splitlines(), [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
+                ])
+                dispatches.unlink()
+
+                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
+                job = yaml_block(replication.splitlines(), "replication-test", 2)
+                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
+                self.steps = named_steps(job)
+                handoff = "Continue functional chain (next: Performance)"
+                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
+                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
+                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
+                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
+                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
+                failed = self.run_step("Run replication suite")
+                self.assertEqual(failed.returncode, 17, failed.stderr)
+                cleaned = self.run_step("Cleanup environment (after)")
+                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
+                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
+                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
+                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
+                forwarded = self.run_step(handoff)
+                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
+                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
+                self.assertEqual(calls, [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+                ] * (min(failed_attempts + 1, 3) if token else 0))
+                if failed_attempts == 3:
+                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
+                    self.assertIn("rustfs-chain-performance", body.read_text())
+                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
+                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+
 
 class FunctionalWorkflowTests(unittest.TestCase):
     JOBS = {
@@ -305,7 +409,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                     "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
                 ))
                 if suite != "performance":
-                    handoff = steps["Chain complete"] if suite == "replication" else next(
+                    handoff = next(
                         value for name, value in steps.items() if name.startswith("Continue functional chain")
                     )
                     self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
@@ -348,13 +452,13 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertIn("partial suite diagnostics", failed.stdout)
                 cleanup = execute("Cleanup environment (after)")
                 self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
-                handoff_name = "Chain complete" if suite == "replication" else next(
+                handoff_name = next(
                     name for name in steps if name.startswith("Continue functional chain")
                 )
                 handoff = execute(handoff_name)
                 self.assertEqual(handoff.returncode, 0, handoff.stderr)
                 markers = (root / "executed").read_text().splitlines()
-                self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
+                self.assertEqual(markers, ["cleanup", "dispatch"])
 
 
 class FunctionalCaseReportTests(unittest.TestCase):