From bcf5184a858a7b9526a07cfa2dad2c64ef2e99bf Mon Sep 17 00:00:00 2001
From: hector <42570491+majinghe@users.noreply.github.com>
Date: Sat, 12 Sep 2026 17:33:29 +0800
Subject: [PATCH] ci: green runs for case failures; red only for harness
 breakdowns (#7698)

A failing product case used to turn the whole workflow red, so the run
conclusion carried no signal beyond 'something failed' and the report
was suppressed. New semantics across the functional suites:

- Suite steps run with continue-on-error: the outcome is still recorded
  for the report and the backlog issue manager (security/tier already
  carried the flag).
- Generate report always publishes the full per-case table plus a
  'Product result: N passed, M failed' summary, and its exit gate is
  harness health: red only when the suite never reached case level (no
  case verdicts), failed wholesale (zero passes, >=3 failures), or was
  cancelled/skipped. performance is unchanged (parked).
- tier's structured gate no longer fails on case failures; it keeps red
  for evidence-init and missing-gate-result breakdowns.
- pool/performance keep their existing red sources (install/benchmark).

Workflow contract tests updated to the new exit semantics: the security
report matrix keys green off the suite outcome, the evidence matrix
expects green for failure outcomes with recorded case rows (except
performance), the heal staged-rerun block expects the per-step table to
always publish, and run steps are now required to carry
continue-on-error.

Verified locally: actionlint clean; test_security_workflow.py 21/21.
---
 .../workflows/rustfs-fault-tolerance-test.yml | 17 ++++++++++
 .github/workflows/rustfs-heal-test.yml        | 28 +++++++++++++++--
 .github/workflows/rustfs-kms-test.yml         | 29 +++++++++++++++--
 .github/workflows/rustfs-pool-expand-test.yml |  3 ++
 .github/workflows/rustfs-replication-test.yml | 29 +++++++++++++++--
 .github/workflows/rustfs-s3-compat-test.yml   | 29 +++++++++++++++--
 .github/workflows/rustfs-security-test.yml    | 27 ++++++++++++----
 .github/workflows/rustfs-storage-test.yml     | 29 +++++++++++++++--
 .github/workflows/rustfs-tier-test.yml        | 20 +++++-------
 .github/workflows/rustfs-upgrade-test.yml     | 31 +++++++++++++++++--
 scripts/test_security_workflow.py             | 29 ++++++++++++-----
 11 files changed, 226 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/rustfs-fault-tolerance-test.yml b/.github/workflows/rustfs-fault-tolerance-test.yml
index 2c4ee0e6e..9bdf3677f 100644
--- a/.github/workflows/rustfs-fault-tolerance-test.yml
+++ b/.github/workflows/rustfs-fault-tolerance-test.yml
@@ -118,6 +118,9 @@ jobs:
 
       - name: Run fault-tolerance scenarios (A, B, C, C2)
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           ARGS=(--all -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
           if [ "${{ inputs.strict }}" = "true" ]; then
@@ -149,6 +152,20 @@ jobs:
               printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
             done
           } > "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          # Product gate: failing FT cases keep the run green — the report above and
+          # the backlog issue manager carry the signal. Only harness/environment
+          # breakdowns (no case verdicts at all) turn the workflow red.
+          FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
+          HARNESS_OK=0
+          if [ "${{ steps.test.outcome }}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Manage backlog issues (dedup / label / auto-close)
         # Signal-based lifecycle: dedups against open issues by label
diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index a86deb332..3d2f1a947 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -143,6 +143,9 @@ jobs:
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
@@ -224,6 +227,20 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing steps keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red.
+          STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
+          STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
@@ -235,19 +252,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${STEPS_TABLE}" ]; then
               cat "${STEPS_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${STEPS_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; step failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index e74f9ddaf..a88524562 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -126,6 +126,9 @@ jobs:
 
       - name: Run KMS suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -174,6 +177,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
@@ -185,19 +203,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml
index f3736675f..6653c348b 100644
--- a/.github/workflows/rustfs-pool-expand-test.yml
+++ b/.github/workflows/rustfs-pool-expand-test.yml
@@ -227,6 +227,9 @@ jobs:
 
       - name: Run pool expansion & decommission test
         id: pool_test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -o pipefail
           STEPS="4,5,6"
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 5e8ca4454..dbfa9ca1c 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -130,6 +130,9 @@ jobs:
 
       - name: Run replication suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -181,6 +184,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
@@ -193,19 +211,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index 7bdc16de4..c0e6facc9 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -105,6 +105,9 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -150,6 +153,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
@@ -162,19 +180,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index b8926b7a1..2144c6e08 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -144,6 +144,8 @@ jobs:
 
       - name: Run security suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -184,6 +186,20 @@ jobs:
           if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red.
+          VERDICTS_TOTAL="$(grep -cE '^\[(PASS|FAIL|UNSUPPORTED)\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_FAIL="$(grep -cE '^\[FAIL\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_TOTAL=$(( ${VERDICTS_TOTAL:-0} + 0 )); VERDICTS_FAIL=$(( ${VERDICTS_FAIL:-0} + 0 ))
+          VERDICTS_PASS=$(( VERDICTS_TOTAL - VERDICTS_FAIL ))
+          HARNESS_OK=0
+          if [ "${TEST_OUTCOME}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${TEST_OUTCOME}" = "failure" ] && [ "${VERDICTS_TOTAL}" -gt 0 ] && [ "${VERDICTS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS security test report"
             echo ""
@@ -196,16 +212,15 @@ jobs:
             echo ""
             # The dashboard prioritizes case rows over the step outcome.
             # Keep partial case results in the artifact when the suite fails.
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
               cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
-            elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
-              echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
-            else
-              echo "The suite did not produce a non-empty report."
+              echo ""
             fi
+            echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed (failing cases are tracked in rustfs/backlog)"
           } > "${SECURITY_ARTIFACTS_DIR}/report.md"
           cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index 3651fe486..c350b2166 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -114,6 +114,9 @@ jobs:
 
       - name: Run storage engine suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -165,6 +168,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
@@ -177,19 +195,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml
index 4314331fa..ec7acba77 100644
--- a/.github/workflows/rustfs-tier-test.yml
+++ b/.github/workflows/rustfs-tier-test.yml
@@ -257,6 +257,8 @@ jobs:
 
       - name: Run tier suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -471,26 +473,18 @@ jobs:
           GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
         run: |
           set -euo pipefail
+          # Product gate: failing cases keep the run green (the structured gate
+          # rc and the report carry the per-case signal, and the backlog issue
+          # manager tracks failures). Only harness/environment breakdowns —
+          # evidence initialization or a missing structured gate result — turn
+          # the workflow red.
           failed=0
           if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
             echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
             failed=1
-          fi
-          if [ "${TEST_OUTCOME}" != "success" ]; then
-            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
-            failed=1
-          fi
-          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-            echo "structured gate result is unavailable because evidence initialization failed" >&2
           elif [ ! -s "${GATE_RC_FILE}" ]; then
             echo "structured gate result is missing" >&2
             failed=1
-          else
-            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
-            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
-              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
-              failed=1
-            fi
           fi
           [ "${failed}" -eq 0 ]
 
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index bad5c1119..ca4b3333e 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -160,6 +160,9 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
@@ -247,6 +250,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
@@ -259,21 +277,28 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${MATRIX_TABLE}" ]; then
               cat "${MATRIX_TABLE}"
               echo ""
+            fi
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index 340033605..10eee9d4e 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -170,8 +170,12 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                     self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
                 self.context["steps.test.outcome"] = outcome
                 report = self.run_step("Generate report")
+                # The report step is red only for harness/environment breakdowns;
+                # the fixture log carries no case verdicts, so failure outcomes
+                # stay red here, and a successful suite is green unconditionally.
                 success = outcome == "success" and mode == "present"
-                self.assertEqual(report.returncode == 0, success, report.stderr)
+                green = outcome == "success"
+                self.assertEqual(report.returncode == 0, green, report.stderr)
                 contents = (self.artifacts / "report.md").read_text()
                 for expected in (
                     "https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
@@ -179,8 +183,8 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                     f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
                 ):
                     self.assertIn(expected, contents)
-                self.assertEqual(CASE_ROW in contents, success)
-                self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success)
+                self.assertEqual(CASE_ROW in contents, mode == "present")
+                self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, mode == "present")
                 if mode == "present":
                     raw = (self.artifacts / "suite-report.md").read_text()
                     self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
@@ -404,7 +408,9 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 steps = named_steps(job)
                 if suite in self.DIRECT_TESTS:
                     test = steps[self.DIRECT_TESTS[suite]]
-                    self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
+                    # Case failures keep the run green; the step records its
+                    # outcome for the report and the backlog issue manager.
+                    self.assertRegex("\n".join(test), r'''(?m)^        continue-on-error: true$''')
                     self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
@@ -621,10 +627,14 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
                     self.context["steps.test.outcome"] = outcome
                     report = self.run_step("Generate report")
                     success = outcome == "success" and log == good
-                    self.assertEqual(report.returncode == 0, success, report.stderr)
+                    # Report steps are red only for harness/environment breakdowns;
+                    # a failure outcome with recorded case rows stays green
+                    # (performance is unchanged and still gates on the suite result).
+                    green = (outcome == "success" or (outcome == "failure" and log in (good, partial))) if suite != "performance" else success
+                    self.assertEqual(report.returncode == 0, green, report.stderr)
                     contents = Path(self.env["REPORT_FILE"]).read_text()
                     self.assertNotIn("OLD RUN EVIDENCE", contents)
-                    self.assertEqual("| PASS |" in contents, success)
+                    self.assertEqual("| PASS |" in contents, success if suite == "performance" else log in (good, partial))
                     for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
                                   f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
                         self.assertIn(value, contents)
@@ -802,11 +812,14 @@ emit_step_result() {
         for failed_log in failed_logs:
             with self.subTest(log=failed_log):
                 Path(self.env["LOG_FILE"]).write_text(failed_log)
+                # A failing heal run exits non-zero; the report step stays green
+                # because the steps ran, and the per-step table is always published.
+                self.context["steps.test.outcome"] = "failure"
                 report = self.run_step("Generate report")
-                self.assertNotEqual(report.returncode, 0, report.stderr)
+                self.assertEqual(report.returncode, 0, report.stderr)
                 contents = Path(self.env["REPORT_FILE"]).read_text()
                 self.assertIn("Test Step Outcome: failure", contents)
-                self.assertNotIn("| PASS |", contents)
+                self.assertIn("| PASS |", contents)
                 if "original failure" in failed_log:
                     self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
                 if "later step failure" in failed_log: