gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-21 18:13:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: green runs for case failures; red only for harness breakdowns (#7698)

Browse Source 
A failing product case used to turn the whole workflow red, so the run
conclusion carried no signal beyond 'something failed' and the report
was suppressed. New semantics across the functional suites:

- Suite steps run with continue-on-error: the outcome is still recorded
  for the report and the backlog issue manager (security/tier already
  carried the flag).
- Generate report always publishes the full per-case table plus a
  'Product result: N passed, M failed' summary, and its exit gate is
  harness health: red only when the suite never reached case level (no
  case verdicts), failed wholesale (zero passes, >=3 failures), or was
  cancelled/skipped. performance is unchanged (parked).
- tier's structured gate no longer fails on case failures; it keeps red
  for evidence-init and missing-gate-result breakdowns.
- pool/performance keep their existing red sources (install/benchmark).

Workflow contract tests updated to the new exit semantics: the security
report matrix keys green off the suite outcome, the evidence matrix
expects green for failure outcomes with recorded case rows (except
performance), the heal staged-rerun block expects the per-step table to
always publish, and run steps are now required to carry
continue-on-error.

Verified locally: actionlint clean; test_security_workflow.py 21/21.
This commit is contained in:
hector
2026-09-12 17:33:29 +08:00
committed by GitHub
parent f90d4071a5
commit bcf5184a85
11 changed files with 226 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/rustfs-fault-tolerance-test.yml
+17
View File
@@ -118,6 +118,9 @@ jobs:
- name: Run fault-tolerance scenarios (A, B, C, C2)
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
ARGS=(--all -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
if [ "${{ inputs.strict }}" = "true" ]; then
@@ -149,6 +152,20 @@ jobs:
printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
done
} > "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
# Product gate: failing FT cases keep the run green — the report above and
# the backlog issue manager carry the signal. Only harness/environment
# breakdowns (no case verdicts at all) turn the workflow red.
FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
HARNESS_OK=0
if [ "${{ steps.test.outcome }}" = "success" ]; then
HARNESS_OK=1
elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
HARNESS_OK=1
fi
[ "${HARNESS_OK}" = "1" ]
- name: Manage backlog issues (dedup / label / auto-close)
# Signal-based lifecycle: dedups against open issues by label
.github/workflows/rustfs-heal-test.yml
+25 -3
View File
@@ -143,6 +143,9 @@ jobs:
- name: Run heal test (write -> outage -> heal -> verify)
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
./auto-testing/rustfs_heal_test.sh \
--steps "3,4,5,6,7" -y \
@@ -224,6 +227,20 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing steps keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns turn the workflow red.
STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS heal test report"
echo ""
@@ -235,19 +252,24 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${STEPS_TABLE}" ]; then
cat "${STEPS_TABLE}"
echo ""
fi
echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)"
echo ""
if [ -s "${STEPS_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; step failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-kms-test.yml
+26 -3
View File
@@ -126,6 +126,9 @@ jobs:
- name: Run KMS suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-kms-test.sh
@@ -174,6 +177,21 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns (the suite never reached case level, or a wholesale
# failure with zero passes) turn the workflow red.
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS KMS test report"
echo ""
@@ -185,19 +203,24 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${CASE_TABLE}" ]; then
cat "${CASE_TABLE}"
echo ""
fi
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
echo ""
if [ -s "${CASE_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-pool-expand-test.yml
+3
View File
@@ -227,6 +227,9 @@ jobs:
- name: Run pool expansion & decommission test
id: pool_test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
set -o pipefail
STEPS="4,5,6"
.github/workflows/rustfs-replication-test.yml
+26 -3
View File
@@ -130,6 +130,9 @@ jobs:
- name: Run replication suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-replication-test.sh
@@ -181,6 +184,21 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns (the suite never reached case level, or a wholesale
# failure with zero passes) turn the workflow red.
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS replication test report"
echo ""
@@ -193,19 +211,24 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${CASE_TABLE}" ]; then
cat "${CASE_TABLE}"
echo ""
fi
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
echo ""
if [ -s "${CASE_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-s3-compat-test.yml
+26 -3
View File
@@ -105,6 +105,9 @@ jobs:
- name: Run S3 compatibility suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -150,6 +153,21 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns (the suite never reached case level, or a wholesale
# failure with zero passes) turn the workflow red.
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS S3 compatibility test report"
echo ""
@@ -162,19 +180,24 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${CASE_TABLE}" ]; then
cat "${CASE_TABLE}"
echo ""
fi
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
echo ""
if [ -s "${CASE_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-security-test.yml
+21 -6
View File
@@ -144,6 +144,8 @@ jobs:
- name: Run security suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
env:
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -184,6 +186,20 @@ jobs:
if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns turn the workflow red.
VERDICTS_TOTAL="$(grep -cE '^\[(PASS|FAIL|UNSUPPORTED)\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)"
VERDICTS_FAIL="$(grep -cE '^\[FAIL\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)"
VERDICTS_TOTAL=$(( ${VERDICTS_TOTAL:-0} + 0 )); VERDICTS_FAIL=$(( ${VERDICTS_FAIL:-0} + 0 ))
VERDICTS_PASS=$(( VERDICTS_TOTAL - VERDICTS_FAIL ))
HARNESS_OK=0
if [ "${TEST_OUTCOME}" = "success" ]; then
HARNESS_OK=1
elif [ "${TEST_OUTCOME}" = "failure" ] && [ "${VERDICTS_TOTAL}" -gt 0 ] && [ "${VERDICTS_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS security test report"
echo ""
@@ -196,16 +212,15 @@ jobs:
echo ""
# The dashboard prioritizes case rows over the step outcome.
# Keep partial case results in the artifact when the suite fails.
if [ "${RESULT}" = "success" ]; then
if [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
else
echo "The suite did not produce a non-empty report."
echo ""
fi
echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed (failing cases are tracked in rustfs/backlog)"
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-storage-test.yml
+26 -3
View File
@@ -114,6 +114,9 @@ jobs:
- name: Run storage engine suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-storage-test.sh
@@ -165,6 +168,21 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns (the suite never reached case level, or a wholesale
# failure with zero passes) turn the workflow red.
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS storage engine test report"
echo ""
@@ -177,19 +195,24 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${CASE_TABLE}" ]; then
cat "${CASE_TABLE}"
echo ""
fi
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
echo ""
if [ -s "${CASE_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
.github/workflows/rustfs-tier-test.yml
+7 -13
View File
@@ -257,6 +257,8 @@ jobs:
- name: Run tier suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
env:
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -471,26 +473,18 @@ jobs:
GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
run: |
set -euo pipefail
# Product gate: failing cases keep the run green (the structured gate
# rc and the report carry the per-case signal, and the backlog issue
# manager tracks failures). Only harness/environment breakdowns —
# evidence initialization or a missing structured gate result — turn
# the workflow red.
failed=0
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
failed=1
fi
if [ "${TEST_OUTCOME}" != "success" ]; then
echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
failed=1
fi
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
echo "structured gate result is unavailable because evidence initialization failed" >&2
elif [ ! -s "${GATE_RC_FILE}" ]; then
echo "structured gate result is missing" >&2
failed=1
else
GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
failed=1
fi
fi
[ "${failed}" -eq 0 ]
.github/workflows/rustfs-upgrade-test.yml
+28 -3
View File
@@ -160,6 +160,9 @@ jobs:
- name: Run upgrade compatibility suite
id: test
# Case failures keep the run green: the report and the backlog
# issue manager carry the product signal.
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
@@ -247,6 +250,21 @@ jobs:
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
# Product gate: failing cases keep the run green — they are reported
# below and tracked in rustfs/backlog. Only harness/environment
# breakdowns (the suite never reached case level, or a wholesale
# failure with zero passes) turn the workflow red.
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
HARNESS_OK=0
if [ '${{ steps.test.outcome }}' = 'success' ]; then
HARNESS_OK=1
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
HARNESS_OK=1
fi
{
echo "# RustFS upgrade compatibility report"
echo ""
@@ -259,21 +277,28 @@ jobs:
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
if [ "${RESULT}" = "success" ]; then
if [ -s "${MATRIX_TABLE}" ]; then
cat "${MATRIX_TABLE}"
echo ""
fi
if [ -s "${CASE_TABLE}" ]; then
cat "${CASE_TABLE}"
echo ""
fi
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
echo ""
if [ -s "${CASE_TABLE}" ]; then
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
# Red only for harness/environment breakdowns; case failures stay green.
[ "${HARNESS_OK}" = "1" ]
- name: Upload functional report to dashboard
if: ${{ always() && steps.evidence.outcome == 'success' }}
scripts/test_security_workflow.py
+21 -8
View File
@@ -170,8 +170,12 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
# The report step is red only for harness/environment breakdowns;
# the fixture log carries no case verdicts, so failure outcomes
# stay red here, and a successful suite is green unconditionally.
success = outcome == "success" and mode == "present"
self.assertEqual(report.returncode == 0, success, report.stderr)
green = outcome == "success"
self.assertEqual(report.returncode == 0, green, report.stderr)
contents = (self.artifacts / "report.md").read_text()
for expected in (
"https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
@@ -179,8 +183,8 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
):
self.assertIn(expected, contents)
self.assertEqual(CASE_ROW in contents, success)
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success)
self.assertEqual(CASE_ROW in contents, mode == "present")
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, mode == "present")
if mode == "present":
raw = (self.artifacts / "suite-report.md").read_text()
self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
@@ -404,7 +408,9 @@ class FunctionalWorkflowTests(unittest.TestCase):
steps = named_steps(job)
if suite in self.DIRECT_TESTS:
test = steps[self.DIRECT_TESTS[suite]]
self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''')
# Case failures keep the run green; the step records its
# outcome for the report and the backlog issue manager.
self.assertRegex("\n".join(test), r'''(?m)^ continue-on-error: true$''')
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
@@ -621,10 +627,14 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
success = outcome == "success" and log == good
self.assertEqual(report.returncode == 0, success, report.stderr)
# Report steps are red only for harness/environment breakdowns;
# a failure outcome with recorded case rows stays green
# (performance is unchanged and still gates on the suite result).
green = (outcome == "success" or (outcome == "failure" and log in (good, partial))) if suite != "performance" else success
self.assertEqual(report.returncode == 0, green, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertNotIn("OLD RUN EVIDENCE", contents)
self.assertEqual("| PASS |" in contents, success)
self.assertEqual("| PASS |" in contents, success if suite == "performance" else log in (good, partial))
for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
self.assertIn(value, contents)
@@ -802,11 +812,14 @@ emit_step_result() {
for failed_log in failed_logs:
with self.subTest(log=failed_log):
Path(self.env["LOG_FILE"]).write_text(failed_log)
# A failing heal run exits non-zero; the report step stays green
# because the steps ran, and the per-step table is always published.
self.context["steps.test.outcome"] = "failure"
report = self.run_step("Generate report")
self.assertNotEqual(report.returncode, 0, report.stderr)
self.assertEqual(report.returncode, 0, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertIn("Test Step Outcome: failure", contents)
self.assertNotIn("| PASS |", contents)
self.assertIn("| PASS |", contents)
if "original failure" in failed_log:
self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
if "later step failure" in failed_log: