gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-22 02:23:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: green runs for case failures; red only for harness breakdowns (#7698)

Browse Source 
A failing product case used to turn the whole workflow red, so the run
conclusion carried no signal beyond 'something failed' and the report
was suppressed. New semantics across the functional suites:

- Suite steps run with continue-on-error: the outcome is still recorded
  for the report and the backlog issue manager (security/tier already
  carried the flag).
- Generate report always publishes the full per-case table plus a
  'Product result: N passed, M failed' summary, and its exit gate is
  harness health: red only when the suite never reached case level (no
  case verdicts), failed wholesale (zero passes, >=3 failures), or was
  cancelled/skipped. performance is unchanged (parked).
- tier's structured gate no longer fails on case failures; it keeps red
  for evidence-init and missing-gate-result breakdowns.
- pool/performance keep their existing red sources (install/benchmark).

Workflow contract tests updated to the new exit semantics: the security
report matrix keys green off the suite outcome, the evidence matrix
expects green for failure outcomes with recorded case rows (except
performance), the heal staged-rerun block expects the per-step table to
always publish, and run steps are now required to carry
continue-on-error.

Verified locally: actionlint clean; test_security_workflow.py 21/21.
This commit is contained in:
hector
2026-09-12 17:33:29 +08:00
committed by GitHub
parent f90d4071a5
commit bcf5184a85
11 changed files with 226 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test_security_workflow.py
+21 -8
View File
@@ -170,8 +170,12 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
# The report step is red only for harness/environment breakdowns;
# the fixture log carries no case verdicts, so failure outcomes
# stay red here, and a successful suite is green unconditionally.
success = outcome == "success" and mode == "present"
self.assertEqual(report.returncode == 0, success, report.stderr)
green = outcome == "success"
self.assertEqual(report.returncode == 0, green, report.stderr)
contents = (self.artifacts / "report.md").read_text()
for expected in (
"https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
@@ -179,8 +183,8 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
):
self.assertIn(expected, contents)
self.assertEqual(CASE_ROW in contents, success)
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success)
self.assertEqual(CASE_ROW in contents, mode == "present")
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, mode == "present")
if mode == "present":
raw = (self.artifacts / "suite-report.md").read_text()
self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
@@ -404,7 +408,9 @@ class FunctionalWorkflowTests(unittest.TestCase):
steps = named_steps(job)
if suite in self.DIRECT_TESTS:
test = steps[self.DIRECT_TESTS[suite]]
self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''')
# Case failures keep the run green; the step records its
# outcome for the report and the backlog issue manager.
self.assertRegex("\n".join(test), r'''(?m)^ continue-on-error: true$''')
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
@@ -621,10 +627,14 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
success = outcome == "success" and log == good
self.assertEqual(report.returncode == 0, success, report.stderr)
# Report steps are red only for harness/environment breakdowns;
# a failure outcome with recorded case rows stays green
# (performance is unchanged and still gates on the suite result).
green = (outcome == "success" or (outcome == "failure" and log in (good, partial))) if suite != "performance" else success
self.assertEqual(report.returncode == 0, green, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertNotIn("OLD RUN EVIDENCE", contents)
self.assertEqual("| PASS |" in contents, success)
self.assertEqual("| PASS |" in contents, success if suite == "performance" else log in (good, partial))
for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
self.assertIn(value, contents)
@@ -802,11 +812,14 @@ emit_step_result() {
for failed_log in failed_logs:
with self.subTest(log=failed_log):
Path(self.env["LOG_FILE"]).write_text(failed_log)
# A failing heal run exits non-zero; the report step stays green
# because the steps ran, and the per-step table is always published.
self.context["steps.test.outcome"] = "failure"
report = self.run_step("Generate report")
self.assertNotEqual(report.returncode, 0, report.stderr)
self.assertEqual(report.returncode, 0, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertIn("Test Step Outcome: failure", contents)
self.assertNotIn("| PASS |", contents)
self.assertIn("| PASS |", contents)
if "original failure" in failed_log:
self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
if "later step failure" in failed_log: