fix(ci): report a cell that never ran a test as failed, not clean (#614)

Render summed FailCount across suites and printed "No failures across any
suite" whenever the total was zero, without asking whether any test had run.

A cell that dies during provisioning emits junit with no testcases at all, so
it took the clean path. Nightly run 30467638631 cell 20 failed with "install
callback never fired after 3 attempts; PXE never triggered install" and its
analysis.md read "No failures across any suite" — absence of evidence rendered
as positive evidence.

Zero testcases now renders a warning naming the layer above the suites. A run
with tests and no failures is unchanged.
This commit is contained in:
Josh Siviour
2026-07-31 00:15:19 +10:00
committed by GitHub
parent 0cdf591c7c
commit 545ca94a8e
2 changed files with 33 additions and 1 deletions
+11 -1
View File
@@ -467,9 +467,19 @@ func Render(r Report) string {
var b strings.Builder
fmt.Fprintf(&b, "## %s\n\n", r.Title)
totalFail := 0
totalFail, totalTests := 0, 0
for _, s := range r.Suites {
totalFail += s.FailCount
totalTests += s.Total
}
// A cell that dies before the suites run emits junit with no testcases at
// all. Reporting that as clean turns "nothing ran" into "nothing wrong", so
// name it and point above the suites.
if totalTests == 0 {
b.WriteString("⚠️ No tests ran — no testcases in any junit file. " +
"The failure is upstream of the suites (provisioning, bootstrap, or reimage); check the job log.\n")
return b.String()
}
if totalFail == 0 {
b.WriteString("✅ No failures across any suite.\n")
@@ -189,3 +189,25 @@ func TestRender_NoFailuresIsClean(t *testing.T) {
t.Errorf("expected zero-failure banner, got:\n%s", out)
}
}
// A cell that fails in provisioning writes junit with no testcases. Rendering
// the clean banner there reports "nothing ran" as "nothing wrong".
func TestRender_NoTestsRanIsNotClean(t *testing.T) {
for _, tc := range []struct {
name string
suites []SuiteReport
}{
{"empty junit file", []SuiteReport{{File: "junit-baremetal.xml", Label: "baremetal", Total: 0, FailCount: 0}}},
{"no junit files at all", nil},
} {
t.Run(tc.name, func(t *testing.T) {
out := Render(Report{Title: "E2E failure analysis", Suites: tc.suites})
if strings.Contains(out, "No failures across any suite") {
t.Errorf("clean banner rendered when no test ran, got:\n%s", out)
}
if !strings.Contains(out, "No tests ran") {
t.Errorf("expected a no-tests-ran warning, got:\n%s", out)
}
})
}
}