diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index f4ed5ffa8..f7d19111c 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -204,6 +204,10 @@ The command prints only `PASS scanner_heal_perf ...` for measured passing ABBA evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance hashes; raw per-cell logs remain in the original artifact tree for audit. +Failed or interrupted ABBA reports that contain only `status`, `performance`, +`completed_cells`, and `error` also summarize as `FAIL`; they do not become +performance evidence, and a missing comparison matrix is accepted only for a +non-passing report. They cover the complete 120-cell schedule, data isolation, missing builds and oracles, zero samples/requests, swallowed request errors, offered-load drift, diff --git a/scripts/summarize_scanner_heal_perf.py b/scripts/summarize_scanner_heal_perf.py index 36f5da782..d2061309a 100755 --- a/scripts/summarize_scanner_heal_perf.py +++ b/scripts/summarize_scanner_heal_perf.py @@ -82,7 +82,14 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: report_path = abba_dir / "report.json" manifest = read_json(manifest_path) report = read_json(report_path) + report_state = report.get("status") + performance_state = report.get("performance") + require(isinstance(report_state, str) and report_state, "report.status missing") + require(isinstance(performance_state, str) and performance_state, "report.performance missing") comparisons = report.get("comparisons") + if comparisons is None: + require(report_state not in PASS_STATES, "passing report requires comparisons") + comparisons = [] require(isinstance(comparisons, list), "report.comparisons must be a list") counts = Counter() @@ -115,15 +122,13 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: if isinstance(p2, list): p2_values.extend(maybe_number(value, "p2_post_stop_work_multiple") for value in p2) - report_state = report.get("status") - performance_state = report.get("performance") - require(isinstance(report_state, str) and report_state, "report.status missing") - require(isinstance(performance_state, str) and performance_state, "report.performance missing") measured = report.get("evidence") == "measured" passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured gate_state = "pass" if passed else "fail" if report_state == "synthetic_validated": reason = "synthetic evidence validates the harness only; measured performance remains pending" + elif report_state in FAIL_STATES and isinstance(report.get("error"), str) and report["error"]: + reason = f"ABBA report status is {report_state}: {report['error']}" elif report_state not in PASS_STATES: reason = f"ABBA report status is {report_state}" elif performance_state not in PASS_STATES: @@ -142,6 +147,8 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: "performance": performance_state, "evidence": report.get("evidence"), "cells": report.get("cells", 0), + "completed_cells": report.get("completed_cells"), + "error": report.get("error"), "comparisons_total": len(comparisons), "comparison_status_counts": dict(sorted(counts.items())), "worst_p99_regression": None if not p99_regressions else float(max(p99_regressions)), @@ -247,6 +254,10 @@ def markdown(summary: dict[str, Any]) -> str: f"- worst_throughput_loss: {pct(throughput)}", f"- p2_worst_post_stop_work_multiple: {ratio(p2)}", ] + if abba.get("completed_cells") is not None: + lines.append(f"- completed_cells: {abba['completed_cells']}") + if abba.get("error"): + lines.append(f"- error: {abba['error']}") if summary.get("cache_cost") is not None: cache = summary["cache_cost"] lines.extend([ diff --git a/scripts/test_summarize_scanner_heal_perf.py b/scripts/test_summarize_scanner_heal_perf.py index ca5c084de..c3aae80fe 100755 --- a/scripts/test_summarize_scanner_heal_perf.py +++ b/scripts/test_summarize_scanner_heal_perf.py @@ -112,6 +112,42 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): self.assertEqual(result["verdict"], "FAIL") self.assertIn("synthetic evidence", result["reason"]) + def test_failed_abba_report_without_comparisons_writes_fail_closed_summary(self): + self.report = { + "status": "failed", + "performance": "pending", + "completed_cells": 7, + "error": "collector failed", + } + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + result = summary.build_summary(args) + self.assertEqual(result["verdict"], "FAIL") + self.assertEqual(result["abba"]["completed_cells"], 7) + self.assertEqual(result["abba"]["comparisons_total"], 0) + self.assertIn("collector failed", result["reason"]) + self.assertIn("- completed_cells: 7", summary.markdown(result)) + self.assertIn("- error: collector failed", summary.markdown(result)) + + def test_passing_abba_report_requires_comparisons(self): + del self.report["comparisons"] + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "passing report requires comparisons"): + summary.build_summary(args) + def test_requires_cache_profile_when_requested(self): args = type("Args", (), { "abba_dir": self.abba,