From 703086d71da230579cd3ac3b6e8392a62ceaeaa9 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 10:14:41 +0800 Subject: [PATCH] test(scanner): require perf summary evidence fields (#7453) Fail closed when measured passing Scanner/Heal ABBA summaries omit W10/W11 foreground pressure, lock wait, or attempt-cost evidence. Co-authored-by: zhi22915 --- docs/operations/scanner-benchmark-runbook.md | 4 ++ scripts/summarize_scanner_heal_perf.py | 46 ++++++++++++++++++++ scripts/test_summarize_scanner_heal_perf.py | 33 ++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index f7d19111c..375e509b8 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -208,6 +208,10 @@ Failed or interrupted ABBA reports that contain only `status`, `performance`, `completed_cells`, and `error` also summarize as `FAIL`; they do not become performance evidence, and a missing comparison matrix is accepted only for a non-passing report. +Measured passing reports must also retain the W10/W11 foreground-pressure, +heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If +those fields are removed, empty, malformed, or length-mismatched, the quiet +summary fails closed instead of treating the report as performance evidence. They cover the complete 120-cell schedule, data isolation, missing builds and oracles, zero samples/requests, swallowed request errors, offered-load drift, diff --git a/scripts/summarize_scanner_heal_perf.py b/scripts/summarize_scanner_heal_perf.py index d2061309a..d7f594b67 100755 --- a/scripts/summarize_scanner_heal_perf.py +++ b/scripts/summarize_scanner_heal_perf.py @@ -77,6 +77,49 @@ def max_decimal(values: list[Decimal | None]) -> Decimal | None: return max(present) +def require_metric_series(value: Any, name: str, minimum: Decimal | None = None, + maximum: Decimal | None = None) -> list[Decimal | None]: + require(isinstance(value, list) and value, f"missing performance evidence field: {name}") + parsed = [maybe_number(item, name) for item in value] + for item in parsed: + if item is None: + continue + if minimum is not None: + require(item >= minimum, f"{name} below minimum") + if maximum is not None: + require(item <= maximum, f"{name} above maximum") + return parsed + + +def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) -> None: + w10_w11 = comparison.get("w10_w11") + require(isinstance(w10_w11, dict), f"comparison {index} missing W10/W11 evidence") + pressure = require_metric_series( + w10_w11.get("foreground_pressure_high_sample_ratios"), + f"comparison {index} foreground_pressure_high_sample_ratios", + Decimal("0"), + Decimal("1"), + ) + lock_wait = require_metric_series( + w10_w11.get("heal_lock_wait_p99_ms"), + f"comparison {index} heal_lock_wait_p99_ms", + Decimal("0"), + ) + attempt_cost = require_metric_series( + w10_w11.get("attempt_cost_per_healed_object"), + f"comparison {index} attempt_cost_per_healed_object", + Decimal("0"), + ) + require(len(pressure) == len(lock_wait) == len(attempt_cost), + f"comparison {index} W10/W11 evidence length mismatch") + candidate_attempt_cost = maybe_number( + w10_w11.get("candidate_attempt_cost_per_healed_object"), + f"comparison {index} candidate_attempt_cost_per_healed_object", + ) + require(candidate_attempt_cost is None or candidate_attempt_cost >= 0, + f"comparison {index} candidate attempt cost below minimum") + + def summarize_abba(abba_dir: Path) -> dict[str, Any]: manifest_path = abba_dir / "manifest.json" report_path = abba_dir / "report.json" @@ -124,6 +167,9 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: measured = report.get("evidence") == "measured" passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured + if passed: + for index, comparison in enumerate(comparisons): + require_measured_comparison_evidence(comparison, index) gate_state = "pass" if passed else "fail" if report_state == "synthetic_validated": reason = "synthetic evidence validates the harness only; measured performance remains pending" diff --git a/scripts/test_summarize_scanner_heal_perf.py b/scripts/test_summarize_scanner_heal_perf.py index c3aae80fe..8c8650c15 100755 --- a/scripts/test_summarize_scanner_heal_perf.py +++ b/scripts/test_summarize_scanner_heal_perf.py @@ -55,6 +55,12 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): "throughput_change": -0.01, "p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01}, "p2_post_stop_work_multiples": [None, 1.1, 1.0, None], + "w10_w11": { + "foreground_pressure_high_sample_ratios": [0.0, 0.25, 0.25, 0.0], + "heal_lock_wait_p99_ms": [12.0, 8.0, 9.0, 13.0], + "attempt_cost_per_healed_object": [None, 1.2, 1.3, None], + "candidate_attempt_cost_per_healed_object": 1.3, + }, } self.report = { "status": "pass", @@ -148,6 +154,33 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "passing report requires comparisons"): summary.build_summary(args) + def test_passing_abba_report_requires_w10_w11_evidence(self): + for fault in ("missing", "pressure", "lock", "attempt", "length", "range"): + with self.subTest(fault=fault): + self.setUp() + if fault == "missing": + del self.comparison["w10_w11"] + elif fault == "pressure": + del self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] + elif fault == "lock": + del self.comparison["w10_w11"]["heal_lock_wait_p99_ms"] + elif fault == "attempt": + del self.comparison["w10_w11"]["attempt_cost_per_healed_object"] + elif fault == "length": + self.comparison["w10_w11"]["attempt_cost_per_healed_object"] = [None] + else: + self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0] + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "W10/W11|performance evidence|length mismatch|above maximum"): + summary.build_summary(args) + def test_requires_cache_profile_when_requested(self): args = type("Args", (), { "abba_dir": self.abba,