test(scanner): require perf summary evidence fields (#7453)

Fail closed when measured passing Scanner/Heal ABBA summaries omit W10/W11 foreground pressure, lock wait, or attempt-cost evidence.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

docs/operations/scanner-benchmark-runbook.md

@@ -208,6 +208,10 @@ Failed or interrupted ABBA reports that contain only `status`, `performance`,
 `completed_cells`, and `error` also summarize as `FAIL`; they do not become
 performance evidence, and a missing comparison matrix is accepted only for a
 non-passing report.
+Measured passing reports must also retain the W10/W11 foreground-pressure,
+heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If
+those fields are removed, empty, malformed, or length-mismatched, the quiet
+summary fails closed instead of treating the report as performance evidence.
 
 They cover the complete 120-cell schedule, data isolation, missing builds and
 oracles, zero samples/requests, swallowed request errors, offered-load drift,

scripts/summarize_scanner_heal_perf.py

@@ -77,6 +77,49 @@ def max_decimal(values: list[Decimal | None]) -> Decimal | None:
     return max(present)
 
 
+def require_metric_series(value: Any, name: str, minimum: Decimal | None = None,
+                          maximum: Decimal | None = None) -> list[Decimal | None]:
+    require(isinstance(value, list) and value, f"missing performance evidence field: {name}")
+    parsed = [maybe_number(item, name) for item in value]
+    for item in parsed:
+        if item is None:
+            continue
+        if minimum is not None:
+            require(item >= minimum, f"{name} below minimum")
+        if maximum is not None:
+            require(item <= maximum, f"{name} above maximum")
+    return parsed
+
+
+def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) -> None:
+    w10_w11 = comparison.get("w10_w11")
+    require(isinstance(w10_w11, dict), f"comparison {index} missing W10/W11 evidence")
+    pressure = require_metric_series(
+        w10_w11.get("foreground_pressure_high_sample_ratios"),
+        f"comparison {index} foreground_pressure_high_sample_ratios",
+        Decimal("0"),
+        Decimal("1"),
+    )
+    lock_wait = require_metric_series(
+        w10_w11.get("heal_lock_wait_p99_ms"),
+        f"comparison {index} heal_lock_wait_p99_ms",
+        Decimal("0"),
+    )
+    attempt_cost = require_metric_series(
+        w10_w11.get("attempt_cost_per_healed_object"),
+        f"comparison {index} attempt_cost_per_healed_object",
+        Decimal("0"),
+    )
+    require(len(pressure) == len(lock_wait) == len(attempt_cost),
+            f"comparison {index} W10/W11 evidence length mismatch")
+    candidate_attempt_cost = maybe_number(
+        w10_w11.get("candidate_attempt_cost_per_healed_object"),
+        f"comparison {index} candidate_attempt_cost_per_healed_object",
+    )
+    require(candidate_attempt_cost is None or candidate_attempt_cost >= 0,
+            f"comparison {index} candidate attempt cost below minimum")
+
+
 def summarize_abba(abba_dir: Path) -> dict[str, Any]:
     manifest_path = abba_dir / "manifest.json"
     report_path = abba_dir / "report.json"
@@ -124,6 +167,9 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
 
     measured = report.get("evidence") == "measured"
     passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured
+    if passed:
+        for index, comparison in enumerate(comparisons):
+            require_measured_comparison_evidence(comparison, index)
     gate_state = "pass" if passed else "fail"
     if report_state == "synthetic_validated":
         reason = "synthetic evidence validates the harness only; measured performance remains pending"

scripts/test_summarize_scanner_heal_perf.py

@@ -55,6 +55,12 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
             "throughput_change": -0.01,
             "p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01},
             "p2_post_stop_work_multiples": [None, 1.1, 1.0, None],
+            "w10_w11": {
+                "foreground_pressure_high_sample_ratios": [0.0, 0.25, 0.25, 0.0],
+                "heal_lock_wait_p99_ms": [12.0, 8.0, 9.0, 13.0],
+                "attempt_cost_per_healed_object": [None, 1.2, 1.3, None],
+                "candidate_attempt_cost_per_healed_object": 1.3,
+            },
         }
         self.report = {
             "status": "pass",
@@ -148,6 +154,33 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
         with self.assertRaisesRegex(ValueError, "passing report requires comparisons"):
             summary.build_summary(args)
 
+    def test_passing_abba_report_requires_w10_w11_evidence(self):
+        for fault in ("missing", "pressure", "lock", "attempt", "length", "range"):
+            with self.subTest(fault=fault):
+                self.setUp()
+                if fault == "missing":
+                    del self.comparison["w10_w11"]
+                elif fault == "pressure":
+                    del self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"]
+                elif fault == "lock":
+                    del self.comparison["w10_w11"]["heal_lock_wait_p99_ms"]
+                elif fault == "attempt":
+                    del self.comparison["w10_w11"]["attempt_cost_per_healed_object"]
+                elif fault == "length":
+                    self.comparison["w10_w11"]["attempt_cost_per_healed_object"] = [None]
+                else:
+                    self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0]
+                self.write_inputs()
+                args = type("Args", (), {
+                    "abba_dir": self.abba,
+                    "cache_cost_log": None,
+                    "require_cache_cost": False,
+                    "json_out": None,
+                    "markdown_out": None,
+                })
+                with self.assertRaisesRegex(ValueError, "W10/W11|performance evidence|length mismatch|above maximum"):
+                    summary.build_summary(args)
+
     def test_requires_cache_profile_when_requested(self):
         args = type("Args", (), {
             "abba_dir": self.abba,