From 7f4bf4f8747204546f4ad0ab905c296e69c242e4 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 19:02:16 +0800 Subject: [PATCH] test(scanner): harden ABBA threshold evaluation Handle exact threshold comparisons without binary floating point boundary drift and mark unstable P1 walk controls inconclusive. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --- scripts/scanner_abba.py | 73 ++++++++++++++++++++++++++++-------- scripts/test_scanner_abba.py | 26 +++++++++++++ 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index 9c461dc7b..950228304 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -2,6 +2,7 @@ """Run isolated scanner/heal ABBA cells through a deployment-specific adapter.""" import argparse +from decimal import Decimal import hashlib import json import math @@ -21,6 +22,8 @@ METRICS = ( "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds", "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests", ) +REPEATABILITY_LIMIT = Decimal("0.05") +P2_WORK_MULTIPLE_LIMIT = Decimal("1.2") def require(condition, message): @@ -34,6 +37,38 @@ def number(value, name, minimum=0): return value +def decimal_number(value, name, minimum=0): + if isinstance(value, Decimal): + require(value.is_finite() and value >= Decimal(str(minimum)), f"invalid {name}") + return value + number(value, name, minimum) + return Decimal(str(value)) + + +def ratio(numerator, denominator, name): + denominator = decimal_number(denominator, f"{name} denominator") + require(denominator > 0, f"invalid {name} denominator") + return decimal_number(numerator, name) / denominator + + +def relative_change(current, baseline, name): + return ratio(current, baseline, name) - Decimal("1") + + +def repeatability_change(first, second, name): + first = decimal_number(first, name) + second = decimal_number(second, name) + if first == 0 and second == 0: + return Decimal("0") + if first == 0 or second == 0: + return Decimal("Infinity") + return abs(second / first - Decimal("1")) + + +def report_number(value): + return None if value.is_infinite() else float(value) + + def digest(path): with Path(path).open("rb") as stream: return hashlib.file_digest(stream, "sha256").hexdigest() @@ -165,37 +200,43 @@ def evaluate(cells): require([cell["leg"] for cell in group] == list(LEGS), "incomplete ABBA group") a1, b1, b2, a2 = (cell["result"]["metrics"] for cell in group) control = group[0]["comparison"] == "background" - drift = max(abs(a2[k] / a1[k] - 1) for k in ("p99_ms", "throughput_ops")) - repeat_drift = max(abs(b2[k] / b1[k] - 1) for k in ("p99_ms", "throughput_ops")) - noise = max(drift, repeat_drift) > 0.05 - a = {key: (a1[key] + a2[key]) / 2 for key in METRICS} - b = {key: (b1[key] + b2[key]) / 2 for key in METRICS} - p99 = b["p99_ms"] / a["p99_ms"] - 1 - throughput = b["throughput_ops"] / a["throughput_ops"] - 1 - thresholds = {"p99_regression": 0.10 if control else 0.05, - "throughput_loss": 0.05 if control else 0.03} + drift = max(abs(relative_change(a2[k], a1[k], k)) for k in ("p99_ms", "throughput_ops")) + repeat_drift = max(abs(relative_change(b2[k], b1[k], k)) for k in ("p99_ms", "throughput_ops")) + noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT + a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS} + b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS} + p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms") + throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops") + thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"), + "throughput_loss": Decimal("0.05") if control else Decimal("0.03")} passed = p99 <= thresholds["p99_regression"] and throughput >= -thresholds["throughput_loss"] p1 = None + work_drift = None if not control: if group[0]["scenario"] == "cold-hot": require(a["cold_walk_objects"] > 0, "cold-hot baseline has no cold walk samples") - required = a["cold_walk_objects"] / a["walk_objects"] * 0.80 - reduction = 1 - b["walk_objects"] / a["walk_objects"] - p1 = {"required_reduction": required, "observed_reduction": reduction} + work_drift = max(repeatability_change(a1[key], a2[key], key) for key in ("walk_objects", "cold_walk_objects")) + work_drift = max(work_drift, *(repeatability_change(b1[key], b2[key], key) for key in ("walk_objects", "cold_walk_objects"))) + noise |= work_drift > REPEATABILITY_LIMIT + required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80") + reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction") + p1 = {"required_reduction": float(required), "observed_reduction": float(reduction), + "repeatability_drift": report_number(work_drift)} if group[0]["scenario"] == "cold-hot": passed &= reduction >= required p2 = [convergence(cell["result"]) if cell["background"] == "on" else None for cell in group] candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")] p2_pending = any(value is None for value in candidate_p2) - passed &= all(value <= 1.2 for value in candidate_p2 if value is not None) + passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None) inconclusive |= noise or p2_pending if not noise and not passed: failed = True comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"], "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"), - "a2_a1_drift": drift, "b2_b1_drift": repeat_drift, - "p99_regression": p99, "throughput_change": throughput, - "thresholds": thresholds, "p1": p1, "p2_max_work_multiple": 1.2, + "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift), + "p99_regression": float(p99), "throughput_change": float(throughput), + "thresholds": {key: float(value) for key, value in thresholds.items()}, + "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT), "p2_post_stop_work_multiples": p2}) return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index 30fc00839..820e203c6 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -63,8 +63,19 @@ def fake_adapter(): result["convergence"]["walk_objects"] = 121 elif fault == "latency-regression" and request["leg"].startswith("B"): result["metrics"]["p99_ms"] = 12 + elif fault == "exact-thresholds" and request["leg"].startswith("B"): + result["metrics"].update(p99_ms=10.5, throughput_ops=97) + elif fault == "just-over-threshold" and request["comparison"] == "build" and request["leg"].startswith("B"): + result["metrics"]["p99_ms"] = 10.500001 elif fault == "p1-regression" and not baseline: result["metrics"]["walk_objects"] = 30 + elif fault == "unstable-p1-control" and request["comparison"] == "build": + if request["leg"] == "A1": + result["metrics"].update(walk_objects=1000, cold_walk_objects=1000) + elif request["leg"] == "A2": + result["metrics"].update(walk_objects=10, cold_walk_objects=10) + elif request["leg"].startswith("B"): + result["metrics"].update(walk_objects=100, cold_walk_objects=0) elif fault == "missing-metric": del result["metrics"]["save_bytes"] elif fault == "incomplete-repair": @@ -136,6 +147,21 @@ class ScannerAbbaTest(unittest.TestCase): with patch.object(harness, "SCENARIOS", ("cold-hot",)): self.assertEqual(self.run_harness(fault), 1) + def test_exact_threshold_boundaries_pass(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("exact-thresholds"), 0) + + def test_just_over_threshold_fails(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("just-over-threshold"), 1) + + def test_unstable_p1_work_control_is_inconclusive(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("unstable-p1-control"), 3) + comparison = harness.read_json(self.root / "out/report.json")["comparisons"][0] + self.assertEqual(comparison["status"], "inconclusive") + self.assertGreater(comparison["p1"]["repeatability_drift"], 0.05) + def test_manifest_rejects_missing_build_or_oracle(self): for section, key in (("baseline", "binary"), ("oracles", "cold-hot")): manifest = copy.deepcopy(self.manifest)