fix(test): preserve inclusive ABBA thresholds

Use decimal boundary comparisons for ABBA ratio checks and cover exact documented p99, throughput, and P1 limits.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

scripts/scanner_abba.py

@@ -2,6 +2,7 @@
 """Run isolated scanner/heal ABBA cells through a deployment-specific adapter."""
 
 import argparse
+from decimal import Decimal
 import hashlib
 import json
 import math
@@ -22,6 +23,8 @@ METRICS = (
     "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds",
     "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests",
 )
+REPEATABILITY_LIMIT = Decimal("0.05")
+P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
 
 
 def require(condition, message):
@@ -35,6 +38,38 @@ def number(value, name, minimum=0):
     return value
 
 
+def decimal_number(value, name, minimum=0):
+    if isinstance(value, Decimal):
+        require(value.is_finite() and value >= Decimal(str(minimum)), f"invalid {name}")
+        return value
+    number(value, name, minimum)
+    return Decimal(str(value))
+
+
+def ratio(numerator, denominator, name):
+    denominator = decimal_number(denominator, f"{name} denominator")
+    require(denominator > 0, f"invalid {name} denominator")
+    return decimal_number(numerator, name) / denominator
+
+
+def relative_change(current, baseline, name):
+    return ratio(current, baseline, name) - Decimal("1")
+
+
+def repeatability_change(first, second, name):
+    first = decimal_number(first, name)
+    second = decimal_number(second, name)
+    if first == 0 and second == 0:
+        return Decimal("0")
+    if first == 0 or second == 0:
+        return Decimal("Infinity")
+    return abs(second / first - Decimal("1"))
+
+
+def report_number(value):
+    return None if value.is_infinite() else float(value)
+
+
 def digest(path):
     with Path(path).open("rb") as stream:
         return hashlib.file_digest(stream, "sha256").hexdigest()
@@ -236,7 +271,7 @@ def convergence(result):
     require(window["full_walk_objects"] > 0, "zero full walk reference")
     require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"],
             "invalid convergence budget window")
-    return window["walk_objects"] / window["full_walk_objects"]
+    return ratio(window["walk_objects"], window["full_walk_objects"], "convergence work")
 
 
 def evaluate(cells):
@@ -248,38 +283,46 @@ def evaluate(cells):
         require([cell["leg"] for cell in group] == list(LEGS), "incomplete ABBA group")
         a1, b1, b2, a2 = (cell["result"]["metrics"] for cell in group)
         control = group[0]["comparison"] == "background"
-        drift = max(abs(a2[k] / a1[k] - 1) for k in ("p99_ms", "throughput_ops"))
-        repeat_drift = max(abs(b2[k] / b1[k] - 1) for k in ("p99_ms", "throughput_ops"))
-        noise = max(drift, repeat_drift) > 0.05
-        a = {key: (a1[key] + a2[key]) / 2 for key in METRICS}
-        b = {key: (b1[key] + b2[key]) / 2 for key in METRICS}
-        p99 = b["p99_ms"] / a["p99_ms"] - 1
-        throughput = b["throughput_ops"] / a["throughput_ops"] - 1
-        thresholds = {"p99_regression": 0.10 if control else 0.05,
-                      "throughput_loss": 0.05 if control else 0.03}
+        drift = max(abs(relative_change(a2[k], a1[k], k)) for k in ("p99_ms", "throughput_ops"))
+        repeat_drift = max(abs(relative_change(b2[k], b1[k], k)) for k in ("p99_ms", "throughput_ops"))
+        noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT
+        a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS}
+        b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS}
+        p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms")
+        throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops")
+        thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"),
+                      "throughput_loss": Decimal("0.05") if control else Decimal("0.03")}
         passed = p99 <= thresholds["p99_regression"] and throughput >= -thresholds["throughput_loss"]
         p1 = None
+        work_drift = None
         if not control:
             if group[0]["scenario"] == "cold-hot":
                 require(a["cold_walk_objects"] > 0, "cold-hot baseline has no cold walk samples")
-            required = a["cold_walk_objects"] / a["walk_objects"] * 0.80
-            reduction = 1 - b["walk_objects"] / a["walk_objects"]
-            p1 = {"required_reduction": required, "observed_reduction": reduction}
+            work_drift = max(repeatability_change(a1[key], a2[key], key) for key in ("walk_objects", "cold_walk_objects"))
+            work_drift = max(work_drift, *(repeatability_change(b1[key], b2[key], key) for key in ("walk_objects", "cold_walk_objects")))
+            noise |= work_drift > REPEATABILITY_LIMIT
+            required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80")
+            reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction")
+            p1 = {"required_reduction": float(required), "observed_reduction": float(reduction),
+                  "repeatability_drift": report_number(work_drift)}
             if group[0]["scenario"] == "cold-hot":
-                passed &= reduction >= required
+                # Compare counts before division can round repeating decimal ratios.
+                passed &= a["walk_objects"] - b["walk_objects"] >= a["cold_walk_objects"] * Decimal("0.80")
         p2 = [convergence(cell["result"]) if cell["background"] == "on" else None for cell in group]
         candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")]
         p2_pending = any(value is None for value in candidate_p2)
-        passed &= all(value <= 1.2 for value in candidate_p2 if value is not None)
+        passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None)
+        p2_report = [None if value is None else float(value) for value in p2]
         inconclusive |= noise or p2_pending
         if not noise and not passed:
             failed = True
         comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"],
                             "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"),
-                            "a2_a1_drift": drift, "b2_b1_drift": repeat_drift,
-                            "p99_regression": p99, "throughput_change": throughput,
-                            "thresholds": thresholds, "p1": p1, "p2_max_work_multiple": 1.2,
-                            "p2_post_stop_work_multiples": p2})
+                            "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift),
+                            "p99_regression": float(p99), "throughput_change": float(throughput),
+                            "thresholds": {key: float(value) for key, value in thresholds.items()},
+                            "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT),
+                            "p2_post_stop_work_multiples": p2_report})
     return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons
 
 

scripts/test_scanner_abba.py

@@ -76,8 +76,15 @@ def fake_adapter():
             result["convergence"]["walk_objects"] = 121
         elif fault == "latency-regression" and request["leg"].startswith("B"):
             result["metrics"]["p99_ms"] = 12
+        elif fault == "exact-thresholds" and request["leg"].startswith("B"):
+            result["metrics"].update(p99_ms=10.5, throughput_ops=97)
+        elif fault == "just-over-threshold" and request["comparison"] == "build" and request["leg"].startswith("B"):
+            result["metrics"]["p99_ms"] = 10.500001
         elif fault == "p1-regression" and not baseline:
             result["metrics"]["walk_objects"] = 30
+        elif fault in ("p1-exact-fraction", "p1-over-fraction"):
+            result["metrics"].update(walk_objects=9 if baseline else 5 + (fault == "p1-over-fraction"),
+                                     cold_walk_objects=5 if baseline else 0)
         elif fault == "missing-metric":
             del result["metrics"]["save_bytes"]
         elif fault == "incomplete-repair":
@@ -285,6 +292,21 @@ class ScannerAbbaTest(unittest.TestCase):
                 with patch.object(harness, "SCENARIOS", ("cold-hot",)):
                     self.assertEqual(self.run_harness(fault), 1)
 
+    def test_exact_threshold_boundaries_pass(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("exact-thresholds"), 0)
+
+    def test_just_over_threshold_fails(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("just-over-threshold"), 1)
+
+    def test_p1_fractional_boundary(self):
+        for fault, expected in (("p1-exact-fraction", 0), ("p1-over-fraction", 1)):
+            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
+                self.root = Path(directory)
+                with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+                    self.assertEqual(self.run_harness(fault), expected)
+
     def test_manifest_rejects_missing_build_or_oracle(self):
         for section, key in (("baseline", "binary"), ("oracles", "cold-hot")):
             manifest = copy.deepcopy(self.manifest)