From 3fa2b334bea3e8a8643323d16624c4ea8549b34d Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 20:48:33 +0800 Subject: [PATCH] test(scanner): require two-hour measured ABBA windows (#7493) Reject measured Scanner/Heal release ABBA manifests and summaries whose evidence window is shorter than the W21 two-hour release requirement. Co-authored-by: zhi22915 --- docs/operations/scanner-benchmark-runbook.md | 6 +++--- scripts/scanner_abba.py | 3 ++- scripts/summarize_scanner_heal_perf.py | 10 +++++++++- scripts/test_scanner_abba.py | 4 ++-- scripts/test_summarize_scanner_heal_perf.py | 14 ++++++++++++++ 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 68f281f2e..a913440a8 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -39,8 +39,8 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/ The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`, `multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least three A1/B1/B2/A2 groups for both baseline/candidate with background work on, -and candidate-only background off/on. A measured leg lasts at least 900 -seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles). +and candidate-only background off/on. A measured release leg lasts at least 7200 +seconds; the minimum matrix contains 120 legs (240 hours before setup/oracles). The existing `performance-ab.yml` supplies the pattern for immutable build provenance and failure propagation, but its short Warp workload is not this scanner gate. No scheduled workflow starts this matrix automatically. @@ -63,7 +63,7 @@ The manifest has the following JSON contract (all fields are required): | Field | Value | |---|---| | `schema`, `evidence` | `1`, and `measured` or `synthetic`. | -| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. | +| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 7200..86400 seconds for measured release runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. | | `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. | | `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. | | `release_evidence` | Required for `measured` runs. It binds the 3x4 EC8+4 topology, multi-pool/multi-set coverage, per-node metrics endpoints, same-window distributed sampling, process restart and crash-restart fault modes, mixed-version reader/writer/rollback participation, and allocation/flamegraph/RSS/save-frequency profile artifact requirements. Synthetic runs do not need this field and still cannot approve release evidence. | diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index 6bb594f9b..ab3d188d4 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -35,6 +35,7 @@ RELEASE_PROFILE_ARTIFACTS = ( "rss-samples", "save-frequency", ) +MIN_MEASURED_RELEASE_DURATION_SECONDS = 7200 RELEASE_FAULT_MODES = ( "process-restart", "process-crash-restart", @@ -130,7 +131,7 @@ def validate_manifest(manifest): number(fixed.get("offered_load_ops"), "offered load", 1) require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10, "rounds must be 3..10") - minimum = 900 if manifest["evidence"] == "measured" else 1 + minimum = MIN_MEASURED_RELEASE_DURATION_SECONDS if manifest["evidence"] == "measured" else 1 require(type(manifest.get("duration_seconds")) is int and minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds") number(manifest.get("min_free_bytes"), "min_free_bytes", 1) diff --git a/scripts/summarize_scanner_heal_perf.py b/scripts/summarize_scanner_heal_perf.py index f11916799..4fa586212 100755 --- a/scripts/summarize_scanner_heal_perf.py +++ b/scripts/summarize_scanner_heal_perf.py @@ -12,7 +12,12 @@ from pathlib import Path import sys from typing import Any -from scanner_abba import LEGS, SCENARIOS, validate_release_evidence_manifest +from scanner_abba import ( + LEGS, + MIN_MEASURED_RELEASE_DURATION_SECONDS, + SCENARIOS, + validate_release_evidence_manifest, +) MAX_JSON_BYTES = 1024 * 1024 CACHE_COST_PREFIX = "CACHE_COST " @@ -124,6 +129,9 @@ def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None: require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch") + require(type(manifest.get("duration_seconds")) is int and + manifest["duration_seconds"] >= MIN_MEASURED_RELEASE_DURATION_SECONDS, + "measured ABBA duration_seconds requires at least two hours") rounds = manifest.get("rounds") require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds") expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS) diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index 116528048..6c2ef7491 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -167,7 +167,7 @@ class ScannerAbbaTest(unittest.TestCase): def measured_manifest(self): manifest = copy.deepcopy(self.manifest) - manifest.update(evidence="measured", duration_seconds=900) + manifest.update(evidence="measured", duration_seconds=harness.MIN_MEASURED_RELEASE_DURATION_SECONDS) candidate_binary = self.root / "candidate-python" candidate_binary.write_bytes(self.binary.read_bytes() + b"\n") candidate_binary.chmod(0o755) @@ -534,7 +534,7 @@ class ScannerAbbaTest(unittest.TestCase): self.manifest["evidence"] = "measured" with self.assertRaisesRegex(ValueError, "duration_seconds"): harness.validate_manifest(self.manifest) - self.manifest["duration_seconds"] = 900 + self.manifest["duration_seconds"] = harness.MIN_MEASURED_RELEASE_DURATION_SECONDS self.manifest["rounds"] = 2 with self.assertRaisesRegex(ValueError, "rounds"): harness.validate_manifest(self.manifest) diff --git a/scripts/test_summarize_scanner_heal_perf.py b/scripts/test_summarize_scanner_heal_perf.py index e62236db1..24157d0cf 100755 --- a/scripts/test_summarize_scanner_heal_perf.py +++ b/scripts/test_summarize_scanner_heal_perf.py @@ -37,6 +37,7 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): "schema": 1, "evidence": "measured", "rounds": 3, + "duration_seconds": summary.MIN_MEASURED_RELEASE_DURATION_SECONDS, "fixed": { "config_sha256": "1" * 64, "dataset_sha256": "2" * 64, @@ -225,6 +226,19 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"): summary.build_summary(args) + def test_passing_measured_report_requires_two_hour_window(self): + self.manifest["duration_seconds"] = summary.MIN_MEASURED_RELEASE_DURATION_SECONDS - 1 + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "two hours"): + summary.build_summary(args) + def test_passing_abba_report_requires_w10_w11_evidence(self): for fault in ("missing", "pressure", "lock", "attempt", "length", "range"): with self.subTest(fault=fault):