From 5355d9f8f8c04cb2c29a03ff6bbd116842778577 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 12:16:38 +0800 Subject: [PATCH] test(scanner): require complete ABBA summary matrix (#7462) Co-authored-by: zhi22915 --- scripts/scanner_abba.py | 16 ++++-- scripts/summarize_scanner_heal_perf.py | 30 ++++++++++- scripts/test_scanner_abba.py | 6 ++- scripts/test_summarize_scanner_heal_perf.py | 56 ++++++++++++++++++--- 4 files changed, 94 insertions(+), 14 deletions(-) diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index a9f7a092b..cdf53c564 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -182,13 +182,19 @@ def validate_release_evidence_manifest(manifest): topology = evidence.get("topology") require(isinstance(topology, dict), "missing release_evidence.topology") - nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 3) - drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 4, 4) + nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 64) + drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 1, 64) + set_size = release_evidence_integer(topology.get("erasure_set_size"), "topology.erasure_set_size", 12, 12) data = release_evidence_integer(topology.get("erasure_data_blocks"), "topology.erasure_data_blocks", 8, 8) parity = release_evidence_integer(topology.get("erasure_parity_blocks"), "topology.erasure_parity_blocks", 4, 4) - require(data + parity == nodes * drives, "release_evidence.topology must be 3x4 EC8+4") - release_evidence_integer(topology.get("pools"), "topology.pools", 2) - release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 2) + require(data + parity == set_size, "release_evidence.topology must be EC8+4") + require(nodes * drives >= set_size, "release_evidence.topology cannot host one EC8+4 set") + pools = release_evidence_integer(topology.get("pools"), "topology.pools", 1) + sets_total = release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 1) + sampled_pools = release_evidence_integer(topology.get("sampled_pools"), "topology.sampled_pools", 2) + sampled_sets = release_evidence_integer(topology.get("sampled_sets"), "topology.sampled_sets", 2) + require(sampled_pools <= pools, "release_evidence.topology sampled pools exceed total pools") + require(sampled_sets <= sets_total, "release_evidence.topology sampled sets exceed total sets") distributed = evidence.get("distributed") require(isinstance(distributed, dict), "missing release_evidence.distributed") diff --git a/scripts/summarize_scanner_heal_perf.py b/scripts/summarize_scanner_heal_perf.py index babbcf595..f11916799 100755 --- a/scripts/summarize_scanner_heal_perf.py +++ b/scripts/summarize_scanner_heal_perf.py @@ -12,7 +12,7 @@ from pathlib import Path import sys from typing import Any -from scanner_abba import validate_release_evidence_manifest +from scanner_abba import LEGS, SCENARIOS, validate_release_evidence_manifest MAX_JSON_BYTES = 1024 * 1024 CACHE_COST_PREFIX = "CACHE_COST " @@ -122,6 +122,33 @@ def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) f"comparison {index} candidate attempt cost below minimum") +def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None: + require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch") + rounds = manifest.get("rounds") + require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds") + expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS) + require( + report.get("cells") == expected_cells, + f"ABBA matrix cell count mismatch: expected {expected_cells}, got {report.get('cells')}", + ) + expected_keys = { + (scenario, comparison, round_id) + for scenario in SCENARIOS + for comparison in ("build", "background") + for round_id in range(1, rounds + 1) + } + observed_keys = [] + for index, comparison in enumerate(comparisons): + key = (comparison.get("scenario"), comparison.get("comparison"), comparison.get("round")) + require(key in expected_keys, f"comparison {index} is outside the ABBA matrix") + require(comparison.get("status") in PASS_STATES, f"comparison {index} did not pass") + observed_keys.append(key) + observed_set = set(observed_keys) + require(len(observed_keys) == len(observed_set), "duplicate ABBA matrix comparison") + missing = sorted(expected_keys - observed_set) + require(not missing, f"missing ABBA matrix comparison: {missing[0] if missing else ''}") + + def summarize_abba(abba_dir: Path) -> dict[str, Any]: manifest_path = abba_dir / "manifest.json" report_path = abba_dir / "report.json" @@ -170,6 +197,7 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: measured = report.get("evidence") == "measured" passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured if passed: + require_complete_abba_matrix(manifest, report, comparisons) validate_release_evidence_manifest({**manifest, "evidence": "measured"}) for index, comparison in enumerate(comparisons): require_measured_comparison_evidence(comparison, index) diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index 01245422d..afb93baec 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -175,6 +175,9 @@ class ScannerAbbaTest(unittest.TestCase): "drives_per_node": 4, "pools": 2, "sets_total": 2, + "sampled_pools": 2, + "sampled_sets": 2, + "erasure_set_size": 12, "erasure_data_blocks": 8, "erasure_parity_blocks": 4, }, @@ -534,7 +537,8 @@ class ScannerAbbaTest(unittest.TestCase): faults = { "missing root": lambda manifest: manifest.pop("release_evidence"), "single-set": lambda manifest: manifest["release_evidence"]["topology"].update(sets_total=1), - "wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(nodes=4), + "unsampled-set": lambda manifest: manifest["release_evidence"]["topology"].update(sampled_sets=1), + "wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(erasure_set_size=11), "duplicate endpoint": lambda manifest: manifest["release_evidence"]["distributed"].update( metrics_endpoints=["https://node-1:9000", "https://node-1:9000", "https://node-3:9000"], ), diff --git a/scripts/test_summarize_scanner_heal_perf.py b/scripts/test_summarize_scanner_heal_perf.py index 645396fb6..e62236db1 100755 --- a/scripts/test_summarize_scanner_heal_perf.py +++ b/scripts/test_summarize_scanner_heal_perf.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import contextlib import hashlib import io @@ -33,6 +34,9 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): self.abba = self.root / "abba" self.abba.mkdir() self.manifest = { + "schema": 1, + "evidence": "measured", + "rounds": 3, "fixed": { "config_sha256": "1" * 64, "dataset_sha256": "2" * 64, @@ -51,6 +55,9 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): "drives_per_node": 4, "pools": 2, "sets_total": 2, + "sampled_pools": 2, + "sampled_sets": 2, + "erasure_set_size": 12, "erasure_data_blocks": 8, "erasure_parity_blocks": 4, }, @@ -97,10 +104,20 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): "performance": "pass", "evidence": "measured", "cells": 120, - "comparisons": [self.comparison], + "comparisons": self.full_comparisons(), } self.write_inputs() + def full_comparisons(self): + comparisons = [] + for scenario in summary.SCENARIOS: + for comparison in ("build", "background"): + for round_id in range(1, 4): + row = copy.deepcopy(self.comparison) + row.update(scenario=scenario, comparison=comparison, round=round_id) + comparisons.append(row) + return comparisons + def write_inputs(self): (self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8") (self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8") @@ -184,22 +201,47 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "passing report requires comparisons"): summary.build_summary(args) + def test_passing_abba_report_requires_complete_matrix(self): + cases = { + "trimmed": lambda: self.report["comparisons"].pop(), + "duplicate": lambda: self.report["comparisons"].__setitem__(1, copy.deepcopy(self.report["comparisons"][0])), + "bad cells": lambda: self.report.update(cells=119), + "bad evidence": lambda: self.manifest.update(evidence="synthetic"), + "outside": lambda: self.report["comparisons"][0].update(round=99), + "failed comparison": lambda: self.report["comparisons"][0].update(status="inconclusive"), + } + for name, mutate in cases.items(): + with self.subTest(fault=name): + self.setUp() + mutate() + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"): + summary.build_summary(args) + def test_passing_abba_report_requires_w10_w11_evidence(self): for fault in ("missing", "pressure", "lock", "attempt", "length", "range"): with self.subTest(fault=fault): self.setUp() + target = self.report["comparisons"][0] if fault == "missing": - del self.comparison["w10_w11"] + del target["w10_w11"] elif fault == "pressure": - del self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] + del target["w10_w11"]["foreground_pressure_high_sample_ratios"] elif fault == "lock": - del self.comparison["w10_w11"]["heal_lock_wait_p99_ms"] + del target["w10_w11"]["heal_lock_wait_p99_ms"] elif fault == "attempt": - del self.comparison["w10_w11"]["attempt_cost_per_healed_object"] + del target["w10_w11"]["attempt_cost_per_healed_object"] elif fault == "length": - self.comparison["w10_w11"]["attempt_cost_per_healed_object"] = [None] + target["w10_w11"]["attempt_cost_per_healed_object"] = [None] else: - self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0] + target["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0] self.write_inputs() args = type("Args", (), { "abba_dir": self.abba,