gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-08 21:25:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(scanner): require complete ABBA summary matrix (#7462)

Browse Source 
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-08 12:16:38 +08:00
committed by GitHub
parent 50d7a049ee
commit 5355d9f8f8
4 changed files with 94 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/scanner_abba.py
+11 -5
View File
@@ -182,13 +182,19 @@ def validate_release_evidence_manifest(manifest):
topology = evidence.get("topology")
require(isinstance(topology, dict), "missing release_evidence.topology")
nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 3)
drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 4, 4)
nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 64)
drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 1, 64)
set_size = release_evidence_integer(topology.get("erasure_set_size"), "topology.erasure_set_size", 12, 12)
data = release_evidence_integer(topology.get("erasure_data_blocks"), "topology.erasure_data_blocks", 8, 8)
parity = release_evidence_integer(topology.get("erasure_parity_blocks"), "topology.erasure_parity_blocks", 4, 4)
require(data + parity == nodes * drives, "release_evidence.topology must be 3x4 EC8+4")
release_evidence_integer(topology.get("pools"), "topology.pools", 2)
release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 2)
require(data + parity == set_size, "release_evidence.topology must be EC8+4")
require(nodes * drives >= set_size, "release_evidence.topology cannot host one EC8+4 set")
pools = release_evidence_integer(topology.get("pools"), "topology.pools", 1)
sets_total = release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 1)
sampled_pools = release_evidence_integer(topology.get("sampled_pools"), "topology.sampled_pools", 2)
sampled_sets = release_evidence_integer(topology.get("sampled_sets"), "topology.sampled_sets", 2)
require(sampled_pools <= pools, "release_evidence.topology sampled pools exceed total pools")
require(sampled_sets <= sets_total, "release_evidence.topology sampled sets exceed total sets")
distributed = evidence.get("distributed")
require(isinstance(distributed, dict), "missing release_evidence.distributed")
scripts/summarize_scanner_heal_perf.py
+29 -1
View File
@@ -12,7 +12,7 @@ from pathlib import Path
import sys
from typing import Any
from scanner_abba import validate_release_evidence_manifest
from scanner_abba import LEGS, SCENARIOS, validate_release_evidence_manifest
MAX_JSON_BYTES = 1024 * 1024
CACHE_COST_PREFIX = "CACHE_COST "
@@ -122,6 +122,33 @@ def require_measured_comparison_evidence(comparison: dict[str, Any], index: int)
f"comparison {index} candidate attempt cost below minimum")
def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None:
require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch")
rounds = manifest.get("rounds")
require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds")
expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS)
require(
report.get("cells") == expected_cells,
f"ABBA matrix cell count mismatch: expected {expected_cells}, got {report.get('cells')}",
)
expected_keys = {
(scenario, comparison, round_id)
for scenario in SCENARIOS
for comparison in ("build", "background")
for round_id in range(1, rounds + 1)
}
observed_keys = []
for index, comparison in enumerate(comparisons):
key = (comparison.get("scenario"), comparison.get("comparison"), comparison.get("round"))
require(key in expected_keys, f"comparison {index} is outside the ABBA matrix")
require(comparison.get("status") in PASS_STATES, f"comparison {index} did not pass")
observed_keys.append(key)
observed_set = set(observed_keys)
require(len(observed_keys) == len(observed_set), "duplicate ABBA matrix comparison")
missing = sorted(expected_keys - observed_set)
require(not missing, f"missing ABBA matrix comparison: {missing[0] if missing else ''}")
def summarize_abba(abba_dir: Path) -> dict[str, Any]:
manifest_path = abba_dir / "manifest.json"
report_path = abba_dir / "report.json"
@@ -170,6 +197,7 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]:
measured = report.get("evidence") == "measured"
passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured
if passed:
require_complete_abba_matrix(manifest, report, comparisons)
validate_release_evidence_manifest({**manifest, "evidence": "measured"})
for index, comparison in enumerate(comparisons):
require_measured_comparison_evidence(comparison, index)
scripts/test_scanner_abba.py
+5 -1
View File
@@ -175,6 +175,9 @@ class ScannerAbbaTest(unittest.TestCase):
"drives_per_node": 4,
"pools": 2,
"sets_total": 2,
"sampled_pools": 2,
"sampled_sets": 2,
"erasure_set_size": 12,
"erasure_data_blocks": 8,
"erasure_parity_blocks": 4,
},
@@ -534,7 +537,8 @@ class ScannerAbbaTest(unittest.TestCase):
faults = {
"missing root": lambda manifest: manifest.pop("release_evidence"),
"single-set": lambda manifest: manifest["release_evidence"]["topology"].update(sets_total=1),
"wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(nodes=4),
"unsampled-set": lambda manifest: manifest["release_evidence"]["topology"].update(sampled_sets=1),
"wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(erasure_set_size=11),
"duplicate endpoint": lambda manifest: manifest["release_evidence"]["distributed"].update(
metrics_endpoints=["https://node-1:9000", "https://node-1:9000", "https://node-3:9000"],
),
scripts/test_summarize_scanner_heal_perf.py
+49 -7
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import copy
import contextlib
import hashlib
import io
@@ -33,6 +34,9 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
self.abba = self.root / "abba"
self.abba.mkdir()
self.manifest = {
"schema": 1,
"evidence": "measured",
"rounds": 3,
"fixed": {
"config_sha256": "1" * 64,
"dataset_sha256": "2" * 64,
@@ -51,6 +55,9 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
"drives_per_node": 4,
"pools": 2,
"sets_total": 2,
"sampled_pools": 2,
"sampled_sets": 2,
"erasure_set_size": 12,
"erasure_data_blocks": 8,
"erasure_parity_blocks": 4,
},
@@ -97,10 +104,20 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
"performance": "pass",
"evidence": "measured",
"cells": 120,
"comparisons": [self.comparison],
"comparisons": self.full_comparisons(),
}
self.write_inputs()
def full_comparisons(self):
comparisons = []
for scenario in summary.SCENARIOS:
for comparison in ("build", "background"):
for round_id in range(1, 4):
row = copy.deepcopy(self.comparison)
row.update(scenario=scenario, comparison=comparison, round=round_id)
comparisons.append(row)
return comparisons
def write_inputs(self):
(self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
(self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8")
@@ -184,22 +201,47 @@ class ScannerHealPerfSummaryTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "passing report requires comparisons"):
summary.build_summary(args)
def test_passing_abba_report_requires_complete_matrix(self):
cases = {
"trimmed": lambda: self.report["comparisons"].pop(),
"duplicate": lambda: self.report["comparisons"].__setitem__(1, copy.deepcopy(self.report["comparisons"][0])),
"bad cells": lambda: self.report.update(cells=119),
"bad evidence": lambda: self.manifest.update(evidence="synthetic"),
"outside": lambda: self.report["comparisons"][0].update(round=99),
"failed comparison": lambda: self.report["comparisons"][0].update(status="inconclusive"),
}
for name, mutate in cases.items():
with self.subTest(fault=name):
self.setUp()
mutate()
self.write_inputs()
args = type("Args", (), {
"abba_dir": self.abba,
"cache_cost_log": None,
"require_cache_cost": False,
"json_out": None,
"markdown_out": None,
})
with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"):
summary.build_summary(args)
def test_passing_abba_report_requires_w10_w11_evidence(self):
for fault in ("missing", "pressure", "lock", "attempt", "length", "range"):
with self.subTest(fault=fault):
self.setUp()
target = self.report["comparisons"][0]
if fault == "missing":
del self.comparison["w10_w11"]
del target["w10_w11"]
elif fault == "pressure":
del self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"]
del target["w10_w11"]["foreground_pressure_high_sample_ratios"]
elif fault == "lock":
del self.comparison["w10_w11"]["heal_lock_wait_p99_ms"]
del target["w10_w11"]["heal_lock_wait_p99_ms"]
elif fault == "attempt":
del self.comparison["w10_w11"]["attempt_cost_per_healed_object"]
del target["w10_w11"]["attempt_cost_per_healed_object"]
elif fault == "length":
self.comparison["w10_w11"]["attempt_cost_per_healed_object"] = [None]
target["w10_w11"]["attempt_cost_per_healed_object"] = [None]
else:
self.comparison["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0]
target["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0]
self.write_inputs()
args = type("Args", (), {
"abba_dir": self.abba,