mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 04:58:12 +00:00
chore(scanner): stage Scanner/Heal follow-up slices (#7374)
* fix(scanner): remove unused digest import Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * feat(scanner): add raw page owner index (#7375) * feat(scanner): add raw page owner index Add a serializable raw enumeration page owner index for scanner resume work. The index exposes unsupported, building, and ready states, validates committed page identity by recomputing digests, and uses generation checks for CAS-style page commits. Focused tests cover small-budget restart progress, page digest/source drift rejection, corrupt deserialized state, CAS failure, precommit crash, empty sources, and invalid entry boundaries. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * feat(scanner): persist raw page owner resume state (#7379) Wire the scanner raw enumeration partial-cache writer to the raw page owner index so interrupted bucket walks can retain validated page-builder state across scanner restarts. Keep complete owner sources terminal-only, add partial-source ingestion for in-progress raw directory reads, and validate the persisted page index through bucket checkpoint preparation. Co-authored-by: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com> * test(scanner): fence segment producer observations (#7381) Require the segment observation fixture to carry source, incarnation, key-format, baseline, process epoch, generation-window, gap, overflow, and producer-coverage proof before accepting a narrowed proposal. Keep the diagnostic path fixture-only and remove its ordinary stderr output. Co-authored-by: zhi22915 <qiuzgang@gmail.com> * fix(ecstore): isolate pool metadata read probes (#7367) Co-authored-by: zhi22915 <qiuzgang@gmail.com> * test(heal): cover MRF crash successor matrix (#7369) * test(heal): cover MRF crash successor matrix Add process-boundary MRF replay coverage for the successor snapshot window after a retained startup journal is flushed but before cleanup deletes it. Extend the mixed authoritative/legacy reader fixture with a scoped v2 journal epoch to pin the no-merge contract. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(heal): cover service-kill MRF replay (#7380) Add a Unix process fixture that waits after publishing the pending MRF successor snapshot, then is terminated by the parent before restart replay. Co-authored-by: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com> * test(heal): cover transport-lost start receipts (#7371) Add gRPC transport fault fixtures for heal-control start admission. The tests distinguish pre-admission transport loss from post-admission response loss, then verify exact envelope retries reuse the canonical receipt while fresh forceStart requests create distinct tasks. Co-authored-by: zhi22915 <qiuzgang@gmail.com> * test(scanner): add crash-restart heal evidence case (#7370) * test(scanner): add crash-restart heal evidence case Add a distinct W21 background target crash case to the scanner/heal evidence registry and oracle path. Keep the existing restart lane on graceful process restart, keep the crash lane on hard kill, and make the wiring checker reject evidence/oracle mismatches. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): support older Python wiring checks Let the scanner/heal evidence wiring checker run under Python 3.9/3.10 by falling back to tomli and chunked SHA-256 hashing when the Python 3.11 standard APIs are unavailable. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com> * fix(scanner): reject stale raw page source seeds (#7382) Do not prefill a resumed raw page owner with previously indexed entries when starting a new raw directory observation pass. The next pass must observe the same prefix again before the page index can advance; otherwise the index is discarded fail-closed. Co-authored-by: zhi22915 <qiuzgang@gmail.com> * fix(scanner): defer raw page revalidation until observed (#7384) A resumed raw page owner index must not prefill entries from older cache state, but it also must not discard a valid multi-entry index before the current raw directory pass has observed enough entries to prove identity. Track the persisted index floor and only run the strict owner identity check once the current pass reaches that floor. Co-authored-by: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
@@ -10,7 +10,6 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
@@ -19,6 +18,11 @@ from unittest import mock
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json
|
||||
|
||||
|
||||
@@ -894,6 +898,10 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
||||
require(isinstance(oracle, str) and oracle.endswith(".json"), f"invalid oracle for {case_id}")
|
||||
path = Path(oracle)
|
||||
require(not path.is_absolute() and ".." not in path.parts, f"oracle path escapes run directory for {case_id}")
|
||||
require(requirement.get("evidence") in ("process-restart", "process-crash-restart"),
|
||||
f"invalid evidence for {case_id}")
|
||||
require(type(requirement.get("unclean_shutdown_marker")) is bool,
|
||||
f"invalid unclean-shutdown marker expectation for {case_id}")
|
||||
names.add(oracle)
|
||||
return tuple(sorted(names))
|
||||
|
||||
@@ -1024,9 +1032,11 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(digest(path) == execution["artifacts"][requirement["oracle"]], "oracle hash mismatch")
|
||||
oracle = read_json(path)
|
||||
evidence_integer(oracle.get("schema"), "oracle schema", 1, 1)
|
||||
require(oracle.get("evidence") == "process-restart", "not real process-restart evidence")
|
||||
require(oracle.get("evidence") == requirement["evidence"], f"not real {requirement['evidence']} evidence")
|
||||
require(oracle.get("case") == name and oracle.get("run_id") == run["run_id"], "oracle belongs to another case/run")
|
||||
require(oracle.get("source_revision") == run["source_revision"], "oracle source mismatch")
|
||||
require(oracle.get("unclean_shutdown_marker") is requirement["unclean_shutdown_marker"],
|
||||
"unclean-shutdown marker evidence mismatch")
|
||||
built = oracle["test_build"]
|
||||
for key in ("source_revision", "dirty", "lock_blob", "features"):
|
||||
require(built[key] == expected_build[key], f"compiled test {key} mismatch")
|
||||
@@ -1240,7 +1250,7 @@ class SelfTests(unittest.TestCase):
|
||||
run_dir.mkdir()
|
||||
registry = read_json(ROOT / ".config/scanner-heal-required-tests.json")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
requirement = registry["cases"]["background-target-restart"]
|
||||
requirements = registry["cases"]
|
||||
binary = directory / "fake-binary"
|
||||
binary.write_bytes(b"parser fixture, not a real build")
|
||||
binary.chmod(0o700)
|
||||
@@ -1251,14 +1261,23 @@ class SelfTests(unittest.TestCase):
|
||||
"lock_blob": "c" * 40, "features": "default"},
|
||||
"started_at": datetime.now(timezone.utc).timestamp() - 1,
|
||||
"binary": build, "test_binary": build})
|
||||
write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {
|
||||
"binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
|
||||
suite = "e2e_test"
|
||||
write_json(run_dir / "listing.json", {"rust-suites": {suite: {
|
||||
"binary-id": suite, "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
|
||||
"testcases": {
|
||||
requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
|
||||
}}}})
|
||||
requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
|
||||
for requirement in requirements.values()
|
||||
}
|
||||
}}})
|
||||
(run_dir / "junit.xml").write_text(
|
||||
f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
|
||||
f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/></testsuite></testsuites>')
|
||||
"<testsuites><testsuite>"
|
||||
+ "".join(
|
||||
f'<testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
|
||||
f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/>'
|
||||
for requirement in requirements.values()
|
||||
)
|
||||
+ "</testsuite></testsuites>"
|
||||
)
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
@@ -1268,15 +1287,17 @@ class SelfTests(unittest.TestCase):
|
||||
"expected_physical": physical, "physical": physical}
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
write_json(run_dir / "background-target-restart.json", {
|
||||
"schema": 1, "evidence": "process-restart", "case": "background-target-restart",
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
|
||||
"features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11,
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
})
|
||||
for case_id, requirement in requirements.items():
|
||||
write_json(run_dir / requirement["oracle"], {
|
||||
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
|
||||
"features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
||||
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
})
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
return root, run_dir
|
||||
|
||||
@@ -1332,6 +1353,19 @@ class SelfTests(unittest.TestCase):
|
||||
self.assertEqual(len(errors), 21)
|
||||
self.assertTrue(all(error.startswith("pending ") for error in errors))
|
||||
|
||||
def test_scanner_heal_crash_case_rejects_restart_oracle(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-crash.json"
|
||||
oracle = read_json(path)
|
||||
oracle["evidence"] = "process-restart"
|
||||
oracle["unclean_shutdown_marker"] = False
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-crash"))
|
||||
|
||||
def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
|
||||
for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
|
||||
"hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):
|
||||
|
||||
Reference in New Issue
Block a user