fix(test): bind scanner evidence to execution and build identity

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

crates/e2e_test/build.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use std::path::Path;
+use std::process::Command;
+
+fn git(root: &Path, args: &[&str]) -> Option<String> {
+    let output = Command::new("git").args(args).current_dir(root).output().ok()?;
+    output
+        .status
+        .success()
+        .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
+}
+
+fn emit(name: &str, value: &str) {
+    let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
+    println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
+}
+
+fn main() {
+    let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
+    let root = Path::new(&manifest).join("../..");
+    // Cover dependency/common sources as well as this crate. HEAD/ref/index
+    // changes must refresh identity even when no Rust source mtime changes.
+    for path in [
+        "crates",
+        "rustfs",
+        "Cargo.toml",
+        "Cargo.lock",
+        "rust-toolchain.toml",
+        ".cargo",
+        ".config",
+    ] {
+        println!("cargo:rerun-if-changed={}", root.join(path).display());
+    }
+    let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
+    if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
+        git_paths.push(reference);
+    }
+    for path in git_paths {
+        if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
+            let path = Path::new(&path);
+            let path = if path.is_absolute() {
+                path.to_owned()
+            } else {
+                root.join(path)
+            };
+            if path.exists() {
+                println!("cargo:rerun-if-changed={}", path.display());
+            }
+        }
+    }
+    let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
+    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
+    let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
+    let mut features = std::env::vars()
+        .filter_map(|(key, _)| {
+            key.strip_prefix("CARGO_FEATURE_")
+                .map(|name| name.to_ascii_lowercase().replace('_', "-"))
+        })
+        .collect::<Vec<_>>();
+    features.sort();
+    emit("COMMIT", &revision);
+    emit("DIRTY", if dirty { "true" } else { "false" });
+    emit("LOCK", &lock);
+    emit("FEATURES", &features.join(","));
+    for name in ["TARGET", "PROFILE"] {
+        emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
+    }
+    println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
+    let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
+    let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
+    emit("RUSTFLAGS_HEX", &flags);
+}

crates/e2e_test/src/chaos.rs

@@ -61,6 +61,8 @@ pub(crate) struct VersionShardCensus {
     pub has_xl_meta: bool,
     pub data_dir: Option<String>,
     pub erasure_index: Option<usize>,
+    pub data_blocks: Option<usize>,
+    pub parity_blocks: Option<usize>,
     pub expected_part_numbers: BTreeSet<usize>,
     pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
     pub inline_data_fingerprint: Option<PartShardFingerprint>,
@@ -88,6 +90,8 @@ impl VersionShardCensus {
             && manifest.is_complete()
             && self.data_dir == manifest.data_dir
             && self.erasure_index == manifest.erasure_index
+            && self.data_blocks == manifest.data_blocks
+            && self.parity_blocks == manifest.parity_blocks
             && self.expected_part_numbers == manifest.expected_part_numbers
             && self.present_part_fingerprints == manifest.present_part_fingerprints
             && self.inline_data_fingerprint == manifest.inline_data_fingerprint
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
             has_xl_meta: false,
             data_dir: None,
             erasure_index: None,
+            data_blocks: None,
+            parity_blocks: None,
             expected_part_numbers: BTreeSet::new(),
             present_part_fingerprints: BTreeMap::new(),
             inline_data_fingerprint: None,
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
         has_xl_meta: true,
         data_dir,
         erasure_index,
+        data_blocks: Some(file_info.erasure.data_blocks),
+        parity_blocks: Some(file_info.erasure.parity_blocks),
         expected_part_numbers,
         present_part_fingerprints,
         inline_data_fingerprint,
@@ -413,6 +421,8 @@ mod tests {
             has_xl_meta: true,
             data_dir: Some("data-dir".to_string()),
             erasure_index: Some(3),
+            data_blocks: Some(2),
+            parity_blocks: Some(2),
             expected_part_numbers: BTreeSet::from([1]),
             present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
             inline_data_fingerprint: None,

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -44,9 +44,10 @@ mod tests {
 
     #[derive(serde::Deserialize)]
     struct RestartEvidenceRun {
+        schema: u32,
         run_id: String,
         source_revision: String,
-        test_sources_sha256: String,
+        test_build: serde_json::Value,
         binary: EvidenceBuild,
         test_binary: EvidenceBuild,
     }
@@ -75,14 +76,13 @@ mod tests {
             return Err("oversized scanner/heal execution receipt".into());
         }
         let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
-        if run.run_id.len() != 32 || run.source_revision.len() != 40 {
+        if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
             return Err("invalid scanner/heal execution identity".into());
         }
-        let mut sources = Sha256::new();
-        sources.update(include_bytes!("heal_erasure_disk_rebuild_test.rs"));
-        sources.update(include_bytes!("chaos.rs"));
-        let source_digest: String = sources.finalize().iter().map(|byte| format!("{byte:02x}")).collect();
-        assert_eq!(source_digest, run.test_sources_sha256, "compiled test sources must match the receipt");
+        let built = compiled_test_identity();
+        for key in ["source_revision", "dirty", "lock_blob", "features"] {
+            assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
+        }
         assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
         assert_eq!(
             file_sha256(&std::env::current_exe()?)?,
@@ -95,6 +95,18 @@ mod tests {
         Ok(Some((directory, run)))
     }
 
+    fn compiled_test_identity() -> serde_json::Value {
+        serde_json::json!({
+            "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
+            "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
+            "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
+            "features": env!("RUSTFS_E2E_BUILD_FEATURES"),
+            "target": env!("RUSTFS_E2E_BUILD_TARGET"),
+            "profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
+            "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
+        })
+    }
+
     struct TcpPortBlackhole {
         port: u16,
         comment: String,
@@ -1530,7 +1542,7 @@ mod tests {
             let evidence = serde_json::json!({
                 "schema": 1, "case": "background-target-restart", "evidence": "process-restart",
                 "run_id": run.run_id, "source_revision": run.source_revision,
-                "test_sources_sha256": run.test_sources_sha256,
+                "test_build": compiled_test_identity(),
                 "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
                 "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
                 "pid_before": target_pid, "pid_after": restarted_pid,

docs/testing/ci-gates.md

@@ -153,9 +153,12 @@ Use a committed source tree, independently built current binaries, sufficient
 free disk space, and a task-owned artifact directory that does not yet exist.
 Set `SERVER_BINARY` and `TEST_BINARY` to those exact executable paths. The begin
 command requires the server's embedded `--version` commit to match the clean
-checkout and its embedded Git status to be clean. The producer also checks
-compile-time hashes of its two oracle source files, so a stale test executable
-cannot acquire current oracle semantics just by copying a receipt. The E2E
+checkout and its embedded Git status to be clean. The E2E crate's build script
+embeds its build-time Git revision/dirty state, lockfile Git blob, enabled crate
+features, target, profile and encoded Rust flags. It tracks the crate/dependency
+trees, Cargo inputs and Git HEAD/ref/index, including `common.rs` restart logic.
+The producer checks this compiled identity against the receipt; it does not
+copy a current source revision into an older test binary's identity. The E2E
 uses its existing temporary cluster directories and cleanup. With a dedicated
 `CARGO_TARGET_DIR`, execute the existing selected case as follows:
 
@@ -163,6 +166,7 @@ uses its existing temporary cluster directories and cleanup. With a dedicated
 CASE=background-target-restart
 FILTER='test(test_cluster_root_heal_recovers_remote_shards_after_background_target_restart)'
 RUN_DIR="$PWD/artifacts/scanner-heal-run"
+export RUSTFS_E2E_EXPECTED_FEATURES=default
 scripts/python_bin.sh scripts/check_test_wiring.py \
   --begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY"
 export RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR"
@@ -179,6 +183,11 @@ scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_D
 scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE"
 ```
 
+Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set,
+including `default` for a default-feature build, comma-separated for extra
+features, or empty for `--no-default-features`. It is mandatory when beginning
+a run. Crate features are distinct from the spawned server's build features.
+
 Do not replace a nonzero command exit with zero. Missing JUnit or an oracle
 emission failure also fails acceptance. Each retry needs a new run directory;
 the producer refuses to overwrite an existing oracle. Keep failed-run logs and
@@ -187,6 +196,14 @@ start/finish times, and the artifact hashes. `listing.json`, `junit.xml`, and
 each oracle are limited to 1 MiB; object evidence has the fixture's 9..65 object
 bound. Credentials are not included in the receipt.
 
+The checker binds nextest's flattened suite `binary-id`/`binary-path` to the
+actual test executable and requires the JUnit testcase's embedded execution
+timestamp to fall inside the receipt window (with millisecond precision).
+Copying an old JUnit file and refreshing its mtime does not make it new evidence.
+Schema versions, topology counts, PIDs, EC geometry and shard indices require
+actual integers: booleans and fractional values are rejected, and an index must
+fit the physical data-plus-parity geometry.
+
 The checker rejects unselected/ignored tests, zero/duplicate JUnit cases,
 failures, skipped tests, retry/flaky records, stale or changed artifacts,
 different builds or run IDs, unchanged process IDs, wrong topology, missing

scripts/check_test_wiring.py

@@ -877,6 +877,11 @@ def check_core_listing(root: Path, listing: Path) -> list[str]:
         return [f"cannot read core nextest listing: {error}"]
 
 
+def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> int:
+    require(type(value) is int and minimum <= value <= maximum, f"invalid integer {name}")
+    return value
+
+
 def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
     """Record an existing build; this command never builds or runs a test."""
     require(not directory.exists(), "scanner/heal run directory must be new")
@@ -894,13 +899,18 @@ def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_b
     embedded_status = re.search(r"^git status\s*:\s*(.*)\Z", version, re.MULTILINE | re.DOTALL)
     require(embedded_revision is not None and embedded_revision[1] == revision, "server binary source revision mismatch")
     require(embedded_status is not None and not embedded_status[1].strip(), "server binary was built from dirty/unknown source")
-    sources = root / "crates/e2e_test/src"
-    test_sources_sha256 = hashlib.sha256((sources / "heal_erasure_disk_rebuild_test.rs").read_bytes() +
-                                         (sources / "chaos.rs").read_bytes()).hexdigest()
+    lock_blob = subprocess.check_output(["git", "hash-object", "Cargo.lock"], cwd=root, text=True).strip()
+    require(re.fullmatch(r"[0-9a-f]{40}", lock_blob), "invalid Cargo.lock identity")
+    features = os.environ.get("RUSTFS_E2E_EXPECTED_FEATURES")
+    require(features is not None, "set RUSTFS_E2E_EXPECTED_FEATURES to the compiled e2e crate feature set")
+    features = ",".join(sorted(set(filter(None, (feature.strip() for feature in features.split(","))))))
+    require(all(re.fullmatch(r"[a-z0-9-]+", feature) for feature in features.split(",") if feature), "invalid expected features")
     directory.mkdir(parents=True)
     write_json(directory / "run.json", {"schema": 1, "run_id": uuid.uuid4().hex,
                                        "source_revision": revision,
-                                       "binary_source_revision": embedded_revision[1], "test_sources_sha256": test_sources_sha256,
+                                       "binary_source_revision": embedded_revision[1],
+                                       "test_build": {"source_revision": revision, "dirty": False,
+                                                      "lock_blob": lock_blob, "features": features},
                                        "started_at": datetime.now(timezone.utc).timestamp(), **builds})
 
 
@@ -925,14 +935,20 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
     """Validate one actual case, or fail the release while required lanes are pending."""
     try:
         registry = read_json(root / ".config/scanner-heal-required-tests.json")
-        require(registry.get("schema") == 1 and registry.get("cases"), "invalid scanner/heal registry")
+        evidence_integer(registry.get("schema"), "registry schema", 1, 1)
+        require(registry.get("cases"), "invalid scanner/heal registry")
         selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
         run = read_json(directory / "run.json")
         execution = read_json(directory / "execution.json")
-        require(run.get("schema") == 1 and re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity")
+        evidence_integer(run.get("schema"), "run schema", 1, 1)
+        require(re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity")
         require(re.fullmatch(r"[0-9a-f]{40}", run["source_revision"]), "invalid source revision")
         require(run.get("binary_source_revision") == run["source_revision"], "server source provenance missing")
-        require(sha(run.get("test_sources_sha256")), "test source provenance missing")
+        expected_build = run["test_build"]
+        require(expected_build["source_revision"] == run["source_revision"] and expected_build["dirty"] is False,
+                "test source provenance missing")
+        require(re.fullmatch(r"[0-9a-f]{40}", expected_build["lock_blob"]), "invalid test lockfile identity")
+        require(isinstance(expected_build["features"], str), "missing test features")
         require(execution.get("run_id") == run["run_id"], "execution belongs to another run")
         require(type(execution.get("exit_code")) is int and execution["exit_code"] == 0, "test command failed or did not run")
         number(run["started_at"], "started_at", 1)
@@ -957,25 +973,49 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
         errors = []
         for name, requirement in selected.items():
             suite, test = requirement["suite"], requirement["name"]
-            listed = suites.get(suite, {}).get("testcases", {}).get(test, {})
+            for key in ("nodes", "drives_per_node"):
+                evidence_integer(requirement["topology"][key], f"required {key}", 1, 16)
+            listing_suite = suites[suite]
+            require(listing_suite["binary-id"] == suite, "nextest suite binary identity mismatch")
+            listed_binary = Path(listing_suite["binary-path"]).resolve(strict=True)
+            require(listed_binary == Path(run["test_binary"]["path"]).resolve(strict=True) and
+                    digest(listed_binary) == run["test_binary"]["sha256"], "nextest selected another test binary")
+            require(listing_suite["package-name"] == "e2e_test" and listing_suite["build-platform"] in ("host", "target"),
+                    "unexpected nextest suite metadata")
+            listed = listing_suite.get("testcases", {}).get(test, {})
             require(listed.get("ignored") is False and listed.get("filter-match", {}).get("status") == "matches",
                     f"required test not selected: {suite}::{test}")
             matches = [case for case in cases if case.get("name") == test and case.get("classname") == suite]
             require(len(matches) == 1, f"missing/duplicate JUnit case: {suite}::{test}")
+            started = datetime.fromisoformat(matches[0].attrib["timestamp"].replace("Z", "+00:00"))
+            require(started.tzinfo is not None, "JUnit timestamp must include timezone")
+            # quick-junit truncates timestamps to milliseconds.
+            require(run["started_at"] - 0.001 <= started.timestamp() <= execution["finished_at"],
+                    "JUnit testcase executed outside this run")
             path = directory / requirement["oracle"]
             require(path.resolve().is_relative_to(directory.resolve()), "oracle path escapes run directory")
             require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], "oracle outside run window")
             require(digest(path) == execution["artifacts"][requirement["oracle"]], "oracle hash mismatch")
             oracle = read_json(path)
-            require(oracle.get("schema") == 1 and oracle.get("evidence") == "process-restart", "not real process-restart evidence")
+            evidence_integer(oracle.get("schema"), "oracle schema", 1, 1)
+            require(oracle.get("evidence") == "process-restart", "not real process-restart evidence")
             require(oracle.get("case") == name and oracle.get("run_id") == run["run_id"], "oracle belongs to another case/run")
             require(oracle.get("source_revision") == run["source_revision"], "oracle source mismatch")
-            require(oracle.get("test_sources_sha256") == run["test_sources_sha256"], "compiled test source mismatch")
+            built = oracle["test_build"]
+            for key in ("source_revision", "dirty", "lock_blob", "features"):
+                require(built[key] == expected_build[key], f"compiled test {key} mismatch")
+            require(built["dirty"] is False, "test binary was compiled from dirty source")
+            require(all(isinstance(built[key], str) and built[key] and built[key] != "unknown" for key in ("target", "profile")),
+                    "missing compiled target/profile")
+            require(isinstance(built["rustflags_hex"], str) and re.fullmatch(r"(?:[0-9a-f]{2})*", built["rustflags_hex"]) is not None,
+                    "invalid compiled rustflags")
             for label in ("binary", "test_binary"):
                 require(oracle.get(f"{label}_sha256") == run[label]["sha256"], f"oracle {label} mismatch")
             require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
-            number(oracle.get("pid_before"), "pid_before", 1)
-            number(oracle.get("pid_after"), "pid_after", 1)
+            for key in ("nodes", "drives_per_node"):
+                evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
+            evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
+            evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
             require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
             objects = oracle["objects"]
             require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"],
@@ -992,8 +1032,13 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
                 physical = obj["physical"]
                 if obj["expected_physical"] is not None:
                     require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest")
+                for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []):
+                    data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16)
+                    parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
+                    require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
+                            "EC geometry differs from this case's single set")
+                    evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
                 require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
-                number(physical["erasure_index"], "target erasure index", 1)
                 parts = physical["expected_part_numbers"]
                 require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage")
                 require(all(type(part) is int and part > 0 for part in parts) and len(set(parts)) == len(parts),
@@ -1171,16 +1216,21 @@ class SelfTests(unittest.TestCase):
         binary.chmod(0o700)
         build = {"path": str(binary), "sha256": digest(binary)}
         write_json(run_dir / "run.json", {"schema": 1, "run_id": "a" * 32, "source_revision": "b" * 40,
-                                         "binary_source_revision": "b" * 40, "test_sources_sha256": "e" * 64,
+                                         "binary_source_revision": "b" * 40,
+                                         "test_build": {"source_revision": "b" * 40, "dirty": False,
+                                                        "lock_blob": "c" * 40, "features": "default"},
                                          "started_at": datetime.now(timezone.utc).timestamp() - 1,
                                          "binary": build, "test_binary": build})
-        write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {"testcases": {
+        write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {
+            "binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
+            "testcases": {
             requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
         }}}})
         (run_dir / "junit.xml").write_text(
-            f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}"/></testsuite></testsuites>')
+            f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
+            f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/></testsuite></testsuites>')
         physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
-                    "erasure_index": 1, "expected_part_numbers": [1],
+                    "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
                     "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
                     "inline_data_fingerprint": None}
         obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
@@ -1191,7 +1241,8 @@ class SelfTests(unittest.TestCase):
         write_json(run_dir / "background-target-restart.json", {
             "schema": 1, "evidence": "process-restart", "case": "background-target-restart",
             "run_id": "a" * 32, "source_revision": "b" * 40,
-            "test_sources_sha256": "e" * 64,
+            "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
+                           "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
             "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
             "topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11,
             "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
@@ -1286,7 +1337,8 @@ class SelfTests(unittest.TestCase):
                     version += "modified source\n"
                 if kind == "unknown":
                     version = "rustfs without build provenance"
-                with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version]):
+                with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version, "c" * 40]), \
+                     mock.patch.dict(os.environ, {"RUSTFS_E2E_EXPECTED_FEATURES": "default"}):
                     directory = Path(tmp) / "fresh"
                     binary = Path(tmp) / "fake-binary"
                     if kind == "current":
@@ -1297,6 +1349,67 @@ class SelfTests(unittest.TestCase):
                             begin_scanner_heal_receipt(root, directory, binary, binary)
                         self.assertFalse(directory.exists())
 
+    def test_scanner_heal_rejects_copied_junit_and_wrong_suite_build(self) -> None:
+        for fault in ("old-junit", "missing-time", "wrong-binary", "common-source", "lockfile", "features", "dirty-build"):
+            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
+                root, run_dir = self.scanner_heal_fixture(Path(tmp))
+                if fault in ("old-junit", "missing-time"):
+                    path = run_dir / "junit.xml"
+                    xml = ET.fromstring(path.read_bytes())
+                    testcase = next(xml.iter("testcase"))
+                    if fault == "old-junit":
+                        testcase.set("timestamp", "2000-01-01T00:00:00.000Z")
+                    else:
+                        del testcase.attrib["timestamp"]
+                    # Rewriting/copying gives an old execution a fresh mtime.
+                    path.write_bytes(ET.tostring(xml))
+                elif fault == "wrong-binary":
+                    path = run_dir / "listing.json"
+                    listing = read_json(path)
+                    another = Path(tmp) / "another-binary"
+                    another.write_bytes(Path(tmp, "fake-binary").read_bytes())
+                    listing["rust-suites"]["e2e_test"]["binary-path"] = str(another)
+                    write_json(path, listing)
+                else:
+                    path = run_dir / "background-target-restart.json"
+                    oracle = read_json(path)
+                    key, value = {"common-source": ("source_revision", "f" * 40), "lockfile": ("lock_blob", "f" * 40),
+                                  "features": ("features", "default,sftp"), "dirty-build": ("dirty", True)}[fault]
+                    oracle["test_build"][key] = value
+                    write_json(path, oracle)
+                (run_dir / "execution.json").unlink()
+                finish_scanner_heal_receipt(run_dir, 0)
+                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault)
+
+    def test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers(self) -> None:
+        valid = {"schema": 1, "nodes": 4, "drives_per_node": 1, "pid_before": 10, "pid_after": 11,
+                 "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2}
+        cases = [(field, value) for field, correct in valid.items() for value in (True, float(correct))]
+        cases += [("erasure_index", 5), ("erasure_index", 0), ("pid_after", -1)]
+        for field, value in cases:
+            with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as tmp:
+                root, run_dir = self.scanner_heal_fixture(Path(tmp))
+                path = run_dir / "background-target-restart.json"
+                oracle = read_json(path)
+                if field in ("nodes", "drives_per_node"):
+                    oracle["topology"][field] = value
+                elif field in ("erasure_index", "data_blocks", "parity_blocks"):
+                    oracle["objects"][-1]["physical"][field] = value
+                else:
+                    oracle[field] = value
+                write_json(path, oracle)
+                (run_dir / "execution.json").unlink()
+                finish_scanner_heal_receipt(run_dir, 0)
+                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
+        for filename in ("run.json", ".config/scanner-heal-required-tests.json"):
+            with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp:
+                root, run_dir = self.scanner_heal_fixture(Path(tmp))
+                path = (root if filename.startswith(".config") else run_dir) / filename
+                content = read_json(path)
+                content["schema"] = True
+                write_json(path, content)
+                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
+
     def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)