test(scanner): verify restart evidence against tested builds (#7232)

* chore(deps): refresh scanner heal batch dependency baseline Regenerate compatible lockfile selections before the next implementation batch. Cargo upgrade leaves direct requirements unchanged. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * chore(deps): refresh profiling dependencies for the next batch Update hotpath and its macro crate to the compatible patch release before the next dependency-ready implementation tasks. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(deps): preserve supported hotpath focus expressions Keep the profiler runtime before its regex-lite compatibility regression. Track the opt-in validation required to remove this constraint in backlog. Refs rustfs/backlog#2302. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): add bounded ABBA validation harness Refs rustfs/backlog#2266 and rustfs/backlog#2240. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): verify real restart evidence before release gates Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): bind scanner evidence to execution and build identity Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * docs(test): use the nextest workspace report directory Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): reap ABBA leaders only after process-group cleanup Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): preserve inclusive ABBA thresholds Use decimal boundary comparisons for ABBA ratio checks and cover exact documented p99, throughput, and P1 limits. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com> Co-authored-by: overtrue <anzhengchao@gmail.com>
2026-09-06 03:59:14 +00:00 · 2026-09-06 10:51:22 +08:00
parent fddecf0afe
commit e99c41a9bf
9 changed files with 1050 additions and 42 deletions
@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use std::path::Path;
+use std::process::Command;
+
+fn git(root: &Path, args: &[&str]) -> Option<String> {
+    let output = Command::new("git").args(args).current_dir(root).output().ok()?;
+    output
+        .status
+        .success()
+        .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
+}
+
+fn emit(name: &str, value: &str) {
+    let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
+    println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
+}
+
+fn main() {
+    let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
+    let root = Path::new(&manifest).join("../..");
+    // Cover dependency/common sources as well as this crate. HEAD/ref/index
+    // changes must refresh identity even when no Rust source mtime changes.
+    for path in [
+        "crates",
+        "rustfs",
+        "Cargo.toml",
+        "Cargo.lock",
+        "rust-toolchain.toml",
+        ".cargo",
+        ".config",
+    ] {
+        println!("cargo:rerun-if-changed={}", root.join(path).display());
+    }
+    let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
+    if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
+        git_paths.push(reference);
+    }
+    for path in git_paths {
+        if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
+            let path = Path::new(&path);
+            let path = if path.is_absolute() {
+                path.to_owned()
+            } else {
+                root.join(path)
+            };
+            if path.exists() {
+                println!("cargo:rerun-if-changed={}", path.display());
+            }
+        }
+    }
+    let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
+    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
+    let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
+    let mut features = std::env::vars()
+        .filter_map(|(key, _)| {
+            key.strip_prefix("CARGO_FEATURE_")
+                .map(|name| name.to_ascii_lowercase().replace('_', "-"))
+        })
+        .collect::<Vec<_>>();
+    features.sort();
+    emit("COMMIT", &revision);
+    emit("DIRTY", if dirty { "true" } else { "false" });
+    emit("LOCK", &lock);
+    emit("FEATURES", &features.join(","));
+    for name in ["TARGET", "PROFILE"] {
+        emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
+    }
+    println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
+    let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
+    let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
+    emit("RUSTFLAGS_HEX", &flags);
+}
@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
 /// A successful S3 GET only proves that a quorum can serve an object. Replacement
 /// tests need this lower-level record to prove that the rebuilt target holds the
 /// `xl.meta` selected for a specific version and every `part.N` it declares.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
 pub(crate) struct VersionShardCensus {
    pub version_id: Option<String>,
    pub has_xl_meta: bool,
    pub data_dir: Option<String>,
    pub erasure_index: Option<usize>,
+    pub data_blocks: Option<usize>,
+    pub parity_blocks: Option<usize>,
    pub expected_part_numbers: BTreeSet<usize>,
    pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
    pub inline_data_fingerprint: Option<PartShardFingerprint>,
 }

-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
 pub(crate) struct PartShardFingerprint {
    pub size: u64,
    pub sha256: String,
@@ -88,13 +90,15 @@ impl VersionShardCensus {
            && manifest.is_complete()
            && self.data_dir == manifest.data_dir
            && self.erasure_index == manifest.erasure_index
+            && self.data_blocks == manifest.data_blocks
+            && self.parity_blocks == manifest.parity_blocks
            && self.expected_part_numbers == manifest.expected_part_numbers
            && self.present_part_fingerprints == manifest.present_part_fingerprints
            && self.inline_data_fingerprint == manifest.inline_data_fingerprint
    }
 }

-fn sha256_hex(data: &[u8]) -> String {
+pub(crate) fn sha256_hex(data: &[u8]) -> String {
    let digest = Sha256::digest(data);
    digest.iter().map(|byte| format!("{byte:02x}")).collect()
 }
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
            has_xl_meta: false,
            data_dir: None,
            erasure_index: None,
+            data_blocks: None,
+            parity_blocks: None,
            expected_part_numbers: BTreeSet::new(),
            present_part_fingerprints: BTreeMap::new(),
            inline_data_fingerprint: None,
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
        has_xl_meta: true,
        data_dir,
        erasure_index,
+        data_blocks: Some(file_info.erasure.data_blocks),
+        parity_blocks: Some(file_info.erasure.parity_blocks),
        expected_part_numbers,
        present_part_fingerprints,
        inline_data_fingerprint,
@@ -413,6 +421,8 @@ mod tests {
            has_xl_meta: true,
            data_dir: Some("data-dir".to_string()),
            erasure_index: Some(3),
+            data_blocks: Some(2),
+            parity_blocks: Some(2),
            expected_part_numbers: BTreeSet::from([1]),
            present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
            inline_data_fingerprint: None,
@@ -16,15 +16,18 @@

 #[cfg(test)]
 mod tests {
-    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
+    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
    use crate::common::{
        FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
+        rustfs_binary_path,
    };
    use crate::storage_api::RUSTFS_META_BUCKET;
    use aws_sdk_s3::primitives::ByteStream;
    use http::Method;
+    use sha2::{Digest, Sha256};
    use std::collections::HashSet;
    use std::error::Error;
+    use std::io::{Read, Write};
    use std::net::SocketAddr;
    use std::path::{Path, PathBuf};
    use std::process::Command;
@@ -34,6 +37,76 @@ mod tests {

    const POOL_METADATA_OBJECT: &str = "pool.bin";

+    #[derive(serde::Deserialize)]
+    struct EvidenceBuild {
+        sha256: String,
+    }
+
+    #[derive(serde::Deserialize)]
+    struct RestartEvidenceRun {
+        schema: u32,
+        run_id: String,
+        source_revision: String,
+        test_build: serde_json::Value,
+        binary: EvidenceBuild,
+        test_binary: EvidenceBuild,
+    }
+
+    fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let mut file = std::fs::File::open(path)?;
+        let mut digest = Sha256::new();
+        let mut buffer = [0_u8; 64 * 1024];
+        loop {
+            let read = file.read(&mut buffer)?;
+            if read == 0 {
+                break;
+            }
+            digest.update(&buffer[..read]);
+        }
+        Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
+    }
+
+    fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
+        let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
+            return Ok(None);
+        };
+        let directory = PathBuf::from(directory);
+        let receipt = directory.join("run.json");
+        if receipt.metadata()?.len() > 1024 * 1024 {
+            return Err("oversized scanner/heal execution receipt".into());
+        }
+        let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
+        if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
+            return Err("invalid scanner/heal execution identity".into());
+        }
+        let built = compiled_test_identity();
+        for key in ["source_revision", "dirty", "lock_blob", "features"] {
+            assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
+        }
+        assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
+        assert_eq!(
+            file_sha256(&std::env::current_exe()?)?,
+            run.test_binary.sha256,
+            "test executable must match the run receipt"
+        );
+        if directory.join("background-target-restart.json").exists() {
+            return Err("scanner/heal oracle already exists; create a new execution receipt".into());
+        }
+        Ok(Some((directory, run)))
+    }
+
+    fn compiled_test_identity() -> serde_json::Value {
+        serde_json::json!({
+            "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
+            "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
+            "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
+            "features": env!("RUSTFS_E2E_BUILD_FEATURES"),
+            "target": env!("RUSTFS_E2E_BUILD_TARGET"),
+            "profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
+            "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
+        })
+    }
+
    struct TcpPortBlackhole {
        port: u16,
        comment: String,
@@ -195,8 +268,9 @@ mod tests {
        clients: &[aws_sdk_s3::Client],
        bucket: &str,
        expected_keys: &HashSet<String>,
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+    ) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
        const PAGE_SIZE: i32 = 10;
+        let mut node_listings = Vec::with_capacity(clients.len());
        for (node_index, client) in clients.iter().enumerate() {
            let mut listed_keys = Vec::new();
            let mut continuation_token = None;
@@ -243,8 +317,10 @@ mod tests {
                &listed_key_set, expected_keys,
                "node {node_index} did not expose the complete recovered namespace"
            );
+            listed_keys.sort();
+            node_listings.push(listed_keys);
        }
-        Ok(())
+        Ok(node_listings)
    }

    fn heal_task_status_diagnostic(body: &str) -> String {
@@ -808,6 +884,13 @@ mod tests {
    }

    async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let server_binary = rustfs_binary_path();
+        let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
+            restart_evidence_run(&server_binary)?
+        } else {
+            None
+        };
+        let mut evidence_objects = Vec::new();
        let (background_enabled, interruption_node, interruption_kind) = match scenario {
            InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
            InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
@@ -855,7 +938,7 @@ mod tests {
        for node_index in 0..cluster.nodes.len() {
            cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
        }
-        cluster.start().await?;
+        cluster.start_with_binary(&server_binary).await?;
        let clients = cluster.create_all_clients()?;

        let bucket = "heal-restart-during-rebuild";
@@ -996,7 +1079,7 @@ mod tests {
            }
        }

-        cluster.start_node(1).await?;
+        cluster.start_node_from_binary(1, &server_binary).await?;

        let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
        let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1274,7 +1357,7 @@ mod tests {
                    }
                }
            }
-            cluster.start_node(interruption_node).await?;
+            cluster.start_node_from_binary(interruption_node, &server_binary).await?;
            if interruption_node == 0 {
                let target = cluster.nodes[1]
                    .process
@@ -1373,7 +1456,7 @@ mod tests {
            .map(|manifest| manifest.key.clone())
            .collect::<HashSet<_>>();
        assert!(expected_keys.insert(outage_key.to_string()));
-        assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
+        let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;

        let target_client = cluster.create_s3_client(1)?;
        for expected in &expected_manifests {
@@ -1381,11 +1464,31 @@ mod tests {
            let actual = response.body.collect().await?.into_bytes();
            let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
            assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
+            if evidence_run.is_some() {
+                evidence_objects.push(serde_json::json!({
+                    "key": expected.key, "version_id": expected.shard_census.version_id,
+                    "expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
+                    "expected_sha256": sha256_hex(&expected_body),
+                    "actual_sha256": sha256_hex(&actual),
+                    "expected_physical": expected.shard_census,
+                    "physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
+                }));
+            }
        }
        let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
        let actual = response.body.collect().await?.into_bytes();
        let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
        assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
+        if evidence_run.is_some() {
+            evidence_objects.push(serde_json::json!({
+                "key": outage_key, "version_id": null,
+                "expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
+                "expected_sha256": sha256_hex(&expected_outage_body),
+                "actual_sha256": sha256_hex(&actual),
+                "expected_physical": null,
+                "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
+            }));
+        }

        let terminal_deadline = Instant::now() + Duration::from_secs(30);
        loop {
@@ -1432,6 +1535,31 @@ mod tests {
            return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
        }

+        if let Some((directory, run)) = evidence_run {
+            let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
+            assert_ne!(target_pid, restarted_pid, "target must be a new process");
+            assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
+            let evidence = serde_json::json!({
+                "schema": 1, "case": "background-target-restart", "evidence": "process-restart",
+                "run_id": run.run_id, "source_revision": run.source_revision,
+                "test_build": compiled_test_identity(),
+                "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
+                "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
+                "pid_before": target_pid, "pid_after": restarted_pid,
+                "objects": evidence_objects, "node_listings": node_listings,
+            });
+            let data = serde_json::to_vec(&evidence)?;
+            if data.len() > 1024 * 1024 {
+                return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
+            }
+            let mut output = std::fs::OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(directory.join("background-target-restart.json"))?;
+            output.write_all(&data)?;
+            output.sync_all()?;
+        }
+
        Ok(())
    }