mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
test(scanner): verify restart evidence against tested builds (#7232)
* chore(deps): refresh scanner heal batch dependency baseline Regenerate compatible lockfile selections before the next implementation batch. Cargo upgrade leaves direct requirements unchanged. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(ecstore): remove duplicate local rename implementation Keep the canonical commit module after concurrent storage changes merged. The control-write and rollback changes are already present there. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * chore(deps): refresh profiling dependencies for the next batch Update hotpath and its macro crate to the compatible patch release before the next dependency-ready implementation tasks. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(deps): preserve supported hotpath focus expressions Keep the profiler runtime before its regex-lite compatibility regression. Track the opt-in validation required to remove this constraint in backlog. Refs rustfs/backlog#2302. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): add bounded ABBA validation harness Refs rustfs/backlog#2266 and rustfs/backlog#2240. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): verify real restart evidence before release gates Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): bind scanner evidence to execution and build identity Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * docs(test): use the nextest workspace report directory Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): reap ABBA leaders only after process-group cleanup Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(test): preserve inclusive ABBA thresholds Use decimal boundary comparisons for ABBA ratio checks and cover exact documented p99, throughput, and P1 limits. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com> Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart.json",
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn emit(name: &str, value: &str) {
|
||||
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
|
||||
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
|
||||
let root = Path::new(&manifest).join("../..");
|
||||
// Cover dependency/common sources as well as this crate. HEAD/ref/index
|
||||
// changes must refresh identity even when no Rust source mtime changes.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={}", root.join(path).display());
|
||||
}
|
||||
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
|
||||
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
|
||||
git_paths.push(reference);
|
||||
}
|
||||
for path in git_paths {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
|
||||
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let mut features = std::env::vars()
|
||||
.filter_map(|(key, _)| {
|
||||
key.strip_prefix("CARGO_FEATURE_")
|
||||
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
features.sort();
|
||||
emit("COMMIT", &revision);
|
||||
emit("DIRTY", if dirty { "true" } else { "false" });
|
||||
emit("LOCK", &lock);
|
||||
emit("FEATURES", &features.join(","));
|
||||
for name in ["TARGET", "PROFILE"] {
|
||||
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
|
||||
}
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
|
||||
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
emit("RUSTFLAGS_HEX", &flags);
|
||||
}
|
||||
@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
@@ -88,13 +90,15 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
pub(crate) fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
@@ -413,6 +421,8 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
|
||||
@@ -16,15 +16,18 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -34,6 +37,76 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join("background-target-restart.json").exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some((directory, run)))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
@@ -195,8 +268,9 @@ mod tests {
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
let mut node_listings = Vec::with_capacity(clients.len());
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
@@ -243,8 +317,10 @@ mod tests {
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
listed_keys.sort();
|
||||
node_listings.push(listed_keys);
|
||||
}
|
||||
Ok(())
|
||||
Ok(node_listings)
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
@@ -808,6 +884,13 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
@@ -855,7 +938,7 @@ mod tests {
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start().await?;
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
@@ -996,7 +1079,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
cluster.start_node(1).await?;
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1274,7 +1357,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node(interruption_node).await?;
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
@@ -1373,7 +1456,7 @@ mod tests {
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
@@ -1381,11 +1464,31 @@ mod tests {
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": expected.shard_census,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": outage_key, "version_id": null,
|
||||
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
@@ -1432,6 +1535,31 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some((directory, run)) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
|
||||
"run_id": run.run_id, "source_revision": run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.join("background-target-restart.json"))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +85,13 @@ at most 1 MiB. Logs are kept separately and require an operator-managed disk
|
||||
quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero
|
||||
samples, and request errors fail the run. Adapters must terminate their own
|
||||
children on failure and `stop` must be idempotent even after partial preparation.
|
||||
The runner keeps its session leader unreaped while stopping a failed command
|
||||
or collector: it sends TERM, allows the existing ten-second grace period, then
|
||||
kills the remaining process group before reaping. This prevents a parent exit
|
||||
from hiding live descendants or allowing the group ID to be reused before its
|
||||
last signal. A successful `prepare` preserves adapter-owned services until
|
||||
`stop`; services that leave the command's process group remain the adapter's
|
||||
cleanup responsibility.
|
||||
|
||||
The request contains the fixed manifest fields, selected build, scenario, round,
|
||||
leg, comparison (`build` or `background`), background mode (`on` or `off`),
|
||||
|
||||
@@ -126,3 +126,108 @@ The manifest records a minimum set of invariants: write quorum, metadata rollbac
|
||||
The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability.
|
||||
|
||||
Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review.
|
||||
## Scanner/Heal Evidence Receipts
|
||||
|
||||
The existing `scripts/check_test_wiring.py` also validates Scanner/Heal case
|
||||
evidence registered in `.config/scanner-heal-required-tests.json`. It records
|
||||
already-built binaries and checks existing nextest output; it does not build,
|
||||
run tests, deploy servers, inject faults, or start another CI lane.
|
||||
|
||||
The initial case is `background-target-restart`, emitted by
|
||||
`heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`.
|
||||
That test already runs in `e2e-nightly`. When `RUSTFS_SCANNER_HEAL_RUN_DIR` is set,
|
||||
it checks the actual server and test-executable hashes against `run.json`, pins
|
||||
the same server binary for all node starts, and writes its oracle only after
|
||||
the real assertions pass. The artifact contains the actual pre/post target
|
||||
PIDs, per-node S3 listings, expected and downloaded complete-body hashes/lengths,
|
||||
and target-disk `VersionShardCensus` fingerprints. Existing baseline objects
|
||||
must match their pre-fault physical manifests; the object created during the
|
||||
outage has no pre-fault target shard and is checked for complete physical parts
|
||||
and exact S3 content.
|
||||
|
||||
This case is a **four-node, one-drive-per-node process-restart test**. It is not
|
||||
power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
|
||||
of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
|
||||
The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending
|
||||
until their actual feature-specific oracles and required topologies exist.
|
||||
Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON
|
||||
and file-hash helpers are reused; its ABBA performance contracts remain in
|
||||
`docs/operations/scanner-benchmark-runbook.md`.
|
||||
|
||||
### Recording One Case
|
||||
|
||||
Use a committed source tree, independently built current binaries, sufficient
|
||||
free disk space, and a task-owned artifact directory that does not yet exist.
|
||||
Set `SERVER_BINARY` and `TEST_BINARY` to those exact executable paths. The begin
|
||||
command requires the server's embedded `--version` commit to match the clean
|
||||
checkout and its embedded Git status to be clean. The E2E crate's build script
|
||||
embeds its build-time Git revision/dirty state, lockfile Git blob, enabled crate
|
||||
features, target, profile and encoded Rust flags. It tracks the crate/dependency
|
||||
trees, Cargo inputs and Git HEAD/ref/index, including `common.rs` restart logic.
|
||||
The producer checks this compiled identity against the receipt; it does not
|
||||
copy a current source revision into an older test binary's identity. The E2E
|
||||
uses its existing temporary cluster directories and cleanup. `CARGO_TARGET_DIR`
|
||||
controls compilation output; nextest's default report store remains the
|
||||
workspace's `target/nextest`. Execute the existing selected case as follows:
|
||||
|
||||
```bash
|
||||
CASE=background-target-restart
|
||||
FILTER='test(test_cluster_root_heal_recovers_remote_shards_after_background_target_restart)'
|
||||
RUN_DIR="$PWD/artifacts/scanner-heal-run"
|
||||
export RUSTFS_E2E_EXPECTED_FEATURES=default
|
||||
scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||
--begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY"
|
||||
export RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR"
|
||||
export CARGO_BIN_EXE_rustfs="$SERVER_BINARY"
|
||||
cargo nextest list --profile e2e-nightly -p e2e_test -E "$FILTER" \
|
||||
--message-format json > "$RUN_DIR/listing.json"
|
||||
rm -f target/nextest/e2e-nightly/junit.xml
|
||||
set +e
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test -E "$FILTER"
|
||||
test_exit=$?
|
||||
set -e
|
||||
cp target/nextest/e2e-nightly/junit.xml "$RUN_DIR/junit.xml"
|
||||
scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_DIR" "$test_exit"
|
||||
scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE"
|
||||
```
|
||||
|
||||
Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set,
|
||||
including `default` for a default-feature build, comma-separated for extra
|
||||
features, or empty for `--no-default-features`. It is mandatory when beginning
|
||||
a run. Crate features are distinct from the spawned server's build features.
|
||||
|
||||
Do not replace a nonzero command exit with zero. Missing JUnit or an oracle
|
||||
emission failure also fails acceptance. Each retry needs a new run directory;
|
||||
the producer refuses to overwrite an existing oracle. Keep failed-run logs and
|
||||
artifacts. The receipt pins source revision, actual binary hashes, run identity,
|
||||
start/finish times, and the artifact hashes. `listing.json`, `junit.xml`, and
|
||||
each oracle are limited to 1 MiB; object evidence has the fixture's 9..65 object
|
||||
bound. Credentials are not included in the receipt.
|
||||
|
||||
The checker binds nextest's flattened suite `binary-id`/`binary-path` to the
|
||||
actual test executable and requires the JUnit testcase's embedded execution
|
||||
timestamp to fall inside the receipt window (with millisecond precision).
|
||||
Copying an old JUnit file and refreshing its mtime does not make it new evidence.
|
||||
Schema versions, topology counts, PIDs, EC geometry and shard indices require
|
||||
actual integers: booleans and fractional values are rejected, and an index must
|
||||
fit the physical data-plus-parity geometry.
|
||||
|
||||
The checker rejects unselected/ignored tests, zero/duplicate JUnit cases,
|
||||
failures, skipped tests, retry/flaky records, stale or changed artifacts,
|
||||
different builds or run IDs, unchanged process IDs, wrong topology, missing
|
||||
shard parts, and mismatched S3 content/listings. The raw oracle JSON is emitted
|
||||
by the real E2E producer, not accepted from an adapter copying expectations.
|
||||
|
||||
`--check-scanner-heal "$RUN_DIR" release` checks available case evidence and
|
||||
returns nonzero for every pending release requirement. A focused case pass
|
||||
does not approve release. In particular, R-E requires fixed-budget real
|
||||
restarts without an unbudgeted final sweep, R-D requires the full
|
||||
manager/event/ledger disposition chain, and R-L requires source-conflict and
|
||||
crash/retirement evidence. Reader-only or unit fixtures cannot substitute for
|
||||
these. The external `rustfs/auto-testing` functional workflows propagate suite
|
||||
failures. Their workflow status does not establish this registry's required
|
||||
case coverage, build provenance, or object-level oracles.
|
||||
|
||||
Run parser/receipt regressions with
|
||||
`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
|
||||
validate the checker only and produce no runtime or performance evidence.
|
||||
|
||||
@@ -12,11 +12,15 @@ import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCHEDULED_ALERT_WORKFLOWS = tuple(
|
||||
@@ -873,6 +877,187 @@ def check_core_listing(root: Path, listing: Path) -> list[str]:
|
||||
return [f"cannot read core nextest listing: {error}"]
|
||||
|
||||
|
||||
def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> int:
|
||||
require(type(value) is int and minimum <= value <= maximum, f"invalid integer {name}")
|
||||
return value
|
||||
|
||||
|
||||
def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
|
||||
"""Record an existing build; this command never builds or runs a test."""
|
||||
require(not directory.exists(), "scanner/heal run directory must be new")
|
||||
require(not subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, text=True).strip(),
|
||||
"commit tracked source changes before creating evidence")
|
||||
builds = {}
|
||||
for label, path in (("binary", binary), ("test_binary", test_binary)):
|
||||
path = path.resolve(strict=True)
|
||||
require(path.is_file() and os.access(path, os.X_OK), f"missing executable {label}")
|
||||
builds[label] = {"path": str(path), "sha256": digest(path)}
|
||||
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
||||
require(re.fullmatch(r"[0-9a-f]{40}", revision), "invalid source revision")
|
||||
version = subprocess.check_output([builds["binary"]["path"], "--version"], text=True, timeout=30)
|
||||
embedded_revision = re.search(r"^git commit\s*:\s*([0-9a-f]{40})\s*$", version, re.MULTILINE)
|
||||
embedded_status = re.search(r"^git status\s*:\s*(.*)\Z", version, re.MULTILINE | re.DOTALL)
|
||||
require(embedded_revision is not None and embedded_revision[1] == revision, "server binary source revision mismatch")
|
||||
require(embedded_status is not None and not embedded_status[1].strip(), "server binary was built from dirty/unknown source")
|
||||
lock_blob = subprocess.check_output(["git", "hash-object", "Cargo.lock"], cwd=root, text=True).strip()
|
||||
require(re.fullmatch(r"[0-9a-f]{40}", lock_blob), "invalid Cargo.lock identity")
|
||||
features = os.environ.get("RUSTFS_E2E_EXPECTED_FEATURES")
|
||||
require(features is not None, "set RUSTFS_E2E_EXPECTED_FEATURES to the compiled e2e crate feature set")
|
||||
features = ",".join(sorted(set(filter(None, (feature.strip() for feature in features.split(","))))))
|
||||
require(all(re.fullmatch(r"[a-z0-9-]+", feature) for feature in features.split(",") if feature), "invalid expected features")
|
||||
directory.mkdir(parents=True)
|
||||
write_json(directory / "run.json", {"schema": 1, "run_id": uuid.uuid4().hex,
|
||||
"source_revision": revision,
|
||||
"binary_source_revision": embedded_revision[1],
|
||||
"test_build": {"source_revision": revision, "dirty": False,
|
||||
"lock_blob": lock_blob, "features": features},
|
||||
"started_at": datetime.now(timezone.utc).timestamp(), **builds})
|
||||
|
||||
|
||||
def finish_scanner_heal_receipt(directory: Path, exit_code: int) -> None:
|
||||
require(type(exit_code) is int and 0 <= exit_code <= 255, "invalid test exit code")
|
||||
require(not (directory / "execution.json").exists(), "execution receipt already exists")
|
||||
run = read_json(directory / "run.json")
|
||||
artifacts = {}
|
||||
for name in ("listing.json", "junit.xml", "background-target-restart.json"):
|
||||
path = directory / name
|
||||
if exit_code != 0 and not path.exists():
|
||||
continue
|
||||
require(path.is_file() and 0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}")
|
||||
require(path.stat().st_mtime >= run["started_at"], f"stale {name}")
|
||||
artifacts[name] = digest(path)
|
||||
write_json(directory / "execution.json", {"run_id": run["run_id"], "exit_code": exit_code,
|
||||
"finished_at": datetime.now(timezone.utc).timestamp(),
|
||||
"artifacts": artifacts})
|
||||
|
||||
|
||||
def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> list[str]:
|
||||
"""Validate one actual case, or fail the release while required lanes are pending."""
|
||||
try:
|
||||
registry = read_json(root / ".config/scanner-heal-required-tests.json")
|
||||
evidence_integer(registry.get("schema"), "registry schema", 1, 1)
|
||||
require(registry.get("cases"), "invalid scanner/heal registry")
|
||||
selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
|
||||
run = read_json(directory / "run.json")
|
||||
execution = read_json(directory / "execution.json")
|
||||
evidence_integer(run.get("schema"), "run schema", 1, 1)
|
||||
require(re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity")
|
||||
require(re.fullmatch(r"[0-9a-f]{40}", run["source_revision"]), "invalid source revision")
|
||||
require(run.get("binary_source_revision") == run["source_revision"], "server source provenance missing")
|
||||
expected_build = run["test_build"]
|
||||
require(expected_build["source_revision"] == run["source_revision"] and expected_build["dirty"] is False,
|
||||
"test source provenance missing")
|
||||
require(re.fullmatch(r"[0-9a-f]{40}", expected_build["lock_blob"]), "invalid test lockfile identity")
|
||||
require(isinstance(expected_build["features"], str), "missing test features")
|
||||
require(execution.get("run_id") == run["run_id"], "execution belongs to another run")
|
||||
require(type(execution.get("exit_code")) is int and execution["exit_code"] == 0, "test command failed or did not run")
|
||||
number(run["started_at"], "started_at", 1)
|
||||
number(execution["finished_at"], "finished_at", run["started_at"])
|
||||
for label in ("binary", "test_binary"):
|
||||
require(sha(run[label]["sha256"]) and digest(Path(run[label]["path"])) == run[label]["sha256"],
|
||||
f"{label} changed or missing")
|
||||
for name in ("listing.json", "junit.xml"):
|
||||
path = directory / name
|
||||
require(0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}")
|
||||
require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], f"{name} outside run window")
|
||||
require(digest(path) == execution["artifacts"][name], f"{name} hash mismatch")
|
||||
suites = read_json(directory / "listing.json")["rust-suites"]
|
||||
xml = (directory / "junit.xml").read_bytes()
|
||||
require(b"<!DOCTYPE" not in xml and b"<!ENTITY" not in xml, "JUnit entities are forbidden")
|
||||
junit = ET.fromstring(xml)
|
||||
cases = list(junit.iter("testcase"))
|
||||
require(bool(cases), "JUnit has zero testcases")
|
||||
for case in cases:
|
||||
require(not any(child.tag in ("failure", "error", "skipped", "rerunFailure", "rerunError", "flakyFailure", "flakyError")
|
||||
for child in case), "JUnit contains failed, skipped or retried tests")
|
||||
errors = []
|
||||
for name, requirement in selected.items():
|
||||
suite, test = requirement["suite"], requirement["name"]
|
||||
for key in ("nodes", "drives_per_node"):
|
||||
evidence_integer(requirement["topology"][key], f"required {key}", 1, 16)
|
||||
listing_suite = suites[suite]
|
||||
require(listing_suite["binary-id"] == suite, "nextest suite binary identity mismatch")
|
||||
listed_binary = Path(listing_suite["binary-path"]).resolve(strict=True)
|
||||
require(listed_binary == Path(run["test_binary"]["path"]).resolve(strict=True) and
|
||||
digest(listed_binary) == run["test_binary"]["sha256"], "nextest selected another test binary")
|
||||
require(listing_suite["package-name"] == "e2e_test" and listing_suite["build-platform"] in ("host", "target"),
|
||||
"unexpected nextest suite metadata")
|
||||
listed = listing_suite.get("testcases", {}).get(test, {})
|
||||
require(listed.get("ignored") is False and listed.get("filter-match", {}).get("status") == "matches",
|
||||
f"required test not selected: {suite}::{test}")
|
||||
matches = [case for case in cases if case.get("name") == test and case.get("classname") == suite]
|
||||
require(len(matches) == 1, f"missing/duplicate JUnit case: {suite}::{test}")
|
||||
started = datetime.fromisoformat(matches[0].attrib["timestamp"].replace("Z", "+00:00"))
|
||||
require(started.tzinfo is not None, "JUnit timestamp must include timezone")
|
||||
# quick-junit truncates timestamps to milliseconds.
|
||||
require(run["started_at"] - 0.001 <= started.timestamp() <= execution["finished_at"],
|
||||
"JUnit testcase executed outside this run")
|
||||
path = directory / requirement["oracle"]
|
||||
require(path.resolve().is_relative_to(directory.resolve()), "oracle path escapes run directory")
|
||||
require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], "oracle outside run window")
|
||||
require(digest(path) == execution["artifacts"][requirement["oracle"]], "oracle hash mismatch")
|
||||
oracle = read_json(path)
|
||||
evidence_integer(oracle.get("schema"), "oracle schema", 1, 1)
|
||||
require(oracle.get("evidence") == "process-restart", "not real process-restart evidence")
|
||||
require(oracle.get("case") == name and oracle.get("run_id") == run["run_id"], "oracle belongs to another case/run")
|
||||
require(oracle.get("source_revision") == run["source_revision"], "oracle source mismatch")
|
||||
built = oracle["test_build"]
|
||||
for key in ("source_revision", "dirty", "lock_blob", "features"):
|
||||
require(built[key] == expected_build[key], f"compiled test {key} mismatch")
|
||||
require(built["dirty"] is False, "test binary was compiled from dirty source")
|
||||
require(all(isinstance(built[key], str) and built[key] and built[key] != "unknown" for key in ("target", "profile")),
|
||||
"missing compiled target/profile")
|
||||
require(isinstance(built["rustflags_hex"], str) and re.fullmatch(r"(?:[0-9a-f]{2})*", built["rustflags_hex"]) is not None,
|
||||
"invalid compiled rustflags")
|
||||
for label in ("binary", "test_binary"):
|
||||
require(oracle.get(f"{label}_sha256") == run[label]["sha256"], f"oracle {label} mismatch")
|
||||
require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
|
||||
for key in ("nodes", "drives_per_node"):
|
||||
evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
|
||||
evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
|
||||
evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
|
||||
require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
|
||||
objects = oracle["objects"]
|
||||
require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"],
|
||||
"incomplete/oversized object oracle")
|
||||
require(len({obj["key"] for obj in objects}) == len(objects), "duplicate object identity")
|
||||
require(sum(obj["expected_physical"] is None for obj in objects) == 1,
|
||||
"only the outage object may lack a pre-fault target manifest")
|
||||
for obj in objects:
|
||||
require(isinstance(obj["key"], str) and 0 < len(obj["key"].encode()) <= 1024, "invalid object identity")
|
||||
require(obj["version_id"] is None, "this case only covers unversioned objects")
|
||||
require(type(obj["expected_bytes"]) is int and obj["expected_bytes"] > 0, "missing expected bytes")
|
||||
require(type(obj["actual_bytes"]) is int and obj["actual_bytes"] == obj["expected_bytes"], "S3 body length mismatch")
|
||||
require(sha(obj["expected_sha256"]) and obj["actual_sha256"] == obj["expected_sha256"], "S3 body digest mismatch")
|
||||
physical = obj["physical"]
|
||||
if obj["expected_physical"] is not None:
|
||||
require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest")
|
||||
for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []):
|
||||
data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16)
|
||||
parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
|
||||
require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
|
||||
"EC geometry differs from this case's single set")
|
||||
evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
|
||||
require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
|
||||
parts = physical["expected_part_numbers"]
|
||||
require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage")
|
||||
require(all(type(part) is int and part > 0 for part in parts) and len(set(parts)) == len(parts),
|
||||
"invalid physical part identity")
|
||||
require({str(part) for part in parts} == set(physical["present_part_fingerprints"]), "target shard parts missing")
|
||||
for part in physical["present_part_fingerprints"].values():
|
||||
require(type(part["size"]) is int and part["size"] > 0 and sha(part["sha256"]), "invalid target shard fingerprint")
|
||||
node_listings = oracle["node_listings"]
|
||||
require(isinstance(node_listings, list) and len(node_listings) == requirement["topology"]["nodes"],
|
||||
"missing per-node S3 listing")
|
||||
require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings),
|
||||
"S3 listing differs from object oracle")
|
||||
if case_id == "release":
|
||||
errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items())
|
||||
return errors
|
||||
except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
|
||||
return [f"scanner/heal evidence rejected: {error}"]
|
||||
|
||||
|
||||
def validate(root: Path) -> list[str]:
|
||||
errors: list[str] = []
|
||||
errors.extend(check_core_fixtures(root))
|
||||
@@ -1022,6 +1207,213 @@ class SelfTests(unittest.TestCase):
|
||||
with mock.patch(__name__ + ".check_quick_checks", return_value=[error]):
|
||||
self.assertIn(error, validate(ROOT))
|
||||
|
||||
def scanner_heal_fixture(self, directory: Path) -> tuple[Path, Path]:
|
||||
"""Parser fixtures only; these files are never runtime evidence."""
|
||||
root, run_dir = directory / "repo", directory / "run"
|
||||
(root / ".config").mkdir(parents=True)
|
||||
run_dir.mkdir()
|
||||
registry = read_json(ROOT / ".config/scanner-heal-required-tests.json")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
requirement = registry["cases"]["background-target-restart"]
|
||||
binary = directory / "fake-binary"
|
||||
binary.write_bytes(b"parser fixture, not a real build")
|
||||
binary.chmod(0o700)
|
||||
build = {"path": str(binary), "sha256": digest(binary)}
|
||||
write_json(run_dir / "run.json", {"schema": 1, "run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"binary_source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False,
|
||||
"lock_blob": "c" * 40, "features": "default"},
|
||||
"started_at": datetime.now(timezone.utc).timestamp() - 1,
|
||||
"binary": build, "test_binary": build})
|
||||
write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {
|
||||
"binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
|
||||
"testcases": {
|
||||
requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
|
||||
}}}})
|
||||
(run_dir / "junit.xml").write_text(
|
||||
f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
|
||||
f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/></testsuite></testsuites>')
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
"inline_data_fingerprint": None}
|
||||
obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
|
||||
"expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
|
||||
"expected_physical": physical, "physical": physical}
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
write_json(run_dir / "background-target-restart.json", {
|
||||
"schema": 1, "evidence": "process-restart", "case": "background-target-restart",
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
|
||||
"features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11,
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
})
|
||||
finish_scanner_heal_receipt(run_dir, 0)
|
||||
return root, run_dir
|
||||
|
||||
def test_scanner_heal_case_does_not_approve_pending_release(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), [])
|
||||
errors = check_scanner_heal_evidence(root, run_dir, "release")
|
||||
self.assertEqual(len(errors), 21)
|
||||
self.assertTrue(any(error.startswith("pending R-E:") for error in errors))
|
||||
self.assertTrue(any(error.startswith("pending R-D:") for error in errors))
|
||||
self.assertTrue(any(error.startswith("pending R-L:") for error in errors))
|
||||
|
||||
def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
|
||||
for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
|
||||
"hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-restart.json"
|
||||
oracle = read_json(path)
|
||||
if fault == "exit":
|
||||
receipt = read_json(run_dir / "execution.json")
|
||||
receipt["exit_code"] = 42
|
||||
write_json(run_dir / "execution.json", receipt)
|
||||
elif fault == "missing":
|
||||
path.unlink()
|
||||
elif fault == "zero":
|
||||
(run_dir / "junit.xml").write_text("<testsuites/>")
|
||||
elif fault in ("skipped", "failed", "retry"):
|
||||
junit = run_dir / "junit.xml"
|
||||
tag = {"skipped": "skipped", "failed": "failure", "retry": "rerunFailure"}[fault]
|
||||
junit.write_text(junit.read_text().replace("/></testsuite>", f"><{tag}/></testcase></testsuite>"))
|
||||
elif fault in ("filtered", "ignored"):
|
||||
listing = read_json(run_dir / "listing.json")
|
||||
case = next(iter(listing["rust-suites"]["e2e_test"]["testcases"].values()))
|
||||
case["ignored"] = fault == "ignored"
|
||||
case["filter-match"]["status"] = "mismatch" if fault == "filtered" else "matches"
|
||||
write_json(run_dir / "listing.json", listing)
|
||||
elif fault == "stale":
|
||||
os.utime(path, (1, 1))
|
||||
elif fault == "hash":
|
||||
path.write_text(path.read_text() + " ")
|
||||
elif fault == "binary":
|
||||
Path(read_json(run_dir / "run.json")["binary"]["path"]).write_bytes(b"another build")
|
||||
else:
|
||||
if fault == "synthetic":
|
||||
oracle["evidence"] = "synthetic"
|
||||
elif fault == "wrong-run":
|
||||
oracle["run_id"] = "f" * 32
|
||||
elif fault == "same-pid":
|
||||
oracle["pid_after"] = oracle["pid_before"]
|
||||
elif fault == "body":
|
||||
oracle["objects"][0]["actual_sha256"] = "e" * 64
|
||||
elif fault == "parts":
|
||||
oracle["objects"][0]["physical"]["present_part_fingerprints"] = {}
|
||||
elif fault == "listing":
|
||||
oracle["node_listings"][0] = []
|
||||
elif fault == "topology":
|
||||
oracle["topology"] = {"nodes": 3, "drives_per_node": 4}
|
||||
write_json(path, oracle)
|
||||
if fault not in ("exit", "missing", "stale", "hash", "binary"):
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0)
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault)
|
||||
|
||||
def test_scanner_heal_receipts_reject_reuse_and_missing_builds(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
with self.assertRaisesRegex(ValueError, "already exists"):
|
||||
finish_scanner_heal_receipt(run_dir, 0)
|
||||
with self.assertRaisesRegex(ValueError, "must be new"):
|
||||
begin_scanner_heal_receipt(root, run_dir, Path("missing"), Path("missing"))
|
||||
with mock.patch("subprocess.check_output", side_effect=["", "b" * 40]):
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
begin_scanner_heal_receipt(root, Path(tmp) / "new-run", Path(tmp) / "missing", Path(tmp) / "missing")
|
||||
|
||||
def test_scanner_heal_begin_requires_embedded_source_provenance(self) -> None:
|
||||
for kind in ("current", "stale", "dirty", "unknown"):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
root, _ = self.scanner_heal_fixture(Path(tmp))
|
||||
sources = root / "crates/e2e_test/src"
|
||||
sources.mkdir(parents=True)
|
||||
(sources / "heal_erasure_disk_rebuild_test.rs").write_bytes(b"oracle source")
|
||||
(sources / "chaos.rs").write_bytes(b"census source")
|
||||
revision = "c" * 40 if kind == "stale" else "b" * 40
|
||||
version = f"rustfs\ngit commit : {revision}\ngit status :\n"
|
||||
if kind == "dirty":
|
||||
version += "modified source\n"
|
||||
if kind == "unknown":
|
||||
version = "rustfs without build provenance"
|
||||
with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version, "c" * 40]), \
|
||||
mock.patch.dict(os.environ, {"RUSTFS_E2E_EXPECTED_FEATURES": "default"}):
|
||||
directory = Path(tmp) / "fresh"
|
||||
binary = Path(tmp) / "fake-binary"
|
||||
if kind == "current":
|
||||
begin_scanner_heal_receipt(root, directory, binary, binary)
|
||||
self.assertEqual(read_json(directory / "run.json")["binary_source_revision"], "b" * 40)
|
||||
else:
|
||||
with self.assertRaisesRegex(ValueError, "server binary"):
|
||||
begin_scanner_heal_receipt(root, directory, binary, binary)
|
||||
self.assertFalse(directory.exists())
|
||||
|
||||
def test_scanner_heal_rejects_copied_junit_and_wrong_suite_build(self) -> None:
|
||||
for fault in ("old-junit", "missing-time", "wrong-binary", "common-source", "lockfile", "features", "dirty-build"):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
if fault in ("old-junit", "missing-time"):
|
||||
path = run_dir / "junit.xml"
|
||||
xml = ET.fromstring(path.read_bytes())
|
||||
testcase = next(xml.iter("testcase"))
|
||||
if fault == "old-junit":
|
||||
testcase.set("timestamp", "2000-01-01T00:00:00.000Z")
|
||||
else:
|
||||
del testcase.attrib["timestamp"]
|
||||
# Rewriting/copying gives an old execution a fresh mtime.
|
||||
path.write_bytes(ET.tostring(xml))
|
||||
elif fault == "wrong-binary":
|
||||
path = run_dir / "listing.json"
|
||||
listing = read_json(path)
|
||||
another = Path(tmp) / "another-binary"
|
||||
another.write_bytes(Path(tmp, "fake-binary").read_bytes())
|
||||
listing["rust-suites"]["e2e_test"]["binary-path"] = str(another)
|
||||
write_json(path, listing)
|
||||
else:
|
||||
path = run_dir / "background-target-restart.json"
|
||||
oracle = read_json(path)
|
||||
key, value = {"common-source": ("source_revision", "f" * 40), "lockfile": ("lock_blob", "f" * 40),
|
||||
"features": ("features", "default,sftp"), "dirty-build": ("dirty", True)}[fault]
|
||||
oracle["test_build"][key] = value
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0)
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault)
|
||||
|
||||
def test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers(self) -> None:
|
||||
valid = {"schema": 1, "nodes": 4, "drives_per_node": 1, "pid_before": 10, "pid_after": 11,
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2}
|
||||
cases = [(field, value) for field, correct in valid.items() for value in (True, float(correct))]
|
||||
cases += [("erasure_index", 5), ("erasure_index", 0), ("pid_after", -1)]
|
||||
for field, value in cases:
|
||||
with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-restart.json"
|
||||
oracle = read_json(path)
|
||||
if field in ("nodes", "drives_per_node"):
|
||||
oracle["topology"][field] = value
|
||||
elif field in ("erasure_index", "data_blocks", "parity_blocks"):
|
||||
oracle["objects"][-1]["physical"][field] = value
|
||||
else:
|
||||
oracle[field] = value
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0)
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
|
||||
for filename in ("run.json", ".config/scanner-heal-required-tests.json"):
|
||||
with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = (root if filename.startswith(".config") else run_dir) / filename
|
||||
content = read_json(path)
|
||||
content["schema"] = True
|
||||
write_json(path, content)
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
|
||||
|
||||
def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -1664,6 +2056,25 @@ def main() -> int:
|
||||
if sys.argv[1:] == ["--self-test"]:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
|
||||
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
|
||||
if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"]):
|
||||
try:
|
||||
if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal":
|
||||
begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]))
|
||||
return 0
|
||||
if len(sys.argv) == 4 and sys.argv[1] == "--finish-scanner-heal":
|
||||
finish_scanner_heal_receipt(Path(sys.argv[2]), int(sys.argv[3]))
|
||||
return 0
|
||||
if len(sys.argv) == 4 and sys.argv[1] == "--check-scanner-heal":
|
||||
errors = check_scanner_heal_evidence(ROOT, Path(sys.argv[2]), sys.argv[3])
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
if not errors:
|
||||
print(f"Case evidence verified: {sys.argv[3]}; this does not approve release")
|
||||
return 1 if errors else 0
|
||||
raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, or --check-scanner-heal DIR CASE|release")
|
||||
except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--check-core":
|
||||
errors = check_core_listing(ROOT, Path(sys.argv[2]))
|
||||
for error in errors:
|
||||
|
||||
+104
-29
@@ -8,6 +8,7 @@ import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import select
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
@@ -126,28 +127,110 @@ def validate_manifest(manifest):
|
||||
require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
|
||||
|
||||
|
||||
class OwnedCommand:
|
||||
"""Keep the session leader unreaped until its group's last signal is sent."""
|
||||
|
||||
def __init__(self, args, log):
|
||||
require(sys.platform == "darwin" or hasattr(os, "waitid"), "non-reaping child observation is unavailable")
|
||||
self.args, self.status = args, None
|
||||
self.queue = select.kqueue() if sys.platform == "darwin" else None
|
||||
self.process = None
|
||||
read_gate, write_gate = os.pipe()
|
||||
try:
|
||||
# The shell has already exec'd when Popen returns. Gate the target
|
||||
# until kqueue is registered; preexec_fn would deadlock Popen here.
|
||||
gate = f'read -r _scanner_gate <&{read_gate} || exit 125; exec {read_gate}<&-; exec "$@"'
|
||||
self.process = subprocess.Popen(["bash", "-c", gate, "scanner-abba", *args], pass_fds=(read_gate,),
|
||||
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
if self.queue is not None:
|
||||
# Darwin NOTE_EXITSTATUS is not exposed by Python's select constants.
|
||||
event = select.kevent(self.process.pid, filter=select.KQ_FILTER_PROC,
|
||||
flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
|
||||
fflags=select.KQ_NOTE_EXIT | 0x04000000)
|
||||
self.queue.control([event], 0, 0)
|
||||
os.write(write_gate, b"\n")
|
||||
except BaseException:
|
||||
try:
|
||||
if self.process is not None:
|
||||
try:
|
||||
self._signal_group(signal.SIGKILL)
|
||||
finally:
|
||||
self.process.wait(timeout=10)
|
||||
finally:
|
||||
if self.queue is not None:
|
||||
self.queue.close()
|
||||
raise
|
||||
finally:
|
||||
os.close(read_gate)
|
||||
os.close(write_gate)
|
||||
|
||||
def wait(self, timeout):
|
||||
if self.status is not None:
|
||||
return self.status
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(self.args, timeout)
|
||||
if self.queue is not None:
|
||||
events = self.queue.control(None, 1, remaining)
|
||||
if events:
|
||||
self.status = os.waitstatus_to_exitcode(events[0].data)
|
||||
return self.status
|
||||
else:
|
||||
result = os.waitid(os.P_PID, self.process.pid, os.WEXITED | os.WNOWAIT | os.WNOHANG)
|
||||
if result is not None:
|
||||
self.status = result.si_status if result.si_code == os.CLD_EXITED else -result.si_status
|
||||
return self.status
|
||||
time.sleep(min(0.05, remaining))
|
||||
|
||||
def _signal_group(self, sig):
|
||||
try:
|
||||
os.killpg(self.process.pid, sig)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
|
||||
def finish(self, terminate=False):
|
||||
if self.process.returncode is not None:
|
||||
return self.process.returncode
|
||||
try:
|
||||
if terminate:
|
||||
try:
|
||||
self._signal_group(signal.SIGTERM)
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline and self._signal_group(0):
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
# Keep the PID reserved through the last group signal, even
|
||||
# when the cleanup grace period itself is interrupted.
|
||||
self._signal_group(signal.SIGKILL)
|
||||
finally:
|
||||
try:
|
||||
returncode = self.process.wait(timeout=10)
|
||||
finally:
|
||||
if self.queue is not None:
|
||||
self.queue.close()
|
||||
return returncode
|
||||
|
||||
|
||||
def invoke(adapter, action, request, timeout):
|
||||
"""The adapter writes bounded JSON separately; stderr/stdout remain raw evidence."""
|
||||
output = request.parent / f"{action}.json"
|
||||
with (request.parent / f"{action}.log").open("wb") as log:
|
||||
process = subprocess.Popen([str(adapter), action, str(request), str(output)],
|
||||
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
process = OwnedCommand([str(adapter), action, str(request), str(output)], log)
|
||||
try:
|
||||
returncode = process.wait(timeout=timeout)
|
||||
returncode = process.wait(timeout)
|
||||
if returncode:
|
||||
raise subprocess.CalledProcessError(returncode, [str(adapter), action])
|
||||
finally:
|
||||
if process.poll() != 0:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
return read_json(output)
|
||||
result = read_json(output)
|
||||
except BaseException:
|
||||
process.finish(terminate=True)
|
||||
raise
|
||||
else:
|
||||
# Successful prepare may intentionally leave adapter-owned services.
|
||||
process.finish()
|
||||
return result
|
||||
|
||||
|
||||
def validate_result(result, request, expected):
|
||||
@@ -188,7 +271,7 @@ def convergence(result):
|
||||
require(window["full_walk_objects"] > 0, "zero full walk reference")
|
||||
require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"],
|
||||
"invalid convergence budget window")
|
||||
return window["walk_objects"] / window["full_walk_objects"]
|
||||
return ratio(window["walk_objects"], window["full_walk_objects"], "convergence work")
|
||||
|
||||
|
||||
def evaluate(cells):
|
||||
@@ -229,6 +312,7 @@ def evaluate(cells):
|
||||
candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")]
|
||||
p2_pending = any(value is None for value in candidate_p2)
|
||||
passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None)
|
||||
p2_report = [None if value is None else float(value) for value in p2]
|
||||
inconclusive |= noise or p2_pending
|
||||
if not noise and not passed:
|
||||
failed = True
|
||||
@@ -238,7 +322,7 @@ def evaluate(cells):
|
||||
"p99_regression": float(p99), "throughput_change": float(throughput),
|
||||
"thresholds": {key: float(value) for key, value in thresholds.items()},
|
||||
"p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT),
|
||||
"p2_post_stop_work_multiples": p2})
|
||||
"p2_post_stop_work_multiples": p2_report})
|
||||
return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons
|
||||
|
||||
|
||||
@@ -254,12 +338,12 @@ def collect_live(prepared, request, request_path, adapter):
|
||||
"--samples", str(request["duration_seconds"] // 60 + 1), "--interval-secs", "60",
|
||||
"--out-dir", str(output)]
|
||||
with (request_path.parent / "collector.log").open("wb") as log:
|
||||
process = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
process = OwnedCommand(args, log)
|
||||
try:
|
||||
started = time.monotonic()
|
||||
result = invoke(adapter, "measure", request_path, request["duration_seconds"] + 300)
|
||||
require(time.monotonic() - started >= request["duration_seconds"], "measurement ended before required window")
|
||||
require(process.wait(timeout=120) == 0, "scanner collector failed")
|
||||
require(process.wait(120) == 0, "scanner collector failed")
|
||||
require(output.joinpath("scanner-summary.csv").stat().st_size > 0, "missing collector samples")
|
||||
samples = list((output / "status").glob("scanner-status.*.json"))
|
||||
require(len(samples) == request["duration_seconds"] // 60 + 1, "missing scanner samples")
|
||||
@@ -286,16 +370,7 @@ def collect_live(prepared, request, request_path, adapter):
|
||||
"missing per-host scanner metrics")
|
||||
return result
|
||||
finally:
|
||||
# Stop telemetry children as well when measurement fails or times out.
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
process.finish(terminate=True)
|
||||
|
||||
|
||||
def run(manifest, adapter, output, data_root):
|
||||
|
||||
@@ -3,13 +3,17 @@
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import fcntl
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
@@ -20,9 +24,18 @@ def fake_adapter():
|
||||
action, request_path, output_path = sys.argv[1:]
|
||||
request = harness.read_json(Path(request_path))
|
||||
fault = os.environ.get("SCANNER_ABBA_TEST_FAULT", "")
|
||||
if fault == "stubborn-child" and action in ("prepare", "measure"):
|
||||
marker = Path(request_path).parent / "stubborn.pid"
|
||||
if os.fork() == 0:
|
||||
os.execv(sys.executable, [sys.executable, str(Path(__file__).resolve()), "--stubborn-worker", str(marker)])
|
||||
wait_for_marker(marker)
|
||||
if action == "measure":
|
||||
time.sleep(60)
|
||||
if action == "prepare":
|
||||
result = {"ready": True}
|
||||
elif action == "stop":
|
||||
if fault == "stubborn-child":
|
||||
reap_fixture(Path(request_path).parent / "stubborn.pid")
|
||||
result = {"stopped": True}
|
||||
elif action == "oracle":
|
||||
if fault == "oracle-exit":
|
||||
@@ -87,6 +100,36 @@ def fake_adapter():
|
||||
return 0
|
||||
|
||||
|
||||
def wait_for_marker(marker):
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if marker.exists() and marker.stat().st_size:
|
||||
return
|
||||
time.sleep(0.01)
|
||||
raise AssertionError("fixture child did not become ready")
|
||||
|
||||
|
||||
def child_released(marker, timeout=1):
|
||||
deadline = time.monotonic() + timeout
|
||||
with marker.open("r+") as stream:
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return True
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
return False
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
def reap_fixture(marker):
|
||||
if marker.exists() and marker.stat().st_size and not child_released(marker, timeout=0):
|
||||
# The unique file lock proves the original fixture process still owns this PID.
|
||||
os.kill(int(marker.read_text()), signal.SIGKILL)
|
||||
if not child_released(marker, timeout=5):
|
||||
raise AssertionError("fixture child did not release its process-owned lock")
|
||||
|
||||
|
||||
class ScannerAbbaTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
@@ -110,6 +153,112 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
|
||||
return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
|
||||
|
||||
def test_adapter_timeout_reaps_group_after_parent_exits_on_term(self):
|
||||
request = self.root / "request.json"
|
||||
harness.write_json(request, {})
|
||||
marker = self.root / "stubborn.pid"
|
||||
try:
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
harness.invoke(self.adapter, "measure", request, 3)
|
||||
wait_for_marker(marker)
|
||||
self.assertTrue(child_released(marker), "TERM-exited parent left its TERM-ignoring child alive")
|
||||
finally:
|
||||
reap_fixture(marker)
|
||||
|
||||
def test_collector_failure_reaps_group_after_parent_exits_on_term(self):
|
||||
request = self.root / "request.json"
|
||||
harness.write_json(request, {})
|
||||
marker = self.root / "stubborn.pid"
|
||||
collector = self.root / "run_scanner_validation_harness.sh"
|
||||
command = [sys.executable, str(self.adapter), "measure", str(request), str(self.root / "unused.json")]
|
||||
collector.write_text("#!/usr/bin/env bash\nexec " + shlex.join(command) + "\n")
|
||||
|
||||
def failed_measure(*_):
|
||||
wait_for_marker(marker)
|
||||
raise ValueError("injected measurement failure")
|
||||
|
||||
try:
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \
|
||||
patch.object(harness, "__file__", str(self.root / "scanner_abba.py")), \
|
||||
patch.object(harness, "invoke", side_effect=failed_measure):
|
||||
with self.assertRaisesRegex(ValueError, "injected measurement failure"):
|
||||
harness.collect_live({"collector": {"alias": "fixture", "endpoint": "fixture", "metrics_endpoints": "fixture"}},
|
||||
{"duration_seconds": 900}, request, self.adapter)
|
||||
self.assertTrue(child_released(marker), "collector parent exit did not end its telemetry child")
|
||||
finally:
|
||||
reap_fixture(marker)
|
||||
|
||||
def test_successful_prepare_keeps_service_alive(self):
|
||||
request = self.root / "request.json"
|
||||
harness.write_json(request, {})
|
||||
marker = self.root / "stubborn.pid"
|
||||
try:
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
|
||||
self.assertEqual(harness.invoke(self.adapter, "prepare", request, 5), {"ready": True})
|
||||
self.assertFalse(child_released(marker, timeout=0), "successful prepare must preserve its service")
|
||||
self.assertEqual(harness.invoke(self.adapter, "stop", request, 5), {"stopped": True})
|
||||
self.assertTrue(child_released(marker), "adapter stop must release its service")
|
||||
finally:
|
||||
reap_fixture(marker)
|
||||
|
||||
def test_reaped_owner_never_signals_a_reused_process_group(self):
|
||||
with (self.root / "owner.log").open("wb") as log:
|
||||
owner = harness.OwnedCommand([sys.executable, "-c", "pass"], log)
|
||||
self.assertEqual(owner.wait(5), 0)
|
||||
self.assertEqual(owner.finish(), 0)
|
||||
with patch.object(harness.os, "killpg", side_effect=AssertionError("released PGID must not be signalled")):
|
||||
self.assertEqual(owner.finish(terminate=True), 0)
|
||||
|
||||
def test_cleanup_interruption_still_kills_group_and_reaps_leader(self):
|
||||
request = self.root / "request.json"
|
||||
harness.write_json(request, {})
|
||||
marker = self.root / "stubborn.pid"
|
||||
original_sleep = time.sleep
|
||||
interrupted = False
|
||||
|
||||
def interrupt_once(delay):
|
||||
nonlocal interrupted
|
||||
if not interrupted:
|
||||
interrupted = True
|
||||
raise KeyboardInterrupt
|
||||
original_sleep(delay)
|
||||
|
||||
with (self.root / "interrupted.log").open("wb") as log:
|
||||
with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
|
||||
owner = harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log)
|
||||
try:
|
||||
wait_for_marker(marker)
|
||||
with patch.object(harness.time, "sleep", side_effect=interrupt_once):
|
||||
with self.assertRaises(KeyboardInterrupt):
|
||||
owner.finish(terminate=True)
|
||||
self.assertTrue(child_released(marker), "cleanup cancellation left its child alive")
|
||||
self.assertIsNotNone(owner.process.returncode, "cleanup cancellation must reap its leader")
|
||||
finally:
|
||||
reap_fixture(marker)
|
||||
owner.process.wait(timeout=5)
|
||||
|
||||
def test_constructor_failure_after_gate_release_kills_group(self):
|
||||
request = self.root / "request.json"
|
||||
harness.write_json(request, {})
|
||||
marker = self.root / "stubborn.pid"
|
||||
original_write = os.write
|
||||
|
||||
def release_then_fail(fd, data):
|
||||
original_write(fd, data)
|
||||
wait_for_marker(marker)
|
||||
raise OSError("injected failure after gate release")
|
||||
|
||||
try:
|
||||
with (self.root / "construction.log").open("wb") as log, \
|
||||
patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \
|
||||
patch.object(harness.os, "write", side_effect=release_then_fail):
|
||||
with self.assertRaisesRegex(OSError, "injected failure after gate release"):
|
||||
harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log)
|
||||
self.assertTrue(child_released(marker), "initialization failure left its child alive")
|
||||
finally:
|
||||
reap_fixture(marker)
|
||||
|
||||
def test_complete_synthetic_matrix_is_not_performance_evidence(self):
|
||||
self.assertEqual(self.run_harness(), 0)
|
||||
report = harness.read_json(self.root / "out/report.json")
|
||||
@@ -201,10 +350,9 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
else:
|
||||
harness.write_json(sample, payload)
|
||||
process = Mock(pid=123, wait=Mock(return_value=1 if name == "collector-exit" else 0))
|
||||
with patch.object(harness.subprocess, "Popen", return_value=process), \
|
||||
with patch.object(harness, "OwnedCommand", return_value=process), \
|
||||
patch.object(harness, "invoke", return_value={"sample_count": 10}), \
|
||||
patch.object(harness.time, "monotonic", side_effect=(0, 900)), \
|
||||
patch.object(harness.os, "killpg"):
|
||||
patch.object(harness.time, "monotonic", side_effect=(0, 900)):
|
||||
if error:
|
||||
with self.assertRaisesRegex(ValueError, error):
|
||||
harness.collect_live(prepared, {"duration_seconds": 900}, self.root / "request.json", self.adapter)
|
||||
@@ -212,6 +360,8 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
self.assertEqual(harness.collect_live(prepared, {"duration_seconds": 900},
|
||||
self.root / "request.json", self.adapter), {"sample_count": 10})
|
||||
|
||||
process.finish.assert_called_once_with(terminate=True)
|
||||
|
||||
def test_unstable_p1_work_control_is_inconclusive(self):
|
||||
with patch.object(harness, "SCENARIOS", ("cold-hot",)):
|
||||
self.assertEqual(self.run_harness("unstable-p1-control"), 3)
|
||||
@@ -259,6 +409,14 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--stubborn-worker":
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
with Path(sys.argv[2]).open("w+") as marker:
|
||||
fcntl.flock(marker, fcntl.LOCK_EX)
|
||||
marker.write(str(os.getpid()))
|
||||
marker.flush()
|
||||
while True:
|
||||
time.sleep(1)
|
||||
if len(sys.argv) == 4 and sys.argv[1] in ("prepare", "measure", "oracle", "stop"):
|
||||
sys.exit(fake_adapter())
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user