From 45f57ca57a200dc76c1d2955e10f6b095c8cc2a2 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 7 Sep 2026 19:54:57 +0800 Subject: [PATCH] test(e2e): allow scanner heal evidence port ranges (#7394) * test(scanner): wire crash evidence runner Add the background target crash release-evidence case and require its oracle to carry process-crash-restart evidence with the unclean shutdown marker. Add a single-case runner that records begin/list/run/finish receipts, validates the concrete case, and confirms the release gate remains blocked by pending mixed-version and release lanes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(scanner): align crash evidence feature identity Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(scanner): harden crash evidence runner Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(e2e): allow scanner heal evidence port ranges Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 --- .config/make/tests.mak | 1 + crates/e2e_test/src/common.rs | 108 ++++++++-- scripts/README.md | 1 + scripts/run_scanner_heal_evidence_case.sh | 234 ++++++++++++++++++++++ 4 files changed, 332 insertions(+), 12 deletions(-) create mode 100755 scripts/run_scanner_heal_evidence_case.sh diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 7d29ad6fc..99eb76f10 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -47,6 +47,7 @@ script-tests: ## Run shell script tests bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test + ./scripts/run_scanner_heal_evidence_case.sh --self-test .PHONY: test test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override) diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index daa744ab8..554f55239 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -57,6 +57,8 @@ const RUSTFS_FULL_FEATURE: &str = "full"; const TEST_PORT_MIN: u16 = 20_000; // Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers. const TEST_PORT_RANGE: u16 = 10_000; +const TEST_PORT_MIN_ENV: &str = "RUSTFS_E2E_TEST_PORT_MIN"; +const TEST_PORT_RANGE_ENV: &str = "RUSTFS_E2E_TEST_PORT_RANGE"; const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port"; const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock"; const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30); @@ -99,22 +101,74 @@ impl Drop for PortAllocatorGuard { } } -fn advance_test_port(port: u16) -> u16 { - let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE; - TEST_PORT_MIN + offset +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct TestPortAllocatorConfig { + min: u16, + range: u16, } -fn seeded_test_port() -> u16 { - let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16; - TEST_PORT_MIN + offset +impl TestPortAllocatorConfig { + fn max_exclusive(self) -> u32 { + u32::from(self.min) + u32::from(self.range) + } + + fn contains(self, port: &u16) -> bool { + (u32::from(self.min)..self.max_exclusive()).contains(&u32::from(*port)) + } } -fn read_next_test_port() -> u16 { +fn parse_test_port_allocator_config( + min_override: Option<&str>, + range_override: Option<&str>, +) -> Result> { + let min = match min_override { + Some(value) => value + .parse::() + .map_err(|err| format!("{TEST_PORT_MIN_ENV} must be a valid u16: {err}"))?, + None => TEST_PORT_MIN, + }; + let range = match range_override { + Some(value) => value + .parse::() + .map_err(|err| format!("{TEST_PORT_RANGE_ENV} must be a valid u16: {err}"))?, + None => TEST_PORT_RANGE, + }; + if range == 0 { + return Err(format!("{TEST_PORT_RANGE_ENV} must be greater than zero").into()); + } + if min < 1024 { + return Err(format!("{TEST_PORT_MIN_ENV} must be at least 1024").into()); + } + let max_exclusive = u32::from(min) + u32::from(range); + if max_exclusive > u32::from(u16::MAX) + 1 { + return Err(format!("{TEST_PORT_MIN_ENV} + {TEST_PORT_RANGE_ENV} exceeds u16 port space").into()); + } + Ok(TestPortAllocatorConfig { min, range }) +} + +fn test_port_allocator_config() -> Result> { + parse_test_port_allocator_config( + std::env::var(TEST_PORT_MIN_ENV).ok().as_deref(), + std::env::var(TEST_PORT_RANGE_ENV).ok().as_deref(), + ) +} + +fn advance_test_port(port: u16, config: TestPortAllocatorConfig) -> u16 { + let offset = (port - config.min + 1) % config.range; + config.min + offset +} + +fn seeded_test_port(config: TestPortAllocatorConfig) -> u16 { + let offset = (Uuid::new_v4().as_u128() % u128::from(config.range)) as u16; + config.min + offset +} + +fn read_next_test_port(config: TestPortAllocatorConfig) -> u16 { stdfs::read_to_string(TEST_PORT_COUNTER_PATH) .ok() .and_then(|value| value.trim().parse::().ok()) - .filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port)) - .unwrap_or_else(seeded_test_port) + .filter(|port| config.contains(port)) + .unwrap_or_else(|| seeded_test_port(config)) } fn remove_stale_port_allocator_lock() { @@ -629,11 +683,12 @@ impl RustFSTestEnvironment { pub async fn find_available_port() -> Result> { use std::net::TcpListener; let _guard = PortAllocatorGuard::acquire().await?; - let mut next_port = read_next_test_port(); + let config = test_port_allocator_config()?; + let mut next_port = read_next_test_port(config); - for _ in 0..TEST_PORT_RANGE { + for _ in 0..config.range { let port = next_port; - next_port = advance_test_port(next_port); + next_port = advance_test_port(next_port, config); write_next_test_port(next_port)?; if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { @@ -2108,6 +2163,35 @@ mod tests { ); } + #[test] + fn e2e_port_allocator_uses_default_range() { + assert_eq!( + parse_test_port_allocator_config(None, None).expect("default port allocator config"), + TestPortAllocatorConfig { + min: TEST_PORT_MIN, + range: TEST_PORT_RANGE + } + ); + } + + #[test] + fn e2e_port_allocator_accepts_explicit_test_range() { + let config = parse_test_port_allocator_config(Some("31000"), Some("128")).expect("explicit port range"); + + assert_eq!(advance_test_port(31127, config), 31000); + assert!(config.contains(&31000)); + assert!(config.contains(&31127)); + assert!(!config.contains(&31128)); + } + + #[test] + fn e2e_port_allocator_rejects_invalid_override() { + assert!(parse_test_port_allocator_config(Some("1023"), Some("1")).is_err()); + assert!(parse_test_port_allocator_config(Some("65000"), Some("1000")).is_err()); + assert!(parse_test_port_allocator_config(Some("31000"), Some("0")).is_err()); + assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err()); + } + #[test] fn resolves_rustfs_binary_in_configured_cargo_target_directory() { let workspace = Path::new("workspace"); diff --git a/scripts/README.md b/scripts/README.md index 25c45c863..50f8ea7a8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -55,6 +55,7 @@ their issue closes. | `run.ps1` | dev-tool | Windows counterpart of `run.sh` | — | | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` | | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | +| `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | | `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` | | `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` | diff --git a/scripts/run_scanner_heal_evidence_case.sh b/scripts/run_scanner_heal_evidence_case.sh new file mode 100755 index 000000000..eb3ef240f --- /dev/null +++ b/scripts/run_scanner_heal_evidence_case.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" +PROFILE="e2e-nightly" +CASE_ID="background-target-crash" +RUN_DIR="" +PLAN_ONLY=0 + +usage() { + cat <<'USAGE' +Usage: scripts/run_scanner_heal_evidence_case.sh [OPTIONS] + +Run one Scanner/Heal release-evidence case through the real e2e test binary, +then validate the produced receipt, nextest listing, JUnit, and case oracle. + +Options: + --case CASE Registry case to run (default: background-target-crash) + --profile PROFILE Nextest profile to use (default: e2e-nightly) + --run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP) + --plan-only Validate registry selection and print the exact filter without running cargo + --self-test Run lightweight CLI/registry checks without building Rust + -h, --help Show this help + +The script intentionally runs a single case, not the release pseudo-case. After +a successful case run it verifies that the release gate still remains blocked. +Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e +port allocator when the default 20000..30000 test range is unavailable. +USAGE +} + +case_field() { + local case_id="$1" + local field="$2" + "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" "$field" <<'PY' +import json +import pathlib +import sys + +registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) +case = registry["cases"][sys.argv[2]] +value = case[sys.argv[3]] +if not isinstance(value, str): + raise SystemExit(f"{sys.argv[3]} is not a string") +print(value) +PY +} + +test_filter_for() { + local case_id="$1" + "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" <<'PY' +import json +import pathlib +import re +import sys + +registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) +case = registry["cases"][sys.argv[2]] +print("test(/^" + re.escape(case["name"]) + "$/)") +PY +} + +test_binary_from_listing() { + local listing="$1" + local case_id="$2" + "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$listing" "$case_id" <<'PY' +import json +import pathlib +import sys + +registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) +listing = json.loads(pathlib.Path(sys.argv[2]).read_text()) +case = registry["cases"][sys.argv[3]] +suite = listing["rust-suites"][case["suite"]] +testcase = suite["testcases"][case["name"]] +if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches": + raise SystemExit("selected case is not matched by the nextest listing") +matches = 0 +for listed_suite in listing.get("rust-suites", {}).values(): + for listed in listed_suite.get("testcases", {}).values(): + if listed.get("filter-match", {}).get("status") == "matches": + matches += 1 +if matches != 1: + raise SystemExit(f"expected exactly one selected case, got {matches}") +print(suite["binary-path"]) +PY +} + +release_gate_must_remain_blocked() { + local run_dir="$1" + local output="$run_dir/release-check.txt" + if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$run_dir" release >"$output" 2>&1; then + echo "release gate unexpectedly approved a single Scanner/Heal evidence run" >&2 + return 1 + fi + if ! grep -Eq 'required test not selected:|pending [A-Z0-9-]+:' "$output"; then + echo "release gate did not explain why the Scanner/Heal release remains blocked" >&2 + return 1 + fi +} + +run_self_test() { + local filter + filter="$(test_filter_for background-target-crash)" + case "$filter" in + *background_target_crash*) ;; + *) + echo "self-test failed: crash case filter missing" >&2 + return 1 + ;; + esac + if "$0" --case release --plan-only >/dev/null 2>&1; then + echo "self-test failed: release pseudo-case must not be runnable" >&2 + return 1 + fi + "$0" --case background-target-crash --plan-only >/dev/null +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --case) + CASE_ID="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --run-dir) + RUN_DIR="$2" + shift 2 + ;; + --plan-only) + PLAN_ONLY=1 + shift + ;; + --self-test) + run_self_test + exit $? + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ "$CASE_ID" == "release" ]]; then + echo "release is a checker-only pseudo-case; run a concrete registry case" >&2 + exit 2 +fi + +case_field "$CASE_ID" name >/dev/null +TEST_FILTER="$(test_filter_for "$CASE_ID")" +if [[ -z "$RUN_DIR" ]]; then + RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)" +elif [[ "$RUN_DIR" != /* ]]; then + RUN_DIR="$ROOT/$RUN_DIR" +fi + +if [[ "$PLAN_ONLY" == 1 ]]; then + echo "case=$CASE_ID" + echo "profile=$PROFILE" + echo "filter=$TEST_FILTER" + echo "run_dir=$RUN_DIR" + exit 0 +fi + +if [[ -e "$RUN_DIR" ]]; then + echo "evidence run directory already exists: $RUN_DIR" >&2 + exit 1 +fi + +cd "$ROOT" +if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then + echo "commit tracked source changes before creating evidence" >&2 + exit 1 +fi +mkdir -p "$(dirname "$RUN_DIR")" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}" +cargo clean -p rustfs +if [[ -n "$BUILD_FEATURES" ]]; then + cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES" +else + cargo build --locked -p rustfs --bins +fi +printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features" + +LISTING_TMP="$TMP_DIR/listing.json" +cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-format json >"$LISTING_TMP" +TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")" + +export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}" +"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY" +cp "$LISTING_TMP" "$RUN_DIR/listing.json" +export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}" +mkdir -p "$RUSTFS_E2E_LOG_DIR" + +JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml" +rm -f "$JUNIT_PATH" +set +e +NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ +HTTP_PROXY= \ +HTTPS_PROXY= \ +RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" \ +cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=fail +STATUS=$? +set -e + +if [[ -f "$JUNIT_PATH" ]]; then + cp "$JUNIT_PATH" "$RUN_DIR/junit.xml" +fi +"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS" + +if [[ "$STATUS" -ne 0 ]]; then + echo "Scanner/Heal evidence case failed: $CASE_ID (exit $STATUS); receipt kept at $RUN_DIR" >&2 + exit "$STATUS" +fi + +"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$RUN_DIR" "$CASE_ID" +release_gate_must_remain_blocked "$RUN_DIR" +echo "Scanner/Heal evidence case verified: $CASE_ID" +echo "Release gate remains blocked; details: $RUN_DIR/release-check.txt" +echo "Evidence directory: $RUN_DIR"