test(e2e): allow scanner heal evidence port ranges (#7394)

* test(scanner): wire crash evidence runner

Add the background target crash release-evidence case and require its oracle to carry process-crash-restart evidence with the unclean shutdown marker.

Add a single-case runner that records begin/list/run/finish receipts, validates the concrete case, and confirms the release gate remains blocked by pending mixed-version and release lanes.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(scanner): align crash evidence feature identity

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(scanner): harden crash evidence runner

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(e2e): allow scanner heal evidence port ranges

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.config/make/tests.mak

@@ -47,6 +47,7 @@ script-tests: ## Run shell script tests
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
+	./scripts/run_scanner_heal_evidence_case.sh --self-test
 
 .PHONY: test
 test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override)

crates/e2e_test/src/common.rs

@@ -57,6 +57,8 @@ const RUSTFS_FULL_FEATURE: &str = "full";
 const TEST_PORT_MIN: u16 = 20_000;
 // Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
 const TEST_PORT_RANGE: u16 = 10_000;
+const TEST_PORT_MIN_ENV: &str = "RUSTFS_E2E_TEST_PORT_MIN";
+const TEST_PORT_RANGE_ENV: &str = "RUSTFS_E2E_TEST_PORT_RANGE";
 const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
 const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
 const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
@@ -99,22 +101,74 @@ impl Drop for PortAllocatorGuard {
     }
 }
 
-fn advance_test_port(port: u16) -> u16 {
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-    let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
+struct TestPortAllocatorConfig {
-    TEST_PORT_MIN + offset
+    min: u16,
+    range: u16,
 }
 
-fn seeded_test_port() -> u16 {
+impl TestPortAllocatorConfig {
-    let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
+    fn max_exclusive(self) -> u32 {
-    TEST_PORT_MIN + offset
+        u32::from(self.min) + u32::from(self.range)
+    }
+
+    fn contains(self, port: &u16) -> bool {
+        (u32::from(self.min)..self.max_exclusive()).contains(&u32::from(*port))
+    }
 }
 
-fn read_next_test_port() -> u16 {
+fn parse_test_port_allocator_config(
+    min_override: Option<&str>,
+    range_override: Option<&str>,
+) -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
+    let min = match min_override {
+        Some(value) => value
+            .parse::<u16>()
+            .map_err(|err| format!("{TEST_PORT_MIN_ENV} must be a valid u16: {err}"))?,
+        None => TEST_PORT_MIN,
+    };
+    let range = match range_override {
+        Some(value) => value
+            .parse::<u16>()
+            .map_err(|err| format!("{TEST_PORT_RANGE_ENV} must be a valid u16: {err}"))?,
+        None => TEST_PORT_RANGE,
+    };
+    if range == 0 {
+        return Err(format!("{TEST_PORT_RANGE_ENV} must be greater than zero").into());
+    }
+    if min < 1024 {
+        return Err(format!("{TEST_PORT_MIN_ENV} must be at least 1024").into());
+    }
+    let max_exclusive = u32::from(min) + u32::from(range);
+    if max_exclusive > u32::from(u16::MAX) + 1 {
+        return Err(format!("{TEST_PORT_MIN_ENV} + {TEST_PORT_RANGE_ENV} exceeds u16 port space").into());
+    }
+    Ok(TestPortAllocatorConfig { min, range })
+}
+
+fn test_port_allocator_config() -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
+    parse_test_port_allocator_config(
+        std::env::var(TEST_PORT_MIN_ENV).ok().as_deref(),
+        std::env::var(TEST_PORT_RANGE_ENV).ok().as_deref(),
+    )
+}
+
+fn advance_test_port(port: u16, config: TestPortAllocatorConfig) -> u16 {
+    let offset = (port - config.min + 1) % config.range;
+    config.min + offset
+}
+
+fn seeded_test_port(config: TestPortAllocatorConfig) -> u16 {
+    let offset = (Uuid::new_v4().as_u128() % u128::from(config.range)) as u16;
+    config.min + offset
+}
+
+fn read_next_test_port(config: TestPortAllocatorConfig) -> u16 {
     stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
         .ok()
         .and_then(|value| value.trim().parse::<u16>().ok())
-        .filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
+        .filter(|port| config.contains(port))
-        .unwrap_or_else(seeded_test_port)
+        .unwrap_or_else(|| seeded_test_port(config))
 }
 
 fn remove_stale_port_allocator_lock() {
@@ -629,11 +683,12 @@ impl RustFSTestEnvironment {
     pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
         use std::net::TcpListener;
         let _guard = PortAllocatorGuard::acquire().await?;
-        let mut next_port = read_next_test_port();
+        let config = test_port_allocator_config()?;
+        let mut next_port = read_next_test_port(config);
 
-        for _ in 0..TEST_PORT_RANGE {
+        for _ in 0..config.range {
             let port = next_port;
-            next_port = advance_test_port(next_port);
+            next_port = advance_test_port(next_port, config);
             write_next_test_port(next_port)?;
 
             if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
@@ -2108,6 +2163,35 @@ mod tests {
         );
     }
 
+    #[test]
+    fn e2e_port_allocator_uses_default_range() {
+        assert_eq!(
+            parse_test_port_allocator_config(None, None).expect("default port allocator config"),
+            TestPortAllocatorConfig {
+                min: TEST_PORT_MIN,
+                range: TEST_PORT_RANGE
+            }
+        );
+    }
+
+    #[test]
+    fn e2e_port_allocator_accepts_explicit_test_range() {
+        let config = parse_test_port_allocator_config(Some("31000"), Some("128")).expect("explicit port range");
+
+        assert_eq!(advance_test_port(31127, config), 31000);
+        assert!(config.contains(&31000));
+        assert!(config.contains(&31127));
+        assert!(!config.contains(&31128));
+    }
+
+    #[test]
+    fn e2e_port_allocator_rejects_invalid_override() {
+        assert!(parse_test_port_allocator_config(Some("1023"), Some("1")).is_err());
+        assert!(parse_test_port_allocator_config(Some("65000"), Some("1000")).is_err());
+        assert!(parse_test_port_allocator_config(Some("31000"), Some("0")).is_err());
+        assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err());
+    }
+
     #[test]
     fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
         let workspace = Path::new("workspace");

scripts/README.md

@@ -55,6 +55,7 @@ their issue closes.
 | `run.ps1` | dev-tool | Windows counterpart of `run.sh` | — |
 | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` |
 | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` |
+| `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` |
 | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — |
 | `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` |
 | `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` |

scripts/run_scanner_heal_evidence_case.sh

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}"
+PROFILE="e2e-nightly"
+CASE_ID="background-target-crash"
+RUN_DIR=""
+PLAN_ONLY=0
+
+usage() {
+    cat <<'USAGE'
+Usage: scripts/run_scanner_heal_evidence_case.sh [OPTIONS]
+
+Run one Scanner/Heal release-evidence case through the real e2e test binary,
+then validate the produced receipt, nextest listing, JUnit, and case oracle.
+
+Options:
+  --case CASE          Registry case to run (default: background-target-crash)
+  --profile PROFILE   Nextest profile to use (default: e2e-nightly)
+  --run-dir DIR       New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP)
+  --plan-only         Validate registry selection and print the exact filter without running cargo
+  --self-test         Run lightweight CLI/registry checks without building Rust
+  -h, --help          Show this help
+
+The script intentionally runs a single case, not the release pseudo-case. After
+a successful case run it verifies that the release gate still remains blocked.
+Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e
+port allocator when the default 20000..30000 test range is unavailable.
+USAGE
+}
+
+case_field() {
+    local case_id="$1"
+    local field="$2"
+    "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" "$field" <<'PY'
+import json
+import pathlib
+import sys
+
+registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
+case = registry["cases"][sys.argv[2]]
+value = case[sys.argv[3]]
+if not isinstance(value, str):
+    raise SystemExit(f"{sys.argv[3]} is not a string")
+print(value)
+PY
+}
+
+test_filter_for() {
+    local case_id="$1"
+    "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" <<'PY'
+import json
+import pathlib
+import re
+import sys
+
+registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
+case = registry["cases"][sys.argv[2]]
+print("test(/^" + re.escape(case["name"]) + "$/)")
+PY
+}
+
+test_binary_from_listing() {
+    local listing="$1"
+    local case_id="$2"
+    "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$listing" "$case_id" <<'PY'
+import json
+import pathlib
+import sys
+
+registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
+listing = json.loads(pathlib.Path(sys.argv[2]).read_text())
+case = registry["cases"][sys.argv[3]]
+suite = listing["rust-suites"][case["suite"]]
+testcase = suite["testcases"][case["name"]]
+if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches":
+    raise SystemExit("selected case is not matched by the nextest listing")
+matches = 0
+for listed_suite in listing.get("rust-suites", {}).values():
+    for listed in listed_suite.get("testcases", {}).values():
+        if listed.get("filter-match", {}).get("status") == "matches":
+            matches += 1
+if matches != 1:
+    raise SystemExit(f"expected exactly one selected case, got {matches}")
+print(suite["binary-path"])
+PY
+}
+
+release_gate_must_remain_blocked() {
+    local run_dir="$1"
+    local output="$run_dir/release-check.txt"
+    if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$run_dir" release >"$output" 2>&1; then
+        echo "release gate unexpectedly approved a single Scanner/Heal evidence run" >&2
+        return 1
+    fi
+    if ! grep -Eq 'required test not selected:|pending [A-Z0-9-]+:' "$output"; then
+        echo "release gate did not explain why the Scanner/Heal release remains blocked" >&2
+        return 1
+    fi
+}
+
+run_self_test() {
+    local filter
+    filter="$(test_filter_for background-target-crash)"
+    case "$filter" in
+        *background_target_crash*) ;;
+        *)
+            echo "self-test failed: crash case filter missing" >&2
+            return 1
+            ;;
+    esac
+    if "$0" --case release --plan-only >/dev/null 2>&1; then
+        echo "self-test failed: release pseudo-case must not be runnable" >&2
+        return 1
+    fi
+    "$0" --case background-target-crash --plan-only >/dev/null
+}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --case)
+            CASE_ID="$2"
+            shift 2
+            ;;
+        --profile)
+            PROFILE="$2"
+            shift 2
+            ;;
+        --run-dir)
+            RUN_DIR="$2"
+            shift 2
+            ;;
+        --plan-only)
+            PLAN_ONLY=1
+            shift
+            ;;
+        --self-test)
+            run_self_test
+            exit $?
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        *)
+            echo "unknown option: $1" >&2
+            usage >&2
+            exit 2
+            ;;
+    esac
+done
+
+if [[ "$CASE_ID" == "release" ]]; then
+    echo "release is a checker-only pseudo-case; run a concrete registry case" >&2
+    exit 2
+fi
+
+case_field "$CASE_ID" name >/dev/null
+TEST_FILTER="$(test_filter_for "$CASE_ID")"
+if [[ -z "$RUN_DIR" ]]; then
+    RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)"
+elif [[ "$RUN_DIR" != /* ]]; then
+    RUN_DIR="$ROOT/$RUN_DIR"
+fi
+
+if [[ "$PLAN_ONLY" == 1 ]]; then
+    echo "case=$CASE_ID"
+    echo "profile=$PROFILE"
+    echo "filter=$TEST_FILTER"
+    echo "run_dir=$RUN_DIR"
+    exit 0
+fi
+
+if [[ -e "$RUN_DIR" ]]; then
+    echo "evidence run directory already exists: $RUN_DIR" >&2
+    exit 1
+fi
+
+cd "$ROOT"
+if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
+    echo "commit tracked source changes before creating evidence" >&2
+    exit 1
+fi
+mkdir -p "$(dirname "$RUN_DIR")"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
+cargo clean -p rustfs
+if [[ -n "$BUILD_FEATURES" ]]; then
+    cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES"
+else
+    cargo build --locked -p rustfs --bins
+fi
+printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features"
+
+LISTING_TMP="$TMP_DIR/listing.json"
+cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-format json >"$LISTING_TMP"
+TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")"
+
+export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}"
+"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY"
+cp "$LISTING_TMP" "$RUN_DIR/listing.json"
+export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}"
+mkdir -p "$RUSTFS_E2E_LOG_DIR"
+
+JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml"
+rm -f "$JUNIT_PATH"
+set +e
+NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
+HTTP_PROXY= \
+HTTPS_PROXY= \
+RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" \
+cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=fail
+STATUS=$?
+set -e
+
+if [[ -f "$JUNIT_PATH" ]]; then
+    cp "$JUNIT_PATH" "$RUN_DIR/junit.xml"
+fi
+"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS"
+
+if [[ "$STATUS" -ne 0 ]]; then
+    echo "Scanner/Heal evidence case failed: $CASE_ID (exit $STATUS); receipt kept at $RUN_DIR" >&2
+    exit "$STATUS"
+fi
+
+"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$RUN_DIR" "$CASE_ID"
+release_gate_must_remain_blocked "$RUN_DIR"
+echo "Scanner/Heal evidence case verified: $CASE_ID"
+echo "Release gate remains blocked; details: $RUN_DIR/release-check.txt"
+echo "Evidence directory: $RUN_DIR"