#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" PROFILE="" CASE_ID="background-target-crash" RUN_DIR="" PLAN_ONLY=0 usage() { cat <<'USAGE' Usage: scripts/run_scanner_heal_evidence_case.sh [OPTIONS] Run one Scanner/Heal release-evidence case through the real e2e test binary, then validate the produced receipt, nextest listing, JUnit, and case oracle. Options: --case CASE Registry case to run (default: background-target-crash) --profile PROFILE Nextest profile to use (default: registry lane) --run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP) --plan-only Validate registry selection and print the exact filter without running cargo --self-test Run lightweight CLI/registry checks without building Rust -h, --help Show this help The script intentionally runs a single case, not the release pseudo-case. After a successful case run it verifies that the release gate still remains blocked. Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e port allocator when the default 20000..30000 test range is unavailable. Set RUSTFS_SCANNER_HEAL_SKIP_CLEAN=1 to reuse an existing cargo target directory while narrowing a case locally; release evidence should keep the default clean build. USAGE } case_ids() { "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" <<'PY' import json import pathlib import sys registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) for case_id in sorted(registry["cases"]): print(case_id) PY } case_field() { local case_id="$1" local field="$2" "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" "$field" <<'PY' import json import pathlib import sys registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) case = registry["cases"][sys.argv[2]] value = case[sys.argv[3]] if not isinstance(value, str): raise SystemExit(f"{sys.argv[3]} is not a string") print(value) PY } test_filter_for() { local case_id="$1" "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" <<'PY' import json import pathlib import re import sys registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) case = registry["cases"][sys.argv[2]] print("test(/^" + re.escape(case["name"]) + "$/)") PY } runtime_profile_for() { local case_id="$1" case "$case_id" in background-target-crash|background-target-restart) echo "background-4x1" ;; background-target-crash-ec8-4|background-target-restart-ec8-4) echo "background-ec8-4" ;; background-target-restart-ec8-4-multi-set) echo "background-ec8-4-multi-set" ;; background-target-crash-ec8-4-multi-pool) echo "background-ec8-4-multi-pool" ;; ec84-target-drive-restart) echo "distributed-ec8-4" ;; *) echo "default" ;; esac } apply_runtime_profile() { local case_id="$1" case "$(runtime_profile_for "$case_id")" in background-4x1) export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-120}" ;; background-ec8-4) export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-32}" export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}" ;; background-ec8-4-multi-set) export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}" ;; background-ec8-4-multi-pool) export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}" export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}" ;; esac } test_binary_from_listing() { local listing="$1" local case_id="$2" "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$listing" "$case_id" <<'PY' import json import pathlib import sys registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) listing = json.loads(pathlib.Path(sys.argv[2]).read_text()) case = registry["cases"][sys.argv[3]] suite = listing["rust-suites"][case["suite"]] testcase = suite["testcases"][case["name"]] if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches": raise SystemExit("selected case is not matched by the nextest listing") matches = 0 for listed_suite in listing.get("rust-suites", {}).values(): for listed in listed_suite.get("testcases", {}).values(): if listed.get("filter-match", {}).get("status") == "matches": matches += 1 if matches != 1: raise SystemExit(f"expected exactly one selected case, got {matches}") print(suite["binary-path"]) PY } release_gate_must_remain_blocked() { local run_dir="$1" local output="$run_dir/release-status.json" if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release "$run_dir" >"$output"; then echo "release gate unexpectedly approved a single Scanner/Heal evidence run" >&2 return 1 fi "$PYTHON_BIN" - "$output" <<'PY' import json import pathlib import sys status = json.loads(pathlib.Path(sys.argv[1]).read_text()) if status.get("decision") != "blocked" or status.get("release_approved") is not False: raise SystemExit("release status did not record a blocked decision") if not status.get("pending_gates"): raise SystemExit("release status did not retain pending gates") PY } nextest_junit_candidates() { local target_dir="$1" local profile="$2" local primary="$target_dir/nextest/$profile/junit.xml" local fallback="$ROOT/target/nextest/$profile/junit.xml" printf '%s\n' "$primary" if [[ "$fallback" != "$primary" ]]; then printf '%s\n' "$fallback" fi } remove_nextest_junit_candidates() { local target_dir="$1" local profile="$2" local candidate while IFS= read -r candidate; do rm -f "$candidate" done < <(nextest_junit_candidates "$target_dir" "$profile") } copy_nextest_junit() { local target_dir="$1" local profile="$2" local run_dir="$3" local candidate while IFS= read -r candidate; do if [[ -f "$candidate" ]]; then cp "$candidate" "$run_dir/junit.xml" return 0 fi done < <(nextest_junit_candidates "$target_dir" "$profile") return 1 } run_self_test() { if "$0" --case release --plan-only >/dev/null 2>&1; then echo "self-test failed: release pseudo-case must not be runnable" >&2 return 1 fi local case_id while IFS= read -r case_id; do local expected_filter expected_profile plan expected_filter="$(test_filter_for "$case_id")" expected_profile="$(case_field "$case_id" lane)" expected_runtime_profile="$(runtime_profile_for "$case_id")" plan="$("$0" --case "$case_id" --plan-only)" if [[ "$plan" != *"case=$case_id"* ]] || [[ "$plan" != *"profile=$expected_profile"* ]] || [[ "$plan" != *"runtime_profile=$expected_runtime_profile"* ]] || [[ "$plan" != *"filter=$expected_filter"* ]] || [[ "$plan" != *"run_dir=$ROOT/target/scanner-heal-evidence/$case_id-"* ]]; then echo "self-test failed: registry case plan mismatch for $case_id" >&2 return 1 fi done < <(case_ids) local junit_profile="scanner-heal-junit-self-test-$$" local junit_run_dir="$ROOT/target/scanner-heal-junit-self-test-$$" local junit_target_dir="$ROOT/target/scanner-heal-junit-custom-target-$$" local junit_fallback="$ROOT/target/nextest/$junit_profile/junit.xml" mkdir -p "$(dirname "$junit_fallback")" "$junit_run_dir" printf '\n' >"$junit_fallback" if ! copy_nextest_junit "$junit_target_dir" "$junit_profile" "$junit_run_dir"; then echo "self-test failed: nextest junit fallback was not copied" >&2 rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")" return 1 fi if ! cmp -s "$junit_fallback" "$junit_run_dir/junit.xml"; then echo "self-test failed: copied nextest junit fallback changed content" >&2 rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")" return 1 fi rm -rf "$junit_run_dir" "$junit_target_dir" "$(dirname "$junit_fallback")" } while [[ $# -gt 0 ]]; do case "$1" in --case) CASE_ID="$2" shift 2 ;; --profile) PROFILE="$2" shift 2 ;; --run-dir) RUN_DIR="$2" shift 2 ;; --plan-only) PLAN_ONLY=1 shift ;; --self-test) run_self_test exit $? ;; -h|--help) usage exit 0 ;; *) echo "unknown option: $1" >&2 usage >&2 exit 2 ;; esac done if [[ "$CASE_ID" == "release" ]]; then echo "release is a checker-only pseudo-case; run a concrete registry case" >&2 exit 2 fi case_field "$CASE_ID" name >/dev/null TEST_FILTER="$(test_filter_for "$CASE_ID")" if [[ -z "$PROFILE" ]]; then PROFILE="$(case_field "$CASE_ID" lane)" fi apply_runtime_profile "$CASE_ID" if [[ -z "$RUN_DIR" ]]; then RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)" elif [[ "$RUN_DIR" != /* ]]; then RUN_DIR="$ROOT/$RUN_DIR" fi if [[ "$PLAN_ONLY" == 1 ]]; then echo "case=$CASE_ID" echo "profile=$PROFILE" echo "runtime_profile=$(runtime_profile_for "$CASE_ID")" echo "filter=$TEST_FILTER" echo "run_dir=$RUN_DIR" exit 0 fi if [[ -e "$RUN_DIR" ]]; then echo "evidence run directory already exists: $RUN_DIR" >&2 exit 1 fi cd "$ROOT" if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then echo "commit tracked source changes before creating evidence" >&2 exit 1 fi NOFILE_SOFT="$(ulimit -Sn)" NOFILE_HARD="$(ulimit -Hn)" if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT" -lt 65535 ]]; then if [[ "$NOFILE_HARD" -ge 65535 ]]; then ulimit -n 65535 || true elif [[ "$NOFILE_HARD" -gt "$NOFILE_SOFT" ]]; then ulimit -n "$NOFILE_HARD" || true fi fi RUN_PARENT="$(dirname "$RUN_DIR")" mkdir -p "$RUN_PARENT" RUN_TMP_ROOT_CREATED=0 if [[ -z "${TMPDIR:-}" ]]; then RUN_TMP_ROOT="$RUN_PARENT/.tmp-$(basename "$RUN_DIR")" mkdir -p "$RUN_TMP_ROOT" export TMPDIR="$RUN_TMP_ROOT" RUN_TMP_ROOT_CREATED=1 else RUN_TMP_ROOT="" fi TMP_DIR="$(mktemp -d "${TMPDIR%/}/rustfs-scanner-heal-evidence.XXXXXX")" cleanup_tmp() { rm -rf "$TMP_DIR" if [[ "$RUN_TMP_ROOT_CREATED" == 1 ]]; then rm -rf "$RUN_TMP_ROOT" fi } trap cleanup_tmp EXIT BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}" TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}" if [[ "$TARGET_DIR" != /* ]]; then TARGET_DIR="$ROOT/$TARGET_DIR" fi DEBUG_DIR="$TARGET_DIR/debug" SERVER_BINARY="$DEBUG_DIR/rustfs" if [[ "${RUSTFS_SCANNER_HEAL_SKIP_CLEAN:-0}" != "1" ]]; then cargo clean -p rustfs fi "$PYTHON_BIN" "$ROOT/scripts/e2e_binary.py" build --bins --features "$BUILD_FEATURES" LISTING_TMP="$TMP_DIR/listing.json" NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ HTTP_PROXY= \ HTTPS_PROXY= \ RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" \ cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-run --no-tests=fail cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-format json >"$LISTING_TMP" TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")" export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}" "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY" cp "$LISTING_TMP" "$RUN_DIR/listing.json" export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}" export RUSTFS_HEAL_CHAOS_LOG_DIR="${RUSTFS_HEAL_CHAOS_LOG_DIR:-$RUSTFS_E2E_LOG_DIR}" mkdir -p "$RUSTFS_E2E_LOG_DIR" remove_nextest_junit_candidates "$TARGET_DIR" "$PROFILE" set +e NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ HTTP_PROXY= \ HTTPS_PROXY= \ RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" \ "$PYTHON_BIN" "$ROOT/scripts/e2e_binary.py" run --features "$BUILD_FEATURES" -- \ cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=fail STATUS=$? set -e copy_nextest_junit "$TARGET_DIR" "$PROFILE" "$RUN_DIR" || true "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS" if [[ "$STATUS" -ne 0 ]]; then echo "Scanner/Heal evidence case failed: $CASE_ID (exit $STATUS); receipt kept at $RUN_DIR" >&2 exit "$STATUS" fi "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$RUN_DIR" "$CASE_ID" release_gate_must_remain_blocked "$RUN_DIR" echo "Scanner/Heal evidence case verified: $CASE_ID" echo "Release gate remains blocked; status: $RUN_DIR/release-status.json" echo "Evidence directory: $RUN_DIR"