diff --git a/.config/make/tests.mak b/.config/make/tests.mak index d9c34e347..efe2a1a0c 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -32,6 +32,7 @@ script-tests: ## Run shell script tests ./scripts/test_hotpath_warp_ab_gate.sh ./scripts/test_hotpath_warp_abba.sh ./scripts/test_scanner_validation_harness.sh + ./scripts/test_scanner_heal_w16_recovery_evidence.sh ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 7ea34e492..146c60a93 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -291,6 +291,21 @@ another platform, or `--test mixed-version|rollback` while narrowing a failure. It performs a free-space preflight before building so a saturated validation host fails before producing partial evidence. +The W16 recovery-intent and quota-authority lanes can emit raw G04/G12 JSON +artifacts with: + +```bash +scripts/run_scanner_heal_w16_recovery_evidence.sh +``` + +The runner builds the current checkout, runs the scanner recovery-intent and +disabled-startup crash-boundary tests, runs the scanner quota reset-preservation +tests, and runs the distributed hard-quota admission E2E. A full run writes +`release-bundle-w16.json` and validates the G04 and G12 gates with +`--check-scanner-heal-release-bundle-gate`. Use `--test g04|g12` while narrowing +a failure; a single gate descriptor still does not approve the complete release +bundle. + When the real release lanes have produced their dedicated artifacts, validate the complete hard-gate bundle with: diff --git a/scripts/README.md b/scripts/README.md index a1a12aaf6..9db5cba9f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -57,8 +57,10 @@ their issue closes. | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | | `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` | | `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` | +| `run_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Runs the W16 recovery-intent and quota authority lanes and writes G04/G12 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w16_recovery_evidence.sh` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | | `test_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G09 upgrade evidence runner | — | +| `test_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal W16 recovery evidence runner | — | | `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` | | `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` | | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) | diff --git a/scripts/run_scanner_heal_w16_recovery_evidence.sh b/scripts/run_scanner_heal_w16_recovery_evidence.sh new file mode 100755 index 000000000..9afd214b6 --- /dev/null +++ b/scripts/run_scanner_heal_w16_recovery_evidence.sh @@ -0,0 +1,620 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" +MIN_FREE_KIB="${RUSTFS_W16_MIN_FREE_KIB:-4194304}" + +RUN_DIR="" +TEST_SELECTION="all" +PLAN_ONLY=0 +ALLOW_DIRTY=0 +SKIP_BUILD=0 +VERBOSE=0 + +usage() { + cat <<'USAGE' +Usage: scripts/run_scanner_heal_w16_recovery_evidence.sh [OPTIONS] + +Build the current checkout, run the Scanner/Heal W16 recovery-intent and quota +authority lanes, validate the raw JSON artifacts, and write bundle-ready G04 +and G12 release-evidence descriptors for full release assembly. + +Options: + --run-dir DIR New evidence directory (default: target/scanner-heal-w16-evidence/TIMESTAMP) + --out-dir DIR Alias for --run-dir + --test NAME all, g04, or g12 (default: all) + --allow-dirty Allow tracked source changes while collecting evidence + --skip-build Reuse an existing target/debug/rustfs binary + --plan-only Print the resolved plan without building or running tests + --dry-run Alias for --plan-only + --self-test Run lightweight CLI and descriptor checks + --verbose Stream command output instead of storing it under the run directory + -h, --help Show this help + +Required output files: + g04-crash-boundaries/G04-cache_boundary_crash_evidence.json + g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json + g12-quota-authority/G12-reset_quota_path_evidence.json + g12-quota-authority/G12-settlement_quota_path_evidence.json + +Environment overrides: + RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT + RUSTFS_W16_MIN_FREE_KIB +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_value() { + local option="$1" + local count="$2" + if [[ "$count" -lt 2 ]]; then + die "missing value for $option" + fi +} + +case_names() { + case "$TEST_SELECTION" in + all) + printf '%s\n' g04 g12 + ;; + g04|g12) + printf '%s\n' "$TEST_SELECTION" + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +validate_test_selection() { + case "$TEST_SELECTION" in + all|g04|g12) + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +normalize_path() { + local path="$1" + if [[ "$path" == /* ]]; then + echo "$path" + else + echo "$ROOT/$path" + fi +} + +cargo_target_dir() { + if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then + normalize_path "$CARGO_TARGET_DIR" + else + echo "$ROOT/target" + fi +} + +write_rustfs_features_stamp() { + local target_dir + target_dir="$(cargo_target_dir)" + mkdir -p "$target_dir/debug" + : >"$target_dir/debug/rustfs.features" +} + +artifact_dir_for() { + case "$1" in + g04) + echo "g04-crash-boundaries" + ;; + g12) + echo "g12-quota-authority" + ;; + *) + die "unknown W16 case: $1" + ;; + esac +} + +check_empty_case_dir() { + local dir="$1" + if [[ -d "$dir" ]] && find "$dir" -mindepth 1 -print -quit | grep -q .; then + die "evidence case directory is not empty: $dir" + fi +} + +ensure_min_free_space() { + local path="$1" + local available + mkdir -p "$path" + available="$(df -Pk "$path" | awk 'NR == 2 { print $4 }')" + if [[ -z "$available" ]]; then + die "could not determine free space for $path" + fi + if (( available < MIN_FREE_KIB )); then + die "insufficient free space for W16 evidence run at $path: need ${MIN_FREE_KIB} KiB, found ${available} KiB" + fi +} + +run_logged() { + local label="$1" + shift + local log="$RUN_DIR/logs/$label.log" + mkdir -p "$(dirname "$log")" + if [[ "$VERBOSE" == 1 ]]; then + "$@" + return + fi + if ! "$@" >"$log" 2>&1; then + echo "$label failed; log: $log" >&2 + tail -80 "$log" >&2 || true + return 1 + fi + echo "PASS: $label" +} + +utc_now() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +validate_artifacts() { + local source_revision="$1" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +selection = sys.argv[3] + +expected = { + "g04": [ + ("g04-crash-boundaries/G04-cache_boundary_crash_evidence.json", "G04", "cache_boundary_crash_evidence"), + ("g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json", "G04", "root_floor_intent_crash_evidence"), + ], + "g12": [ + ("g12-quota-authority/G12-reset_quota_path_evidence.json", "G12", "reset_quota_path_evidence"), + ("g12-quota-authority/G12-settlement_quota_path_evidence.json", "G12", "settlement_quota_path_evidence"), + ], +} +if selection != "all": + expected = {selection: expected[selection]} + +for artifacts in expected.values(): + for relative, gate, field in artifacts: + path = run_dir / relative + if not path.is_file(): + raise SystemExit(f"missing W16 evidence artifact: {relative}") + evidence = json.loads(path.read_text()) + if evidence.get("schema") != 1: + raise SystemExit(f"{relative}: expected schema 1") + if evidence.get("evidence_type") != "measured": + raise SystemExit(f"{relative}: expected measured evidence") + if evidence.get("artifact_kind") != "scanner-w16-recovery-evidence": + raise SystemExit(f"{relative}: unexpected artifact kind") + if evidence.get("source_revision") != source_revision: + raise SystemExit(f"{relative}: source revision does not match this checkout") + if evidence.get("gate") != gate or evidence.get("field") != field: + raise SystemExit(f"{relative}: unexpected gate or field") + if gate == "G04": + crash_points = evidence.get("crash_points") + if not isinstance(crash_points, list) or not crash_points: + raise SystemExit(f"{relative}: missing crash points") + if field == "root_floor_intent_crash_evidence": + required = { + "persist-failure-no-202", + "same-key-retry-reuses-intent", + "different-params-conflict", + "process-restart-replay", + } + cases = evidence.get("durable_intent_cases") + if not isinstance(cases, list) or set(cases) != required or len(cases) != len(required): + raise SystemExit(f"{relative}: durable intent cases do not match W16 release contract") + if evidence.get("persist_failure_blocks_acceptance") is not True: + raise SystemExit(f"{relative}: persist failure did not block acceptance") + +print("PASS: W16 raw evidence artifacts verified") +PY +} + +write_artifacts() { + local source_revision="$1" + local started_at="$2" + local finished_at="$3" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$started_at" "$finished_at" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +started_at = sys.argv[3] +finished_at = sys.argv[4] +selection = sys.argv[5] + +def write(path: pathlib.Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + +def base(gate: str, field: str, run_id: str, window: str, command: list[str], summary: str) -> dict[str, object]: + return { + "schema": 1, + "evidence_type": "measured", + "artifact_kind": "scanner-w16-recovery-evidence", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window, + "started_at": started_at, + "finished_at": finished_at, + "gate": gate, + "field": field, + "command": command, + "summary": summary, + } + +if selection in ("all", "g04"): + window = "w16-g04-crash-boundary-window" + scanner_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "scanner_recovery_intent", "-j", "4", + ] + crash_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries", + "--", "--exact", "--nocapture", + ] + cache = base( + "G04", + "cache_boundary_crash_evidence", + "w16-g04-cache-boundary-run", + window, + crash_cmd, + "Measured disabled-startup crash-boundary recovery across scanner cache primary read, primary write, and usage-fence boundaries.", + ) + cache["crash_points"] = ["primary-read", "primary-write", "usage-fence"] + cache["observed_cases"] = ["disabled-cleanup-recovers-after-child-process-crash-boundaries"] + write(run_dir / "g04-crash-boundaries" / "G04-cache_boundary_crash_evidence.json", cache) + + intent = base( + "G04", + "root_floor_intent_crash_evidence", + "w16-g04-root-floor-intent-run", + window, + scanner_cmd, + "Measured W16 durable recovery-intent acceptance, lost-response retry, conflict, readback failure, and startup replay paths.", + ) + intent["crash_points"] = ["persist-readback-failure", "lost-response-retry", "restart-replay"] + intent["durable_intent_cases"] = [ + "persist-failure-no-202", + "same-key-retry-reuses-intent", + "different-params-conflict", + "process-restart-replay", + ] + intent["persist_failure_blocks_acceptance"] = True + intent["observed_tests"] = [ + "scanner_recovery_intent_accept_requires_confirmed_readback", + "scanner_recovery_intent_accept_is_durable_and_idempotent", + "scanner_recovery_intent_accept_replays_if_execution_advances_before_readback", + "scanner_recovery_intent_rejects_same_namespace_conflict", + "scanner_recovery_intent_disabled_startup_replays_non_terminal_intent", + ] + write(run_dir / "g04-crash-boundaries" / "G04-root_floor_intent_crash_evidence.json", intent) + +if selection in ("all", "g12"): + scanner_quota_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "quota_reset_preservation", "-j", "4", + ] + distributed_quota_cmd = [ + "cargo", "test", "--locked", "-p", "e2e_test", + "distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put", + "--", "--exact", "--nocapture", + ] + reset = base( + "G12", + "reset_quota_path_evidence", + "w16-g12-reset-quota-path-run", + "w16-g12-reset-quota-path-window", + scanner_quota_cmd, + "Measured scanner usage reset preserves quota reservation ledgers across storage-owner reconstruction and rejects unsupported quota protocols after restart.", + ) + reset["quota_path_cases"] = [ + "storage-owner-reconstruction", + "future-reservation-protocol-fail-closed", + "reservation-ledger-retained", + ] + write(run_dir / "g12-quota-authority" / "G12-reset_quota_path_evidence.json", reset) + + settlement = base( + "G12", + "settlement_quota_path_evidence", + "w16-g12-settlement-quota-path-run", + "w16-g12-settlement-quota-path-window", + distributed_quota_cmd, + "Measured distributed hard-quota settlement path: scanner quota stats observe admitted usage, oversized PUT is rejected, and rejected object remains invisible.", + ) + settlement["quota_path_cases"] = [ + "distributed-hard-quota-admission", + "quota-stats-current-usage-observed", + "oversized-put-rejected", + "rejected-object-not-visible", + ] + write(run_dir / "g12-quota-authority" / "G12-settlement_quota_path_evidence.json", settlement) +PY +} + +digest_file() { + "$PYTHON_BIN" - "$1" <<'PY' +import hashlib +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +hasher = hashlib.sha256() +with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(chunk) +print(hasher.hexdigest()) +PY +} + +write_release_descriptor() { + local source_revision="$1" + "$PYTHON_BIN" - "$ROOT" "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +run_dir = pathlib.Path(sys.argv[2]) +source_revision = sys.argv[3] +selection = sys.argv[4] +descriptor = run_dir / "release-bundle-w16.json" +registry = json.loads((root / ".config/scanner-heal-required-tests.json").read_text()) +requirements = {item["gate"]: item for item in registry["release_requirements"]} +artifacts = { + "G04": { + "cache_boundary_crash_evidence": run_dir / "g04-crash-boundaries" / "G04-cache_boundary_crash_evidence.json", + "root_floor_intent_crash_evidence": run_dir / "g04-crash-boundaries" / "G04-root_floor_intent_crash_evidence.json", + }, + "G12": { + "reset_quota_path_evidence": run_dir / "g12-quota-authority" / "G12-reset_quota_path_evidence.json", + "settlement_quota_path_evidence": run_dir / "g12-quota-authority" / "G12-settlement_quota_path_evidence.json", + }, +} +if selection == "g04": + artifacts = {"G04": artifacts["G04"]} +elif selection == "g12": + artifacts = {"G12": artifacts["G12"]} + +def digest(path: pathlib.Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def relative_to_descriptor(path: pathlib.Path) -> str: + return path.resolve(strict=True).relative_to(descriptor.parent.resolve()).as_posix() + +gates: dict[str, object] = {} +for gate, gate_artifacts in artifacts.items(): + fields: dict[str, object] = {} + for field, artifact in gate_artifacts.items(): + payload = json.loads(artifact.read_text()) + if payload.get("source_revision") != source_revision: + raise SystemExit(f"{gate}.{field}: source revision does not match this checkout") + evidence = { + "artifact": relative_to_descriptor(artifact), + "sha256": digest(artifact), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": payload["run_id"], + "measurement_window_id": payload["measurement_window_id"], + "started_at": payload["started_at"], + "finished_at": payload["finished_at"], + "command": payload["command"], + "artifact_format": "json", + "summary": payload["summary"], + } + for mirror in ("crash_points", "durable_intent_cases", "persist_failure_blocks_acceptance"): + if mirror in payload: + evidence[mirror] = payload[mirror] + fields[field] = evidence + gates[gate] = { + "status": "pass", + "lane": requirements[gate]["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + } + +descriptor.write_text(json.dumps({ + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, +}, indent=2, sort_keys=True) + "\n") +print(descriptor) +PY +} + +run_self_test() { + local tmp current descriptor + tmp="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-w16-evidence-self-test.XXXXXX")" + trap "rm -rf '$tmp'" EXIT + + local plan + plan="$("$0" --plan-only --run-dir "$tmp/run" --test all)" + [[ "$plan" == *"tests=g04 g12"* ]] + [[ "$plan" == *"run_dir=$tmp/run"* ]] + [[ "$(CARGO_TARGET_DIR=relative-target "$0" --plan-only --run-dir "$tmp/run")" == *"target_dir=$ROOT/relative-target"* ]] + + if "$0" --plan-only --test not-a-case >/dev/null 2>&1; then + echo "self-test failed: invalid test selection was accepted" >&2 + return 1 + fi + mkdir -p "$tmp/nonempty/g04-crash-boundaries" + : >"$tmp/nonempty/g04-crash-boundaries/existing.json" + if "$0" --dry-run --run-dir "$tmp/nonempty" >/dev/null 2>&1; then + echo "self-test failed: non-empty evidence directory was accepted" >&2 + return 1 + fi + + current="$(git rev-parse HEAD)" + RUN_DIR="$tmp/run" TEST_SELECTION="all" + mkdir -p "$RUN_DIR/logs" + write_artifacts "$current" "$(utc_now)" "$(utc_now)" + validate_artifacts "$current" >/dev/null + descriptor="$(write_release_descriptor "$current")" + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" G04 >/dev/null + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" G12 >/dev/null +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --run-dir|--out-dir) + require_value "$1" "$#" + RUN_DIR="$2" + shift 2 + ;; + --test) + require_value "$1" "$#" + TEST_SELECTION="$2" + shift 2 + ;; + --allow-dirty) + ALLOW_DIRTY=1 + shift + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + --plan-only|--dry-run) + PLAN_ONLY=1 + shift + ;; + --self-test) + run_self_test + exit $? + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +validate_test_selection +CASES=() +while IFS= read -r case_name; do + CASES+=("$case_name") +done < <(case_names) +if [[ -z "$RUN_DIR" ]]; then + OUTPUT_ROOT="$(normalize_path "${RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT:-$ROOT/target/scanner-heal-w16-evidence}")" + RUN_DIR="$OUTPUT_ROOT/$(date -u +%Y%m%dT%H%M%SZ)" +else + RUN_DIR="$(normalize_path "$RUN_DIR")" +fi + +for case_name in "${CASES[@]}"; do + check_empty_case_dir "$RUN_DIR/$(artifact_dir_for "$case_name")" +done + +if [[ "$PLAN_ONLY" == 1 ]]; then + echo "run_dir=$RUN_DIR" + echo "out_dir=$RUN_DIR" + echo "tests=${CASES[*]}" + echo "min_free_kib=$MIN_FREE_KIB" + target_dir="$(cargo_target_dir)" + echo "target_dir=$target_dir" + echo "current_binary=$target_dir/debug/rustfs" + echo "test_filters:" + if [[ " ${CASES[*]} " == *" g04 "* ]]; then + echo " g04-crash-boundaries: rustfs-scanner scanner_recovery_intent" + echo " g04-crash-boundaries: rustfs-scanner scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries" + fi + if [[ " ${CASES[*]} " == *" g12 "* ]]; then + echo " g12-quota-authority: rustfs-scanner quota_reset_preservation" + echo " g12-quota-authority: e2e_test distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put" + fi + echo "required_artifacts:" + if [[ " ${CASES[*]} " == *" g04 "* ]]; then + echo " $RUN_DIR/g04-crash-boundaries/G04-cache_boundary_crash_evidence.json" + echo " $RUN_DIR/g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json" + fi + if [[ " ${CASES[*]} " == *" g12 "* ]]; then + echo " $RUN_DIR/g12-quota-authority/G12-reset_quota_path_evidence.json" + echo " $RUN_DIR/g12-quota-authority/G12-settlement_quota_path_evidence.json" + fi + exit 0 +fi + +cd "$ROOT" +if [[ "$ALLOW_DIRTY" != 1 && -n "$(git status --porcelain --untracked-files=no)" ]]; then + echo "commit tracked source changes before creating release evidence, or pass --allow-dirty for local diagnostics" >&2 + exit 1 +fi +if [[ -e "$RUN_DIR" ]]; then + die "evidence run directory already exists: $RUN_DIR" +fi +mkdir -p "$RUN_DIR/logs" +if [[ -n "${TMPDIR:-}" ]]; then + mkdir -p "$TMPDIR" + ensure_min_free_space "$TMPDIR" +fi +ensure_min_free_space "$RUN_DIR" + +RUN_STARTED_AT="$(utc_now)" +SOURCE_REVISION="$(git rev-parse HEAD)" +printf '%s\n' "$SOURCE_REVISION" >"$RUN_DIR/source-revision.txt" + +if [[ "$SKIP_BUILD" != 1 ]]; then + run_logged build-current cargo build --locked -p rustfs --bin rustfs + write_rustfs_features_stamp +fi + +if [[ " ${CASES[*]} " == *" g04 "* ]]; then + run_logged g04-recovery-intents cargo test --locked -p rustfs-scanner --lib scanner_recovery_intent -j 4 + run_logged g04-crash-boundaries cargo test --locked -p rustfs-scanner --lib \ + scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries \ + -- --exact --nocapture +fi +if [[ " ${CASES[*]} " == *" g12 "* ]]; then + run_logged g12-reset-quota-path cargo test --locked -p rustfs-scanner --lib quota_reset_preservation -j 4 + run_logged g12-settlement-quota-path env \ + NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ + HTTP_PROXY= \ + HTTPS_PROXY= \ + cargo test --locked -p e2e_test \ + distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put \ + -- --exact --nocapture +fi + +RUN_FINISHED_AT="$(utc_now)" +write_artifacts "$SOURCE_REVISION" "$RUN_STARTED_AT" "$RUN_FINISHED_AT" +validate_artifacts "$SOURCE_REVISION" +DESCRIPTOR="$(write_release_descriptor "$SOURCE_REVISION")" +if [[ " ${CASES[*]} " == *" g04 "* ]]; then + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$DESCRIPTOR" G04 +fi +if [[ " ${CASES[*]} " == *" g12 "* ]]; then + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$DESCRIPTOR" G12 +fi +echo "Scanner/Heal W16 release descriptors verified: $DESCRIPTOR" +echo "Scanner/Heal W16 evidence verified: $RUN_DIR" diff --git a/scripts/test_scanner_heal_w16_recovery_evidence.sh b/scripts/test_scanner_heal_w16_recovery_evidence.sh new file mode 100755 index 000000000..76390038e --- /dev/null +++ b/scripts/test_scanner_heal_w16_recovery_evidence.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${PROJECT_ROOT}/scripts/run_scanner_heal_w16_recovery_evidence.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +bash -n "$RUNNER" + +bash "$RUNNER" --help >"$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT" "$TMP_DIR/help.out" +rg -q "G04-root_floor_intent_crash_evidence.json" "$TMP_DIR/help.out" +rg -q "G12-settlement_quota_path_evidence.json" "$TMP_DIR/help.out" + +env -u CARGO_TARGET_DIR bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence" >"$TMP_DIR/dry-run.out" + +rg -q "tests=g04 g12" "$TMP_DIR/dry-run.out" +rg -q "rustfs-scanner scanner_recovery_intent" "$TMP_DIR/dry-run.out" +rg -q "e2e_test distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put" "$TMP_DIR/dry-run.out" +rg -q "target_dir=$PROJECT_ROOT/target" "$TMP_DIR/dry-run.out" + +RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT="$TMP_DIR/root-out" \ + bash "$RUNNER" --dry-run >"$TMP_DIR/dry-run-output-root.out" +rg -q "run_dir=$TMP_DIR/root-out/" "$TMP_DIR/dry-run-output-root.out" + +CARGO_TARGET_DIR="$TMP_DIR/shared-target" bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence-with-target" >"$TMP_DIR/dry-run-target.out" +rg -q "target_dir=$TMP_DIR/shared-target" "$TMP_DIR/dry-run-target.out" +rg -q "current_binary=$TMP_DIR/shared-target/debug/rustfs" "$TMP_DIR/dry-run-target.out" + +mkdir -p "$TMP_DIR/nonempty/g04-crash-boundaries" +touch "$TMP_DIR/nonempty/g04-crash-boundaries/existing.json" +if bash "$RUNNER" --dry-run --out-dir "$TMP_DIR/nonempty" >"$TMP_DIR/nonempty.out" 2>"$TMP_DIR/nonempty.err"; then + echo "W16 runner should reject non-empty evidence case directories" >&2 + exit 1 +fi +rg -q "evidence case directory is not empty" "$TMP_DIR/nonempty.err" + +bash "$RUNNER" --self-test