diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index bde2ee748..8680a2e63 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -77,4 +77,18 @@ cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distribut The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check. +Scanner/Heal G09 release evidence can be collected on Linux x86_64 with the +same pinned previous-release binary used by `e2e-upgrade.yml`: + +```bash +scripts/run_scanner_heal_g09_upgrade_evidence.sh +``` + +The runner builds the current `rustfs` binary, downloads and verifies the +pinned previous release, runs the mixed-version rolling-upgrade and rollback +tests, and leaves the required raw G09 artifacts under +`target/scanner-heal-g09-upgrade-evidence//`. These artifacts are +inputs for the Scanner/Heal release bundle gate; the runner does not mark the +full release matrix complete by itself. + Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed ` after adding or renaming a case. diff --git a/scripts/README.md b/scripts/README.md index b9fc299f1..a1a12aaf6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -56,8 +56,9 @@ their issue closes. | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` | | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | | `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` | -| `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml` | +| `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | +| `test_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G09 upgrade evidence runner | — | | `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` | | `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` | | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) | diff --git a/scripts/run_scanner_heal_g09_upgrade_evidence.sh b/scripts/run_scanner_heal_g09_upgrade_evidence.sh index f417e061a..08977c1ca 100755 --- a/scripts/run_scanner_heal_g09_upgrade_evidence.sh +++ b/scripts/run_scanner_heal_g09_upgrade_evidence.sh @@ -4,10 +4,10 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" -SOURCE_REPOSITORY="${RUSTFS_UPGRADE_SOURCE_REPOSITORY:-rustfs/rustfs}" -SOURCE_VERSION="${RUSTFS_UPGRADE_SOURCE_VERSION:-1.0.0-rc.5}" -SOURCE_ASSET="${RUSTFS_UPGRADE_SOURCE_ASSET:-rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip}" -SOURCE_SHA256="${RUSTFS_UPGRADE_SOURCE_SHA256:-3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d}" +SOURCE_REPOSITORY="${RUSTFS_UPGRADE_SOURCE_REPOSITORY:-${UPGRADE_SOURCE_REPOSITORY:-rustfs/rustfs}}" +SOURCE_VERSION="${RUSTFS_UPGRADE_SOURCE_VERSION:-${UPGRADE_SOURCE_VERSION:-1.0.0-rc.5}}" +SOURCE_ASSET="${RUSTFS_UPGRADE_SOURCE_ASSET:-${UPGRADE_SOURCE_ASSET:-rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip}}" +SOURCE_SHA256="${RUSTFS_UPGRADE_SOURCE_SHA256:-${UPGRADE_SOURCE_SHA256:-3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d}}" MIN_FREE_KIB="${RUSTFS_G09_MIN_FREE_KIB:-6291456}" RUN_DIR="" @@ -17,6 +17,7 @@ TEST_SELECTION="all" PLAN_ONLY=0 ALLOW_DIRTY=0 SKIP_BUILD=0 +SKIP_DOWNLOAD=0 VERBOSE=0 usage() { @@ -31,10 +32,17 @@ Options: --out-dir DIR Alias for --run-dir --source-dir DIR Cache directory for the pinned previous release binary --source-binary BIN Use an existing previous-release rustfs binary + --version VERSION Previous release tag (default: 1.0.0-rc.5) + --asset NAME Previous release asset zip name + --sha256 HEX Expected SHA-256 for the previous release asset + --repository OWNER/REPO + GitHub repository used to download the release asset (default: rustfs/rustfs) --test NAME all, mixed-version, or rollback (default: all) --allow-dirty Allow tracked source changes while collecting evidence --skip-build Reuse an existing target/debug/rustfs binary + --skip-download Reuse SOURCE_DIR/rustfs instead of downloading the previous release --plan-only Print the resolved plan without building or running tests + --dry-run Validate configuration and print the commands without running them --self-test Run lightweight CLI and artifact-validator checks --verbose Stream command output instead of storing it under the run directory -h, --help Show this help @@ -43,9 +51,41 @@ The default pinned release asset is Linux x86_64. Use --source-binary when running against a custom previous-release binary on another platform. The script requires at least 6 GiB free by default; override RUSTFS_G09_MIN_FREE_KIB only for a deliberately smaller diagnostic run. + +Environment overrides: + RUSTFS_SCANNER_HEAL_G09_OUTPUT_ROOT + RUSTFS_UPGRADE_SOURCE_DIR + RUSTFS_UPGRADE_SOURCE_BINARY + RUSTFS_UPGRADE_SOURCE_REPOSITORY / UPGRADE_SOURCE_REPOSITORY + RUSTFS_UPGRADE_SOURCE_VERSION / UPGRADE_SOURCE_VERSION + RUSTFS_UPGRADE_SOURCE_ASSET / UPGRADE_SOURCE_ASSET + RUSTFS_UPGRADE_SOURCE_SHA256 / UPGRADE_SOURCE_SHA256 + +Per-case test environment set by the runner: + RUSTFS_UPGRADE_SOURCE_BINARY + RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR + RUSTFS_E2E_LOG_DIR + +Required output files: + mixed-version-upgrade/G09-mixed_version_reader_evidence.json + mixed-version-upgrade/G09-mixed_version_writer_evidence.json + bucket-config-rollback/G09-rollback_payload_evidence.json USAGE } +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_value() { + local option="$1" + local count="$2" + if [[ "$count" -lt 2 ]]; then + die "missing value for $option" + fi +} + case_names() { case "$TEST_SELECTION" in all) @@ -55,8 +95,7 @@ case_names() { printf '%s\n' "$TEST_SELECTION" ;; *) - echo "unknown test selection: $TEST_SELECTION" >&2 - exit 2 + die "unknown test selection: $TEST_SELECTION" ;; esac } @@ -70,8 +109,7 @@ artifact_for() { echo "bucket-config-rollback" ;; *) - echo "unknown G09 case: $1" >&2 - exit 2 + die "unknown G09 case: $1" ;; esac } @@ -85,8 +123,7 @@ test_filter_for() { echo "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata" ;; *) - echo "unknown G09 case: $1" >&2 - exit 2 + die "unknown G09 case: $1" ;; esac } @@ -140,6 +177,17 @@ verify_sha256() { fi } +validate_source_sha256() { + [[ "$SOURCE_SHA256" =~ ^[0-9a-f]{64}$ ]] || die "--sha256 must be a 64-character lowercase hex digest" +} + +check_empty_case_dir() { + local dir="$1" + if [[ -d "$dir" ]] && find "$dir" -mindepth 1 -print -quit | grep -q .; then + die "evidence case directory is not empty: $dir" + fi +} + ensure_min_free_space() { local path="$1" local available @@ -175,6 +223,9 @@ resolve_source_binary() { echo "$binary" return fi + if [[ "$SKIP_DOWNLOAD" == 1 ]]; then + die "previous release binary does not exist or is not executable: $binary" + fi mkdir -p "$SOURCE_DIR" local archive="$SOURCE_DIR/$SOURCE_ASSET" @@ -278,6 +329,16 @@ run_self_test() { echo "self-test failed: invalid test selection was accepted" >&2 return 1 fi + if "$0" --dry-run --sha256 bad >/dev/null 2>&1; then + echo "self-test failed: invalid SHA-256 was accepted" >&2 + return 1 + fi + mkdir -p "$tmp/nonempty/mixed-version-upgrade" + touch "$tmp/nonempty/mixed-version-upgrade/existing.json" + if "$0" --dry-run --run-dir "$tmp/nonempty" --sha256 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >/dev/null 2>&1; then + echo "self-test failed: non-empty evidence directory was accepted" >&2 + return 1 + fi mkdir -p "$tmp/run/mixed-version-upgrade" "$tmp/run/bucket-config-rollback" local current previous @@ -298,18 +359,42 @@ JSON while [[ $# -gt 0 ]]; do case "$1" in --run-dir|--out-dir) + require_value "$1" "$#" RUN_DIR="$2" shift 2 ;; --source-dir) + require_value "$1" "$#" SOURCE_DIR="$2" shift 2 ;; --source-binary) + require_value "$1" "$#" SOURCE_BINARY="$2" shift 2 ;; + --version) + require_value "$1" "$#" + SOURCE_VERSION="$2" + shift 2 + ;; + --asset) + require_value "$1" "$#" + SOURCE_ASSET="$2" + shift 2 + ;; + --sha256) + require_value "$1" "$#" + SOURCE_SHA256="$2" + shift 2 + ;; + --repository) + require_value "$1" "$#" + SOURCE_REPOSITORY="$2" + shift 2 + ;; --test) + require_value "$1" "$#" TEST_SELECTION="$2" shift 2 ;; @@ -321,7 +406,11 @@ while [[ $# -gt 0 ]]; do SKIP_BUILD=1 shift ;; - --plan-only) + --skip-download) + SKIP_DOWNLOAD=1 + shift + ;; + --plan-only|--dry-run) PLAN_ONLY=1 shift ;; @@ -354,9 +443,18 @@ if [[ -z "$RUN_DIR" ]]; then else RUN_DIR="$(normalize_path "$RUN_DIR")" fi +if [[ -n "$SOURCE_DIR" ]]; then + SOURCE_DIR="$(normalize_path "$SOURCE_DIR")" +fi + +validate_source_sha256 +for case_name in "${CASES[@]}"; do + check_empty_case_dir "$RUN_DIR/$(artifact_for "$case_name")" +done if [[ "$PLAN_ONLY" == 1 ]]; then echo "run_dir=$RUN_DIR" + echo "out_dir=$RUN_DIR" echo "tests=${CASES[*]}" echo "source_repository=$SOURCE_REPOSITORY" echo "source_version=$SOURCE_VERSION" @@ -367,11 +465,31 @@ if [[ "$PLAN_ONLY" == 1 ]]; then if [[ -z "$SOURCE_DIR" ]]; then echo "source_dir=$ROOT/target/scanner-heal-g09-source/$SOURCE_VERSION" else - echo "source_dir=$(normalize_path "$SOURCE_DIR")" + echo "source_dir=$SOURCE_DIR" fi echo "source_asset=$SOURCE_ASSET" + echo "download_url=https://github.com/$SOURCE_REPOSITORY/releases/download/$SOURCE_VERSION/$SOURCE_ASSET" fi - echo "target_dir=$(cargo_target_dir)" + target_dir="$(cargo_target_dir)" + echo "target_dir=$target_dir" + echo "current_binary=$target_dir/debug/rustfs" + echo "test_filters:" + for case_name in "${CASES[@]}"; do + echo " $(artifact_for "$case_name"): $(test_filter_for "$case_name")" + done + echo "required_artifacts:" + for case_name in "${CASES[@]}"; do + artifact="$(artifact_for "$case_name")" + case "$case_name" in + mixed-version) + echo " $RUN_DIR/$artifact/G09-mixed_version_reader_evidence.json" + echo " $RUN_DIR/$artifact/G09-mixed_version_writer_evidence.json" + ;; + rollback) + echo " $RUN_DIR/$artifact/G09-rollback_payload_evidence.json" + ;; + esac + done exit 0 fi diff --git a/scripts/test_scanner_heal_g09_upgrade_evidence.sh b/scripts/test_scanner_heal_g09_upgrade_evidence.sh new file mode 100755 index 000000000..8244c2f2e --- /dev/null +++ b/scripts/test_scanner_heal_g09_upgrade_evidence.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${PROJECT_ROOT}/scripts/run_scanner_heal_g09_upgrade_evidence.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +bash -n "$RUNNER" + +bash "$RUNNER" --help >"$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR" "$TMP_DIR/help.out" +rg -q "mixed-version-upgrade/G09-mixed_version_reader_evidence.json" "$TMP_DIR/help.out" +rg -q "bucket-config-rollback/G09-rollback_payload_evidence.json" "$TMP_DIR/help.out" + +if bash "$RUNNER" --dry-run --sha256 bad >"$TMP_DIR/bad-sha.out" 2>"$TMP_DIR/bad-sha.err"; then + echo "G09 runner should reject invalid SHA-256 input" >&2 + exit 1 +fi +rg -q -- "--sha256 must be a 64-character lowercase hex digest" "$TMP_DIR/bad-sha.err" + +VALID_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence" \ + --source-dir "$TMP_DIR/source" \ + --version 1.2.3 \ + --asset rustfs-linux-x86_64-gnu-v1.2.3.zip \ + --sha256 "$VALID_SHA" \ + --repository rustfs/rustfs >"$TMP_DIR/dry-run.out" + +rg -q "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts" "$TMP_DIR/dry-run.out" +rg -q "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata" "$TMP_DIR/dry-run.out" +rg -q "$TMP_DIR/evidence/mixed-version-upgrade/G09-mixed_version_writer_evidence.json" "$TMP_DIR/dry-run.out" +rg -q "target_dir=$PROJECT_ROOT/target" "$TMP_DIR/dry-run.out" +rg -q "https://github.com/rustfs/rustfs/releases/download/1.2.3/rustfs-linux-x86_64-gnu-v1.2.3.zip" "$TMP_DIR/dry-run.out" + +CARGO_TARGET_DIR="$TMP_DIR/shared-target" bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence-with-target" \ + --source-dir "$TMP_DIR/source" \ + --sha256 "$VALID_SHA" >"$TMP_DIR/dry-run-target.out" +rg -q "target_dir=$TMP_DIR/shared-target" "$TMP_DIR/dry-run-target.out" +rg -q "current_binary=$TMP_DIR/shared-target/debug/rustfs" "$TMP_DIR/dry-run-target.out" + +mkdir -p "$TMP_DIR/nonempty/mixed-version-upgrade" +touch "$TMP_DIR/nonempty/mixed-version-upgrade/existing.json" +if bash "$RUNNER" --dry-run --out-dir "$TMP_DIR/nonempty" --sha256 "$VALID_SHA" >"$TMP_DIR/nonempty.out" 2>"$TMP_DIR/nonempty.err"; then + echo "G09 runner should reject non-empty evidence case directories" >&2 + exit 1 +fi +rg -q "evidence case directory is not empty" "$TMP_DIR/nonempty.err"