From d076a434d47eaa4b13d463d1f08e109640a53972 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Thu, 11 Jun 2026 21:09:50 +1000 Subject: [PATCH] Feat/e2e targeted suites (#352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(e2e): add targeted suite selector + shadow-mode CI step select-suites maps a diff to the e2e suites it must run: changed files → services (manifest path/additional_paths) → consumer closure over depends_on vs publishes/subscribes → suites whose covers intersect. Special cases force-all (infra/harness/workflow, ≥3 services, submodule bumps) or a lone suite dir; empty diffs fall back to default suites. An AST scan of each suite's harness.Ensure* call graph derives per-test service coverage and emits an optional -test.run subset (RUN_PATTERN). Wired into e2e.yml as a non-gating 'Select Suites (shadow)' job that logs the computed selection to the run summary so predicted vs actual can be compared before the selector drives SUITES. * ci(e2e): add dispatch-only per-suite job blocks grouped by env Restructure the looped e2e flow into env-grouped provision -> matrix suites -> teardown blocks so each suite surfaces as its own node in the Actions graph instead of a single green tick: single_provision -> single_suite [single,cert,lb,eks] -> single_teardown multi_provision -> multi_suite [multinode] -> multi_teardown Provision resolves the VM/cluster and ships test binaries, emitting WAN IP + ssh key path as job outputs; each matrix leg runs one suite over SSH and uploads its own junit/analysis artifacts; teardown (if:always) reuses the persisted tofu workspace on the same runner to destroy. workflow_dispatch only -- runs alongside the gating e2e.yml for validation and posts no commit status. * test(e2e): add manifest drift lint (fixture-create + NATS subject) Bead 5 of the targeted-suite-selection plan. Two ratcheted drift guards in tests/e2e/manifest-lint, wired into make preflight and the spinifex.yml PR job: - Fixture lint: AST-flags direct resource-create AWS calls (c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...) in suite test files, steering them through harness.Ensure* fixtures. ~104 pre-existing call sites are captured in a checked-in baseline; the lint fails only on NEW creates. Intentional create-path tests opt out with an inline // e2e:allow-create. - Subject lint: extracts NATS subject literals from the natsSub{} registry and literal Subscribe/Publish/Request args, cross-referenced wildcard-aware against service-interfaces.yaml. Code subjects missing from the manifest fail (ratcheted); manifest subjects with no static reference warn only (many are produced dynamically). Fixed real drift surfaced on first run: declared the five spinifex.cluster.shutdown.* phase subjects under spinifex_daemon. manifest-check is now CI-enforced alongside manifest-lint (previously preflight/local-only). * feat: run-scoped e2e teardown sweep Tag every Ensure* fixture resource with e2e:run= when SPINIFEX_E2E_RUN_ID is set, and add a teardown.test suite that sweeps everything carrying that tag at end of run — reclaiming resources leaked by a crashed suite, an interrupted run, or a persistent cluster. - harness/sweep.go: tagRunResources (EC2 CreateTags) + tagRunELB (ELBv2 AddTags) + SweepRunResources in dependency order (LB, instances, NAT, snapshots, volumes, images, SGs, subnets, key pairs). Blank run id no-ops both tagging and sweeping, so the sweep can never touch an untagged or foreign resource. - harness/fixtures.go: stamp the run tag on each created resource. - tests/e2e/teardown: thin suite that runs the sweep; skips without a cluster or run id, logs the report, never fails the run. - e2e-suites.yml: build + ship teardown.test, pass the run id into suite legs, sweep on the VM in both teardown jobs before tofu destroy. --- .github/workflows/e2e-suites.yml | 799 ++++++++++++++++++ .github/workflows/e2e.yml | 37 + .github/workflows/spinifex.yml | 3 + Makefile | 15 +- docs/service-interfaces.yaml | 6 + tests/e2e/harness/fixtures.go | 16 + tests/e2e/harness/sweep.go | 333 ++++++++ tests/e2e/harness/sweep_test.go | 353 ++++++++ tests/e2e/manifest-lint/baseline.txt | 107 +++ .../manifest-lint/cmd/manifest-lint/main.go | 109 +++ tests/e2e/manifest-lint/fixtures.go | 173 ++++ tests/e2e/manifest-lint/lint.go | 127 +++ tests/e2e/manifest-lint/manifestlint_test.go | 251 ++++++ tests/e2e/manifest-lint/subjects.go | 255 ++++++ tests/e2e/select-suites/ast.go | 196 +++++ .../select-suites/cmd/select-suites/main.go | 97 +++ tests/e2e/select-suites/selector.go | 319 +++++++ tests/e2e/select-suites/selector_test.go | 305 +++++++ tests/e2e/teardown/teardown_test.go | 47 ++ 19 files changed, 3546 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/e2e-suites.yml create mode 100644 tests/e2e/harness/sweep.go create mode 100644 tests/e2e/harness/sweep_test.go create mode 100644 tests/e2e/manifest-lint/baseline.txt create mode 100644 tests/e2e/manifest-lint/cmd/manifest-lint/main.go create mode 100644 tests/e2e/manifest-lint/fixtures.go create mode 100644 tests/e2e/manifest-lint/lint.go create mode 100644 tests/e2e/manifest-lint/manifestlint_test.go create mode 100644 tests/e2e/manifest-lint/subjects.go create mode 100644 tests/e2e/select-suites/ast.go create mode 100644 tests/e2e/select-suites/cmd/select-suites/main.go create mode 100644 tests/e2e/select-suites/selector.go create mode 100644 tests/e2e/select-suites/selector_test.go create mode 100644 tests/e2e/teardown/teardown_test.go diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml new file mode 100644 index 000000000..abc71fea1 --- /dev/null +++ b/.github/workflows/e2e-suites.yml @@ -0,0 +1,799 @@ +# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only. +# +# Restructures the looped e2e.yml into env-grouped job blocks so each suite +# surfaces as its own node in the Actions graph (mulga-siv-283): +# +# single_provision ─┬─ single_suite (matrix: single, cert, lb, eks) +# └─ single_teardown (if: always) +# multi_provision ─┬─ multi_suite (matrix: multinode) +# └─ multi_teardown (if: always) +# +# Constraint: tofu state is runner-filesystem-local, so *_provision and +# *_teardown must execute on the same self-hosted runner (one runner per label +# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does +# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach +# the already-provisioned VM over its WAN IP using the key path the provision +# job emits; they share the runner home dir, so the key file persists. +# +# This file does NOT post the gating `e2e` commit status — it runs alongside +# e2e.yml for validation. Once green across a few manual runs, fold the +# triggers from e2e.yml here and retire the loop. + +name: E2E (suite blocks) + +on: + workflow_dispatch: + inputs: + single_suites: + description: "Single-node suites to run as matrix legs" + required: false + default: "single cert lb eks" + run_multinode: + description: "Also run the multinode env block" + type: boolean + required: false + default: true + test_run: + description: "Optional -test.run regex (empty = all tests in suite)" + required: false + default: "" + +permissions: + contents: read + +env: + GIT_BRANCH: ${{ github.ref_name }} + TEST_RUN: ${{ inputs.test_run || '' }} + +jobs: + # --------------------------------------------------------------------------- + # Single-node env + # --------------------------------------------------------------------------- + single_provision: + name: Single env · provision + runs-on: [ self-hosted, ci-single ] + timeout-minutes: 40 + outputs: + wan_ip: ${{ steps.vm.outputs.wan_ip }} + ssh_key: ${{ steps.vm.outputs.ssh_key }} + suites: ${{ steps.suites.outputs.list }} + matrix: ${{ steps.suites.outputs.matrix }} + env: + TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log + steps: + - name: Reap Orphan Procs + run: | + set +e + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM; sleep 3; reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + true + + - name: Clean Workspace + run: | + awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \ + | sort -r | xargs -r -I{} sudo umount -l -R {} || true + sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga + + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Re-enable Go toolchain auto-download + run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" + + - name: Compute suite matrix + id: suites + run: | + LIST="${{ inputs.single_suites || 'single cert lb eks' }}" + echo "list=${LIST}" >> "$GITHUB_OUTPUT" + JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .) + echo "matrix=${JSON}" >> "$GITHUB_OUTPUT" + + - name: Prune rootless docker cache + run: | + docker container prune -f || true + docker image prune -af || true + docker buildx prune -af --filter "until=24h" || true + docker system df || true + + - name: Build Install Artifacts + working-directory: mulga/scripts/tofu-cluster + run: | + ./build-install-artifacts.sh "${GIT_BRANCH}" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Provision VM + working-directory: mulga/scripts/tofu-cluster + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + export TF_VAR_git_branch="${GIT_BRANCH}" + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "VM provisioned" + + - name: Bootstrap via Binary Install + working-directory: mulga/scripts/tofu-cluster + run: | + SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ + ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Resolve VM SSH details + id: vm + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT" + echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" + + - name: Build eks-node AMI (libguestfs, runner) + if: contains(steps.suites.outputs.list, 'eks') + working-directory: spinifex + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + banner "build eks-node AMI (libguestfs — no host devices)" + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils + export LIBGUESTFS_BACKEND=direct + sudo chmod 0644 /boot/vmlinuz-* || true + group_open "make build-eks-node-image" + make build-eks-node-image + group_close + ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)" + + - name: Import eks-node AMI on VM + if: contains(steps.suites.outputs.list, 'eks') + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \ + spinifex/scripts/images/eks-node/manifest.conf)" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + banner "import eks-node AMI to VM image store" + scp "${SSH_OPTS[@]}" \ + /tmp/eks-node-image-build/eks-node-alpine.raw \ + "$VM":/tmp/eks-node-alpine.raw + ssh "${SSH_OPTS[@]}" "$VM" \ + "spx admin images import --file /tmp/eks-node-alpine.raw \ + --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \ + --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null + ok "eks-node AMI imported" + + - name: Install kubectl on VM + if: contains(steps.suites.outputs.list, 'eks') + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + banner "install kubectl on VM (eks data-plane subtest)" + ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF' + set -euo pipefail + KVER=v1.32.5 + case "$(uname -m)" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; + esac + cd /tmp + curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl" + curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum -c - + sudo install -m 0755 kubectl /usr/local/bin/kubectl + kubectl version --client + EOF + ok "kubectl installed on VM" + + - name: Build E2E test binaries on runner + working-directory: spinifex/tests/e2e + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + # teardown always builds: the run-scoped sweep binary runs in the + # teardown job, it is never a selectable suite leg. + SUITES_BUILD="${{ steps.suites.outputs.list }} teardown" + banner "build ${SUITES_BUILD// /, }" + group_open "go test -c" + make e2e-build SUITES_BUILD="${SUITES_BUILD}" + group_close + for f in _bin/*.test; do + ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" + done + + - name: Ship test binaries to VM + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" + tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" + ok "shipped binaries to $VM" + + - name: Start packet capture on VM br-wan + run: | + set +e + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ + "$VM":/tmp/start-net-capture.sh + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true + + single_suite: + name: Single env · suite + needs: single_provision + runs-on: [ self-hosted, ci-single ] + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }} + env: + WAN_IP: ${{ needs.single_provision.outputs.wan_ip }} + SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }} + ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }} + steps: + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Install go-junit-report + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Run ${{ matrix.suite }} + id: e2e + run: | + set +e + SUITE="${{ matrix.suite }}" + VM_HOST="tf-user@${WAN_IP}" + TEST_RUN_FLAG="" + [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}" + cleanup_remote() { + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ + "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \ + 2>/dev/null || true + } + trap cleanup_remote EXIT INT TERM + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \ + R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ + printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ + 2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \ + | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ + exit \${PIPESTATUS[0]}" + echo "rc=$?" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Collect suite artifacts from VM + if: always() + run: | + set +e + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}") + VM=tf-user@${WAN_IP} + ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + true + + - name: Convert test log to JUnit XML + if: always() + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log); SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Publish test summary + if: always() + uses: ./spinifex/.github/actions/junit-report + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + title: "E2E ${{ matrix.suite }} results" + + - name: Analyze e2e failures + if: always() + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "E2E ${{ matrix.suite }} failure analysis" + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-single-${{ matrix.suite }}-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: warn + retention-days: 14 + + - name: Fail job if suite failed + if: always() + run: | + if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then + echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})" + exit 1 + fi + + single_teardown: + name: Single env · teardown + needs: [ single_provision, single_suite ] + if: always() + runs-on: [ self-hosted, ci-single ] + timeout-minutes: 20 + env: + TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log + ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }} + WAN_IP: ${{ needs.single_provision.outputs.wan_ip }} + SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }} + steps: + # Reuse the persisted mulga/ tofu-cluster workspace from single_provision. + # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga + # again would wipe the local tofu state and orphan the VM. + - uses: actions/checkout@v6 + with: + path: spinifex + + - name: Capture network/IPsec diagnostics on VM + if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }} + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}") + VM=tf-user@${WAN_IP} + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + "$VM":/tmp/capture-net-diag.sh 2>/dev/null + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null + ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null + mkdir -p "${ARTIFACT_DIR}/lb-console" + ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \ + | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null + true + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-single-diag-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: ignore + retention-days: 14 + + # Reclaim any resource leaked by a crashed/interrupted suite before the VM + # is destroyed. Scoped to this run's tag, best-effort, never fails the job. + - name: Sweep leaked run resources on VM + if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }} + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}") + VM=tf-user@${WAN_IP} + banner "sweep run-tagged resources (run ${{ github.run_id }})" + ssh "${SSH_OPTS[@]}" "$VM" \ + "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \ + SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \ + ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \ + 2>&1 || true + true + + - name: Teardown VM + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \ + tf-user@${WAN_IP} \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + fi + UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac + done + } + reap TERM; sleep 2; reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" + ok "teardown complete" + + # --------------------------------------------------------------------------- + # Multinode env + # --------------------------------------------------------------------------- + multi_provision: + name: Multi env · provision + if: ${{ inputs.run_multinode }} + runs-on: [ self-hosted, ci-multi ] + timeout-minutes: 45 + outputs: + node1_ip: ${{ steps.cluster.outputs.node1_ip }} + ssh_key: ${{ steps.cluster.outputs.ssh_key }} + nodes_csv: ${{ steps.cluster.outputs.nodes_csv }} + env: + TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log + steps: + - name: Reap Orphan Procs + run: | + set +e + UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac + done + } + reap TERM; sleep 3; reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + true + + - name: Clean Workspace + run: | + awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \ + | sort -r | xargs -r -I{} sudo umount -l -R {} || true + sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga + + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Re-enable Go toolchain auto-download + run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" + + - name: Prune rootless docker cache + run: | + docker container prune -f || true + docker image prune -af || true + docker buildx prune -af --filter "until=24h" || true + docker system df || true + + - name: Build Install Artifacts + working-directory: mulga/scripts/tofu-cluster + run: | + ./build-install-artifacts.sh "${GIT_BRANCH}" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Provision VMs + working-directory: mulga/scripts/tofu-cluster + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + export TF_VAR_git_branch="${GIT_BRANCH}" + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "3-node cluster provisioned" + + - name: Bootstrap via Binary Install + working-directory: mulga/scripts/tofu-cluster + run: | + SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ + ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Resolve cluster SSH details + id: cluster + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}" + NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + NODES_CSV="" + for i in $(seq 0 $((NODE_COUNT - 1))); do + ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") + if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi + done + echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT" + echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" + echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT" + + - name: Build multinode binary on runner + working-directory: spinifex/tests/e2e + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + banner "build multinode, teardown" + group_open "go test -c" + make e2e-build SUITES_BUILD="multinode teardown" + group_close + ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)" + + - name: Ship test binaries to node1 + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + VM=tf-user@${{ steps.cluster.outputs.node1_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" + tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" + ok "shipped binaries to $VM" + + - name: Start packet capture on all nodes + run: | + set +e + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + for ip in "${NODE_LIST[@]}"; do + [ -z "$ip" ] && continue + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ + tf-user@"$ip":/tmp/start-net-capture.sh + ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true + done + + multi_suite: + name: Multi env · suite + needs: multi_provision + runs-on: [ self-hosted, ci-multi ] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + suite: [ multinode ] + env: + NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }} + SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }} + NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }} + ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }} + steps: + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Install go-junit-report + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Run ${{ matrix.suite }} + id: e2e + run: | + set +e + SUITE="${{ matrix.suite }}" + VM_HOST="tf-user@${NODE1_IP}" + TEST_RUN_FLAG="" + [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}" + cleanup_remote() { + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ + "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \ + 2>/dev/null || true + } + trap cleanup_remote EXIT INT TERM + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \ + printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \ + SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \ + SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \ + SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ + 2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \ + | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ + exit \${PIPESTATUS[0]}" + echo "rc=$?" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Collect suite artifacts from node1 + if: always() + run: | + set +e + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}") + ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + true + + - name: Convert test log to JUnit XML + if: always() + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log); SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Publish test summary + if: always() + uses: ./spinifex/.github/actions/junit-report + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + title: "Multi-Node E2E ${{ matrix.suite }} results" + + - name: Analyze e2e failures + if: always() + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "Multi-Node E2E ${{ matrix.suite }} failure analysis" + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: warn + retention-days: 14 + + - name: Fail job if suite failed + if: always() + run: | + if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then + echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})" + exit 1 + fi + + multi_teardown: + name: Multi env · teardown + needs: [ multi_provision, multi_suite ] + if: always() && inputs.run_multinode + runs-on: [ self-hosted, ci-multi ] + timeout-minutes: 20 + env: + TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log + SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }} + NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }} + steps: + - uses: actions/checkout@v6 + with: + path: spinifex + + # Reclaim leaked run-tagged resources before the cluster is destroyed. + - name: Sweep leaked run resources on node1 + if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }} + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}") + VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }} + banner "sweep run-tagged resources (run ${{ github.run_id }})" + ssh "${SSH_OPTS[@]}" "$VM" \ + "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \ + SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \ + SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \ + ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \ + 2>&1 || true + true + + - name: Teardown cluster + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then + IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}" + for ip in "${NODE_LIST[@]}"; do + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \ + tf-user@"$ip" \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + done + fi + UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac + done + } + reap TERM; sleep 2; reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" + ok "teardown complete" + + e2e_suites_status: + name: E2E suite blocks · summary + if: always() + needs: [ single_suite, multi_suite ] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Summarize + run: | + echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY" + echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY" + echo "multi_suite: ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY" + case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac + case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 02110e4b9..c269bf38f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,6 +36,43 @@ env: TEST_RUN: ${{ inputs.test_run || '' }} jobs: + # Shadow-mode suite selector (Bead 4, mulga-siv-278). Computes which suites + # the diff *would* run and logs it to the job summary. Does NOT gate or + # override the real SUITES env yet — it runs alongside the full matrix so we + # can compare predicted vs actual before flipping it on (Bead 6). + select_suites: + name: Select Suites (shadow) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + - name: Compute targeted suite selection + run: | + set -euo pipefail + BASE="${{ github.event.before }}" + HEAD="${{ github.sha }}" + # First push to a branch reports an all-zero before-SHA; fall back to + # the first parent so the diff is still meaningful. + if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then + BASE="${HEAD}~1" + fi + echo "## Targeted suite selection (shadow)" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "base=$BASE head=$HEAD" >> "$GITHUB_STEP_SUMMARY" + go run ./tests/e2e/select-suites/cmd/select-suites \ + --manifest docs/service-interfaces.yaml \ + --base-sha "$BASE" --head-sha "$HEAD" --repo-root . \ + 2>>"$GITHUB_STEP_SUMMARY" \ + | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "Shadow only — actual run uses SUITES='${SUITES}'." >> "$GITHUB_STEP_SUMMARY" + e2e_single: name: Single-Node E2E # Skip when the dispatch input requests no in-VM suites (e.g. diff --git a/.github/workflows/spinifex.yml b/.github/workflows/spinifex.yml index 4bf6adb44..d5a0ae5a6 100644 --- a/.github/workflows/spinifex.yml +++ b/.github/workflows/spinifex.yml @@ -58,6 +58,9 @@ jobs: - name: Build Spinifex run: make -C spinifex build + - name: Manifest check + drift lint + run: make -C spinifex manifest-check manifest-lint + - name: Run Tests with Coverage run: make -C spinifex test-cover diff --git a/Makefile b/Makefile index c3d491150..f2c7ad85a 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share # Preflight — runs the same checks as GitHub Actions (lint + vuln + tests). # Use this before committing to catch CI failures locally. preflight: - @$(MAKE) --no-print-directory QUIET=1 manifest-check lint govulncheck test-cover diff-coverage test-race test-harness + @$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness @echo -e "\n ✅ Preflight passed — safe to commit." # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the @@ -130,6 +130,17 @@ manifest-check: @echo -e "\n....Checking service-interfaces.yaml...." @go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml +# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint, +# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW +# drift beyond the baseline. +manifest-lint: + @echo -e "\n....Linting manifest drift (fixtures + subjects)...." + @go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . + +# Accept current drift into the baseline. Run after an intentional change. +manifest-lint-update: + @go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update + # Run unit tests test: @echo -e "\n....Running tests for $(GO_PROJECT_NAME)...." @@ -334,7 +345,7 @@ ansible-cluster-bootstrap: $(if $(POOL),-e cluster_external_pool=$(POOL),) \ $(_ANSIBLE_EXTRA) -.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check \ +.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ diff --git a/docs/service-interfaces.yaml b/docs/service-interfaces.yaml index af509c988..9c6f4b287 100644 --- a/docs/service-interfaces.yaml +++ b/docs/service-interfaces.yaml @@ -84,6 +84,12 @@ services: - "spinifex.node.vms" - "spinifex.nodes.discover" - "spinifex.cluster.shutdown.progress" + # Coordinated cluster-shutdown phases (fan-out, no queue group). + - "spinifex.cluster.shutdown.gate" + - "spinifex.cluster.shutdown.drain" + - "spinifex.cluster.shutdown.storage" + - "spinifex.cluster.shutdown.persist" + - "spinifex.cluster.shutdown.infra" - "spinifex.storage.config" # ELBv2 control plane (handlers/elbv2/service_nats.go topic map). - "elbv2.CreateLoadBalancer" diff --git a/tests/e2e/harness/fixtures.go b/tests/e2e/harness/fixtures.go index 044d244e1..ff05b3336 100644 --- a/tests/e2e/harness/fixtures.go +++ b/tests/e2e/harness/fixtures.go @@ -67,6 +67,11 @@ type Fixture struct { // namespace-flat). scratch string + // runID is the cross-process run id (SPINIFEX_E2E_RUN_ID). When set, + // Ensure* stamps the e2e:run= tag on every resource it creates so + // the teardown sweep can reclaim them by id. Empty disables tagging. + runID string + mu sync.Mutex memo map[string]string cleanups map[string]struct{} @@ -113,6 +118,7 @@ func NewProcessFixture(aws *AWSClient) (*Fixture, error) { EC2: aws.EC2, ELBv2: aws.ELBv2, scratch: scratch, + runID: os.Getenv(RunIDEnv), memo: map[string]string{}, cleanups: map[string]struct{}{}, }, nil @@ -132,6 +138,7 @@ func newFixture(t *testing.T, ec2c ec2iface.EC2API, elbc elbv2iface.ELBV2API) *F EC2: ec2c, ELBv2: elbc, scratch: scratch, + runID: os.Getenv(RunIDEnv), memo: map[string]string{}, cleanups: map[string]struct{}{}, } @@ -311,6 +318,7 @@ func EnsureKeyPair(t *testing.T, fx *Fixture, artifactsDir string) (string, stri if err := os.WriteFile(pemPath, []byte(aws.StringValue(out.KeyMaterial)), 0o600); err != nil { return "", nil, fmt.Errorf("write pem %s: %w", pemPath, err) } + fx.tagRunResources(aws.StringValue(out.KeyPairId)) return aws.StringValue(out.KeyName), func() error { _, derr := fx.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(name)}) return derr @@ -378,6 +386,7 @@ func EnsureAMI(t *testing.T, fx *Fixture, src AMISource) string { return "", nil, fmt.Errorf("CreateImage: %w", err) } imageID = aws.StringValue(out.ImageId) + fx.tagRunResources(imageID) } if err := pollUntil(t, 10*time.Minute, 2*time.Second, func() (bool, error) { @@ -525,6 +534,7 @@ func EnsureSubnet(t *testing.T, fx *Fixture, vpcID, cidr, az string) string { return "", nil, fmt.Errorf("CreateSubnet %s: %w", cidr, err) } subnetID := aws.StringValue(out.Subnet.SubnetId) + fx.tagRunResources(subnetID) return subnetID, func() error { _, derr := fx.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{ SubnetId: aws.String(subnetID), @@ -557,6 +567,7 @@ func EnsureSG(t *testing.T, fx *Fixture, vpcID, namePrefix string) string { return "", nil, fmt.Errorf("CreateSecurityGroup %s: %w", name, err) } sgID := aws.StringValue(out.GroupId) + fx.tagRunResources(sgID) return sgID, func() error { _, derr := fx.EC2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ GroupId: aws.String(sgID), @@ -621,6 +632,7 @@ func EnsureInstance(t *testing.T, fx *Fixture, spec InstanceSpec) string { return "", nil, fmt.Errorf("RunInstances returned 0 instances") } instID := aws.StringValue(out.Instances[0].InstanceId) + fx.tagRunResources(instID) if err := pollUntil(t, 5*time.Minute, 2*time.Second, func() (bool, error) { d, derr := fx.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ @@ -672,6 +684,7 @@ func EnsureVolume(t *testing.T, fx *Fixture, az string, sizeGiB int64) string { return "", nil, fmt.Errorf("CreateVolume: %w", err) } volID := aws.StringValue(out.VolumeId) + fx.tagRunResources(volID) if err := pollUntil(t, 5*time.Minute, 2*time.Second, func() (bool, error) { d, derr := fx.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{ @@ -738,6 +751,7 @@ func EnsureSnapshot(t *testing.T, fx *Fixture, spec SnapshotSpec) string { return "", nil, fmt.Errorf("CreateSnapshot: %w", err) } snapID := aws.StringValue(out.SnapshotId) + fx.tagRunResources(snapID) if err := pollUntil(t, 10*time.Minute, 2*time.Second, func() (bool, error) { d, derr := fx.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{ @@ -795,6 +809,7 @@ func EnsureNATGateway(t *testing.T, fx *Fixture, subnetID, allocationID string) return "", nil, fmt.Errorf("CreateNatGateway: %w", err) } ngwID := aws.StringValue(out.NatGateway.NatGatewayId) + fx.tagRunResources(ngwID) if err := pollUntil(t, 5*time.Minute, 5*time.Second, func() (bool, error) { d, derr := fx.EC2.DescribeNatGateways(&ec2.DescribeNatGatewaysInput{ @@ -869,6 +884,7 @@ func EnsureLoadBalancer(t *testing.T, fx *Fixture, spec LoadBalancerSpec) string return "", nil, fmt.Errorf("CreateLoadBalancer returned 0 LBs") } lbARN := aws.StringValue(out.LoadBalancers[0].LoadBalancerArn) + fx.tagRunELB(lbARN) if err := pollUntil(t, 10*time.Minute, 5*time.Second, func() (bool, error) { d, derr := fx.ELBv2.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{ diff --git a/tests/e2e/harness/sweep.go b/tests/e2e/harness/sweep.go new file mode 100644 index 000000000..750ae74cb --- /dev/null +++ b/tests/e2e/harness/sweep.go @@ -0,0 +1,333 @@ +//go:build e2e + +// Run-scoped resource tagging + teardown sweep (mulga-siv-277). +// +// Every resource the Ensure* fixtures create is tagged e2e:run= when +// SPINIFEX_E2E_RUN_ID is set (CI: github.run_id; local: a temp id). A thin +// teardown.test binary then sweeps everything carrying that tag at the end of a +// run — reclaiming resources leaked by a crashed suite, an interrupted run, or +// a persistent (non-tofu-destroyed) cluster. The run-id scoping keeps the sweep +// precise: with no run id, tagging and sweeping both no-op, so the sweep can +// never delete an untagged or foreign resource. +package harness + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" +) + +const ( + // RunIDEnv carries the cross-process run id shared by every suite binary + // and the teardown sweep within one e2e run. + RunIDEnv = "SPINIFEX_E2E_RUN_ID" + // runTagKey is the EC2/ELBv2 tag key stamped on fixture-created resources. + runTagKey = "e2e:run" +) + +// tagRunResources stamps the run tag on the given EC2 resource IDs when a run +// id is configured. Best-effort: a tagging failure is logged, not fatal — the +// resource still functions, it just won't be reclaimable by the id sweep. +func (f *Fixture) tagRunResources(ids ...string) { + if f.runID == "" || len(ids) == 0 { + return + } + _, err := f.EC2.CreateTags(&ec2.CreateTagsInput{ + Resources: aws.StringSlice(ids), + Tags: []*ec2.Tag{{Key: aws.String(runTagKey), Value: aws.String(f.runID)}}, + }) + if err != nil { + f.logf("tagRunResources %v: %v", ids, err) + } +} + +// tagRunELB stamps the run tag on an ELBv2 resource (load balancers use a +// distinct AddTags API, not EC2 CreateTags). Best-effort like tagRunResources. +func (f *Fixture) tagRunELB(arns ...string) { + if f.runID == "" || len(arns) == 0 { + return + } + _, err := f.ELBv2.AddTags(&elbv2.AddTagsInput{ + ResourceArns: aws.StringSlice(arns), + Tags: []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String(f.runID)}}, + }) + if err != nil { + f.logf("tagRunELB %v: %v", arns, err) + } +} + +// SweepReport records what a sweep deleted and any non-fatal errors hit along +// the way. Sweeping is best-effort: one resource's deletion failure never stops +// the rest. +type SweepReport struct { + Deleted map[string][]string // resource kind → deleted IDs + Errors []error +} + +func newSweepReport() *SweepReport { return &SweepReport{Deleted: map[string][]string{}} } + +func (r *SweepReport) deleted(kind string, ids ...string) { + if len(ids) > 0 { + r.Deleted[kind] = append(r.Deleted[kind], ids...) + } +} + +func (r *SweepReport) errf(format string, args ...any) { + r.Errors = append(r.Errors, fmt.Errorf(format, args...)) +} + +// SweepRunResources deletes every EC2/ELBv2 resource tagged e2e:run=, +// in dependency order (load balancers → instances → NAT gateways → snapshots → +// volumes → images → security groups → subnets → key pairs). It is idempotent +// and best-effort; callers inspect SweepReport but a non-empty Errors list is +// not necessarily a run failure (resources may already be gone, or still +// settling). A blank runID returns an empty report without touching anything. +func SweepRunResources(ec2c ec2iface.EC2API, elbc elbv2iface.ELBV2API, runID string) *SweepReport { + rep := newSweepReport() + if runID == "" { + return rep + } + + sweepLoadBalancers(elbc, runID, rep) + sweepInstances(ec2c, runID, rep) + sweepNatGateways(ec2c, runID, rep) + sweepSnapshots(ec2c, runID, rep) + sweepVolumes(ec2c, runID, rep) + sweepImages(ec2c, runID, rep) + sweepSecurityGroups(ec2c, runID, rep) + sweepSubnets(ec2c, runID, rep) + sweepKeyPairs(ec2c, runID, rep) + + return rep +} + +func runFilter(runID string) []*ec2.Filter { + return []*ec2.Filter{{ + Name: aws.String("tag:" + runTagKey), + Values: []*string{aws.String(runID)}, + }} +} + +func sweepInstances(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeInstances(&ec2.DescribeInstancesInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeInstances: %w", err) + return + } + var ids []string + for _, r := range out.Reservations { + for _, inst := range r.Instances { + if s := aws.StringValue(inst.State.Name); s == "terminated" || s == "shutting-down" { + continue + } + ids = append(ids, aws.StringValue(inst.InstanceId)) + } + } + if len(ids) == 0 { + return + } + if _, err := ec2c.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: aws.StringSlice(ids), + }); err != nil { + rep.errf("TerminateInstances %v: %w", ids, err) + return + } + rep.deleted("instance", ids...) + // Wait (bounded) for termination so dependent SG/subnet deletes don't fail + // with DependencyViolation. + waitInstancesTerminated(ec2c, ids, rep) +} + +func waitInstancesTerminated(ec2c ec2iface.EC2API, ids []string, rep *SweepReport) { + deadline := time.Now().Add(3 * time.Minute) + for { + out, err := ec2c.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice(ids), + }) + if err != nil { + rep.errf("wait terminate DescribeInstances: %w", err) + return + } + allDone := true + for _, r := range out.Reservations { + for _, inst := range r.Instances { + if aws.StringValue(inst.State.Name) != "terminated" { + allDone = false + } + } + } + if allDone || time.Now().After(deadline) { + return + } + time.Sleep(3 * time.Second) + } +} + +func sweepNatGateways(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeNatGateways(&ec2.DescribeNatGatewaysInput{Filter: runFilter(runID)}) + if err != nil { + rep.errf("DescribeNatGateways: %w", err) + return + } + for _, ngw := range out.NatGateways { + if s := aws.StringValue(ngw.State); s == "deleted" || s == "deleting" { + continue + } + id := aws.StringValue(ngw.NatGatewayId) + if _, err := ec2c.DeleteNatGateway(&ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(id)}); err != nil { + rep.errf("DeleteNatGateway %s: %w", id, err) + continue + } + rep.deleted("natgateway", id) + } +} + +func sweepSnapshots(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeSnapshots(&ec2.DescribeSnapshotsInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeSnapshots: %w", err) + return + } + for _, s := range out.Snapshots { + id := aws.StringValue(s.SnapshotId) + if _, err := ec2c.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: aws.String(id)}); err != nil { + rep.errf("DeleteSnapshot %s: %w", id, err) + continue + } + rep.deleted("snapshot", id) + } +} + +func sweepVolumes(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeVolumes(&ec2.DescribeVolumesInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeVolumes: %w", err) + return + } + for _, v := range out.Volumes { + if aws.StringValue(v.State) == "deleting" { + continue + } + id := aws.StringValue(v.VolumeId) + if _, err := ec2c.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(id)}); err != nil { + rep.errf("DeleteVolume %s: %w", id, err) + continue + } + rep.deleted("volume", id) + } +} + +func sweepImages(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeImages(&ec2.DescribeImagesInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeImages: %w", err) + return + } + for _, img := range out.Images { + id := aws.StringValue(img.ImageId) + if _, err := ec2c.DeregisterImage(&ec2.DeregisterImageInput{ImageId: aws.String(id)}); err != nil { + rep.errf("DeregisterImage %s: %w", id, err) + continue + } + rep.deleted("image", id) + } +} + +func sweepSecurityGroups(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeSecurityGroups: %w", err) + return + } + for _, sg := range out.SecurityGroups { + if aws.StringValue(sg.GroupName) == "default" { + continue // never delete a VPC's default SG + } + id := aws.StringValue(sg.GroupId) + if _, err := ec2c.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: aws.String(id)}); err != nil { + rep.errf("DeleteSecurityGroup %s: %w", id, err) + continue + } + rep.deleted("securitygroup", id) + } +} + +func sweepSubnets(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeSubnets(&ec2.DescribeSubnetsInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeSubnets: %w", err) + return + } + for _, sn := range out.Subnets { + id := aws.StringValue(sn.SubnetId) + if _, err := ec2c.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(id)}); err != nil { + rep.errf("DeleteSubnet %s: %w", id, err) + continue + } + rep.deleted("subnet", id) + } +} + +func sweepKeyPairs(ec2c ec2iface.EC2API, runID string, rep *SweepReport) { + out, err := ec2c.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{Filters: runFilter(runID)}) + if err != nil { + rep.errf("DescribeKeyPairs: %w", err) + return + } + for _, kp := range out.KeyPairs { + name := aws.StringValue(kp.KeyName) + if _, err := ec2c.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(name)}); err != nil { + rep.errf("DeleteKeyPair %s: %w", name, err) + continue + } + rep.deleted("keypair", name) + } +} + +// sweepLoadBalancers deletes ELBv2 load balancers carrying the run tag. ELBv2 +// Describe has no tag filter, so it lists then matches via DescribeTags. +func sweepLoadBalancers(elbc elbv2iface.ELBV2API, runID string, rep *SweepReport) { + if elbc == nil { + return + } + out, err := elbc.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{}) + if err != nil { + rep.errf("DescribeLoadBalancers: %w", err) + return + } + var arns []string + for _, lb := range out.LoadBalancers { + arns = append(arns, aws.StringValue(lb.LoadBalancerArn)) + } + for _, arn := range arns { + tags, terr := elbc.DescribeTags(&elbv2.DescribeTagsInput{ResourceArns: []*string{aws.String(arn)}}) + if terr != nil { + rep.errf("DescribeTags %s: %w", arn, terr) + continue + } + if !elbHasRunTag(tags, runID) { + continue + } + if _, err := elbc.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: aws.String(arn)}); err != nil { + rep.errf("DeleteLoadBalancer %s: %w", arn, err) + continue + } + rep.deleted("loadbalancer", arn) + } +} + +func elbHasRunTag(out *elbv2.DescribeTagsOutput, runID string) bool { + for _, td := range out.TagDescriptions { + for _, tag := range td.Tags { + if aws.StringValue(tag.Key) == runTagKey && aws.StringValue(tag.Value) == runID { + return true + } + } + } + return false +} diff --git a/tests/e2e/harness/sweep_test.go b/tests/e2e/harness/sweep_test.go new file mode 100644 index 000000000..0b19c6b41 --- /dev/null +++ b/tests/e2e/harness/sweep_test.go @@ -0,0 +1,353 @@ +//go:build e2e + +package harness + +import ( + "sort" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" +) + +// sweepFakeEC2 returns a seeded set of resources from each Describe* and +// records every delete/terminate call. The embedded ec2iface.EC2API panics on +// any un-mocked method so the sweep can't silently grow a new dependency. +type sweepFakeEC2 struct { + ec2iface.EC2API + + instances []*ec2.Instance + natGateways []*ec2.NatGateway + snapshots []*ec2.Snapshot + volumes []*ec2.Volume + images []*ec2.Image + securityGroups []*ec2.SecurityGroup + subnets []*ec2.Subnet + keyPairs []*ec2.KeyPairInfo + + deleted map[string][]string + tagged map[string][]string // runID → resource IDs passed to CreateTags + describeN map[string]int +} + +func newSweepFakeEC2() *sweepFakeEC2 { + return &sweepFakeEC2{deleted: map[string][]string{}, tagged: map[string][]string{}, describeN: map[string]int{}} +} + +func (f *sweepFakeEC2) rec(kind string, ids ...string) { + f.deleted[kind] = append(f.deleted[kind], ids...) +} + +func (f *sweepFakeEC2) CreateTags(in *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { + var runID string + for _, t := range in.Tags { + if aws.StringValue(t.Key) == runTagKey { + runID = aws.StringValue(t.Value) + } + } + f.tagged[runID] = append(f.tagged[runID], aws.StringValueSlice(in.Resources)...) + return &ec2.CreateTagsOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeInstances(in *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + // Wait-for-terminated path describes by explicit IDs — report them gone so + // waitInstancesTerminated returns on the first poll. + if len(in.InstanceIds) > 0 { + var insts []*ec2.Instance + for _, id := range in.InstanceIds { + insts = append(insts, &ec2.Instance{ + InstanceId: id, + State: &ec2.InstanceState{Name: aws.String("terminated")}, + }) + } + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{Instances: insts}}, + }, nil + } + f.describeN["instances"]++ + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{Instances: f.instances}}, + }, nil +} + +func (f *sweepFakeEC2) TerminateInstances(in *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { + f.rec("instance", aws.StringValueSlice(in.InstanceIds)...) + return &ec2.TerminateInstancesOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeNatGateways(*ec2.DescribeNatGatewaysInput) (*ec2.DescribeNatGatewaysOutput, error) { + f.describeN["natgateways"]++ + return &ec2.DescribeNatGatewaysOutput{NatGateways: f.natGateways}, nil +} + +func (f *sweepFakeEC2) DeleteNatGateway(in *ec2.DeleteNatGatewayInput) (*ec2.DeleteNatGatewayOutput, error) { + f.rec("natgateway", aws.StringValue(in.NatGatewayId)) + return &ec2.DeleteNatGatewayOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeSnapshots(*ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error) { + f.describeN["snapshots"]++ + return &ec2.DescribeSnapshotsOutput{Snapshots: f.snapshots}, nil +} + +func (f *sweepFakeEC2) DeleteSnapshot(in *ec2.DeleteSnapshotInput) (*ec2.DeleteSnapshotOutput, error) { + f.rec("snapshot", aws.StringValue(in.SnapshotId)) + return &ec2.DeleteSnapshotOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeVolumes(*ec2.DescribeVolumesInput) (*ec2.DescribeVolumesOutput, error) { + f.describeN["volumes"]++ + return &ec2.DescribeVolumesOutput{Volumes: f.volumes}, nil +} + +func (f *sweepFakeEC2) DeleteVolume(in *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) { + f.rec("volume", aws.StringValue(in.VolumeId)) + return &ec2.DeleteVolumeOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeImages(*ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { + f.describeN["images"]++ + return &ec2.DescribeImagesOutput{Images: f.images}, nil +} + +func (f *sweepFakeEC2) DeregisterImage(in *ec2.DeregisterImageInput) (*ec2.DeregisterImageOutput, error) { + f.rec("image", aws.StringValue(in.ImageId)) + return &ec2.DeregisterImageOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeSecurityGroups(*ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) { + f.describeN["securitygroups"]++ + return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: f.securityGroups}, nil +} + +func (f *sweepFakeEC2) DeleteSecurityGroup(in *ec2.DeleteSecurityGroupInput) (*ec2.DeleteSecurityGroupOutput, error) { + f.rec("securitygroup", aws.StringValue(in.GroupId)) + return &ec2.DeleteSecurityGroupOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeSubnets(*ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) { + f.describeN["subnets"]++ + return &ec2.DescribeSubnetsOutput{Subnets: f.subnets}, nil +} + +func (f *sweepFakeEC2) DeleteSubnet(in *ec2.DeleteSubnetInput) (*ec2.DeleteSubnetOutput, error) { + f.rec("subnet", aws.StringValue(in.SubnetId)) + return &ec2.DeleteSubnetOutput{}, nil +} + +func (f *sweepFakeEC2) DescribeKeyPairs(*ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) { + f.describeN["keypairs"]++ + return &ec2.DescribeKeyPairsOutput{KeyPairs: f.keyPairs}, nil +} + +func (f *sweepFakeEC2) DeleteKeyPair(in *ec2.DeleteKeyPairInput) (*ec2.DeleteKeyPairOutput, error) { + f.rec("keypair", aws.StringValue(in.KeyName)) + return &ec2.DeleteKeyPairOutput{}, nil +} + +// sweepFakeELB seeds DescribeLoadBalancers + per-ARN DescribeTags and records +// deletes / AddTags. +type sweepFakeELB struct { + elbv2iface.ELBV2API + + lbs []*elbv2.LoadBalancer + tags map[string][]*elbv2.Tag // ARN → tags + deleted []string + added map[string][]string // runID → ARNs passed to AddTags +} + +func newSweepFakeELB() *sweepFakeELB { + return &sweepFakeELB{tags: map[string][]*elbv2.Tag{}, added: map[string][]string{}} +} + +func (f *sweepFakeELB) DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) { + return &elbv2.DescribeLoadBalancersOutput{LoadBalancers: f.lbs}, nil +} + +func (f *sweepFakeELB) DescribeTags(in *elbv2.DescribeTagsInput) (*elbv2.DescribeTagsOutput, error) { + var descs []*elbv2.TagDescription + for _, arn := range in.ResourceArns { + descs = append(descs, &elbv2.TagDescription{ + ResourceArn: arn, + Tags: f.tags[aws.StringValue(arn)], + }) + } + return &elbv2.DescribeTagsOutput{TagDescriptions: descs}, nil +} + +func (f *sweepFakeELB) DeleteLoadBalancer(in *elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) { + f.deleted = append(f.deleted, aws.StringValue(in.LoadBalancerArn)) + return &elbv2.DeleteLoadBalancerOutput{}, nil +} + +func (f *sweepFakeELB) AddTags(in *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) { + var runID string + for _, t := range in.Tags { + if aws.StringValue(t.Key) == runTagKey { + runID = aws.StringValue(t.Value) + } + } + f.added[runID] = append(f.added[runID], aws.StringValueSlice(in.ResourceArns)...) + return &elbv2.AddTagsOutput{}, nil +} + +var ( + _ ec2iface.EC2API = (*sweepFakeEC2)(nil) + _ elbv2iface.ELBV2API = (*sweepFakeELB)(nil) +) + +// TestSweepRunResources_BlankRunIDNoop verifies a blank run id deletes nothing +// and never dials AWS. +func TestSweepRunResources_BlankRunIDNoop(t *testing.T) { + ec2c := newSweepFakeEC2() + elbc := newSweepFakeELB() + + rep := SweepRunResources(ec2c, elbc, "") + + if len(rep.Deleted) != 0 || len(rep.Errors) != 0 { + t.Fatalf("blank run id: report = %+v, want empty", rep) + } + if len(ec2c.describeN) != 0 { + t.Fatalf("blank run id issued Describe calls: %v", ec2c.describeN) + } +} + +// TestSweepRunResources_DeletesEachKind seeds one live resource of every kind +// and verifies the sweep deletes them all and records them in the report. +func TestSweepRunResources_DeletesEachKind(t *testing.T) { + ec2c := newSweepFakeEC2() + ec2c.instances = []*ec2.Instance{{ + InstanceId: aws.String("i-1"), + State: &ec2.InstanceState{Name: aws.String("running")}, + }} + ec2c.natGateways = []*ec2.NatGateway{{ + NatGatewayId: aws.String("nat-1"), State: aws.String("available"), + }} + ec2c.snapshots = []*ec2.Snapshot{{SnapshotId: aws.String("snap-1")}} + ec2c.volumes = []*ec2.Volume{{VolumeId: aws.String("vol-1"), State: aws.String("available")}} + ec2c.images = []*ec2.Image{{ImageId: aws.String("ami-1")}} + ec2c.securityGroups = []*ec2.SecurityGroup{{GroupId: aws.String("sg-1"), GroupName: aws.String("e2e-sg")}} + ec2c.subnets = []*ec2.Subnet{{SubnetId: aws.String("subnet-1")}} + ec2c.keyPairs = []*ec2.KeyPairInfo{{KeyName: aws.String("e2e-key")}} + + arn := "arn:lb/1" + elbc := newSweepFakeELB() + elbc.lbs = []*elbv2.LoadBalancer{{LoadBalancerArn: aws.String(arn)}} + elbc.tags[arn] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-1")}} + + rep := SweepRunResources(ec2c, elbc, "run-1") + + want := map[string]string{ + "instance": "i-1", "natgateway": "nat-1", "snapshot": "snap-1", + "volume": "vol-1", "image": "ami-1", "securitygroup": "sg-1", + "subnet": "subnet-1", "keypair": "e2e-key", "loadbalancer": arn, + } + for kind, id := range want { + got := rep.Deleted[kind] + if len(got) != 1 || got[0] != id { + t.Errorf("Deleted[%s] = %v, want [%s]", kind, got, id) + } + } + if len(rep.Errors) != 0 { + t.Fatalf("unexpected errors: %v", rep.Errors) + } +} + +// TestSweepInstances_SkipsTerminated verifies already-terminated instances are +// not re-terminated. +func TestSweepInstances_SkipsTerminated(t *testing.T) { + ec2c := newSweepFakeEC2() + ec2c.instances = []*ec2.Instance{{ + InstanceId: aws.String("i-dead"), + State: &ec2.InstanceState{Name: aws.String("terminated")}, + }} + rep := SweepRunResources(ec2c, newSweepFakeELB(), "run-1") + if len(rep.Deleted["instance"]) != 0 { + t.Fatalf("terminated instance swept: %v", rep.Deleted["instance"]) + } +} + +// TestSweepSecurityGroups_SkipsDefault verifies the default SG is never +// deleted even if it carries the run tag. +func TestSweepSecurityGroups_SkipsDefault(t *testing.T) { + ec2c := newSweepFakeEC2() + ec2c.securityGroups = []*ec2.SecurityGroup{ + {GroupId: aws.String("sg-def"), GroupName: aws.String("default")}, + {GroupId: aws.String("sg-e2e"), GroupName: aws.String("e2e")}, + } + rep := SweepRunResources(ec2c, newSweepFakeELB(), "run-1") + got := rep.Deleted["securitygroup"] + if len(got) != 1 || got[0] != "sg-e2e" { + t.Fatalf("securitygroup deleted = %v, want [sg-e2e]", got) + } +} + +// TestSweepLoadBalancers_MatchesByTag verifies only LBs carrying the matching +// run tag are deleted (ELBv2 Describe has no tag filter). +func TestSweepLoadBalancers_MatchesByTag(t *testing.T) { + elbc := newSweepFakeELB() + elbc.lbs = []*elbv2.LoadBalancer{ + {LoadBalancerArn: aws.String("arn:match")}, + {LoadBalancerArn: aws.String("arn:other")}, + {LoadBalancerArn: aws.String("arn:untagged")}, + } + elbc.tags["arn:match"] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-1")}} + elbc.tags["arn:other"] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-2")}} + + SweepRunResources(newSweepFakeEC2(), elbc, "run-1") + + sort.Strings(elbc.deleted) + if len(elbc.deleted) != 1 || elbc.deleted[0] != "arn:match" { + t.Fatalf("deleted LBs = %v, want [arn:match]", elbc.deleted) + } +} + +// TestSweepLoadBalancers_NilClientNoop verifies a nil ELBv2 client is tolerated +// (suites with no LB surface). +func TestSweepLoadBalancers_NilClientNoop(t *testing.T) { + rep := SweepRunResources(newSweepFakeEC2(), nil, "run-1") + if len(rep.Deleted["loadbalancer"]) != 0 { + t.Fatalf("nil elbc swept LBs: %v", rep.Deleted["loadbalancer"]) + } +} + +// TestTagRunResources_NoRunIDSkips verifies tagging no-ops without a run id. +func TestTagRunResources_NoRunIDSkips(t *testing.T) { + ec2c := newSweepFakeEC2() + fx := newFixture(t, ec2c, newSweepFakeELB()) + fx.runID = "" + + fx.tagRunResources("i-1") + if len(ec2c.tagged) != 0 { + t.Fatalf("tagRunResources tagged with empty run id: %v", ec2c.tagged) + } +} + +// TestTagRunResources_StampsRunTag verifies the configured run id is stamped. +func TestTagRunResources_StampsRunTag(t *testing.T) { + ec2c := newSweepFakeEC2() + fx := newFixture(t, ec2c, newSweepFakeELB()) + fx.runID = "run-9" + + fx.tagRunResources("vol-1", "vol-2") + got := ec2c.tagged["run-9"] + if len(got) != 2 || got[0] != "vol-1" || got[1] != "vol-2" { + t.Fatalf("tagged[run-9] = %v, want [vol-1 vol-2]", got) + } +} + +// TestTagRunELB_StampsRunTag verifies the ELBv2 AddTags path stamps the run id. +func TestTagRunELB_StampsRunTag(t *testing.T) { + elbc := newSweepFakeELB() + fx := newFixture(t, newSweepFakeEC2(), elbc) + fx.runID = "run-9" + + fx.tagRunELB("arn:lb/1") + got := elbc.added["run-9"] + if len(got) != 1 || got[0] != "arn:lb/1" { + t.Fatalf("added[run-9] = %v, want [arn:lb/1]", got) + } +} diff --git a/tests/e2e/manifest-lint/baseline.txt b/tests/e2e/manifest-lint/baseline.txt new file mode 100644 index 000000000..17f39d0f8 --- /dev/null +++ b/tests/e2e/manifest-lint/baseline.txt @@ -0,0 +1,107 @@ +# manifest-lint baseline — pre-existing drift accepted by the ratchet. +# Regenerate with: make manifest-lint-update. Migrate entries away over time. +# Lines: "create\t\t\t#" or "subject\t\t". +create tests/e2e/eks/eks_test.go CreateCluster #1 +create tests/e2e/eks/eks_test.go CreateSubnet #1 +create tests/e2e/eks/eks_test.go CreateVpc #1 +create tests/e2e/gpu/gpu_test.go RunInstances #1 +create tests/e2e/gpu/gpu_test.go RunInstances #2 +create tests/e2e/gpu/helpers_test.go RunInstances #1 +create tests/e2e/lb/lb_test.go CreateInternetGateway #1 +create tests/e2e/lb/lb_test.go CreateKeyPair #1 +create tests/e2e/lb/lb_test.go CreateListener #1 +create tests/e2e/lb/lb_test.go CreateLoadBalancer #1 +create tests/e2e/lb/lb_test.go CreateSubnet #1 +create tests/e2e/lb/lb_test.go CreateTargetGroup #1 +create tests/e2e/lb/lb_test.go CreateTargetGroup #2 +create tests/e2e/lb/lb_test.go CreateVpc #1 +create tests/e2e/lb/lb_test.go RunInstances #1 +create tests/e2e/lb/lb_test.go RunInstances #2 +create tests/e2e/multinode/baseline_test.go RunInstances #1 +create tests/e2e/multinode/helpers_test.go RunInstances #1 +create tests/e2e/multinode/placement_nat_test.go AllocateAddress #1 +create tests/e2e/multinode/placement_nat_test.go CreateInternetGateway #1 +create tests/e2e/multinode/placement_nat_test.go CreateNatGateway #1 +create tests/e2e/multinode/placement_nat_test.go CreateRouteTable #1 +create tests/e2e/multinode/placement_nat_test.go CreateSecurityGroup #1 +create tests/e2e/multinode/placement_nat_test.go CreateSecurityGroup #2 +create tests/e2e/multinode/placement_nat_test.go CreateSubnet #1 +create tests/e2e/multinode/placement_nat_test.go CreateSubnet #2 +create tests/e2e/multinode/placement_nat_test.go CreateVpc #1 +create tests/e2e/multinode/placement_nat_test.go RunInstances #1 +create tests/e2e/multinode/placement_nat_test.go RunInstances #2 +create tests/e2e/multinode/volume_test.go CreateVolume #1 +create tests/e2e/multinode/vpc_test.go CreateSubnet #1 +create tests/e2e/multinode/vpc_test.go CreateVpc #1 +create tests/e2e/multinode/vpc_test.go RunInstances #1 +create tests/e2e/reboot/reboot_test.go CreateInternetGateway #1 +create tests/e2e/reboot/reboot_test.go CreateKeyPair #1 +create tests/e2e/reboot/reboot_test.go CreateListener #1 +create tests/e2e/reboot/reboot_test.go CreateLoadBalancer #1 +create tests/e2e/reboot/reboot_test.go CreateSecurityGroup #1 +create tests/e2e/reboot/reboot_test.go CreateSubnet #1 +create tests/e2e/reboot/reboot_test.go CreateTargetGroup #1 +create tests/e2e/reboot/reboot_test.go CreateVpc #1 +create tests/e2e/reboot/reboot_test.go RunInstances #1 +create tests/e2e/single/account_test.go CreateInternetGateway #1 +create tests/e2e/single/account_test.go CreateInternetGateway #2 +create tests/e2e/single/account_test.go CreateKeyPair #1 +create tests/e2e/single/account_test.go CreateKeyPair #2 +create tests/e2e/single/account_test.go CreateKeyPair #3 +create tests/e2e/single/account_test.go CreateKeyPair #4 +create tests/e2e/single/account_test.go CreateKeyPair #5 +create tests/e2e/single/account_test.go CreateKeyPair #6 +create tests/e2e/single/account_test.go CreateKeyPair #7 +create tests/e2e/single/account_test.go CreateSnapshot #1 +create tests/e2e/single/account_test.go CreateSnapshot #2 +create tests/e2e/single/account_test.go CreateSnapshot #3 +create tests/e2e/single/account_test.go CreateSubnet #1 +create tests/e2e/single/account_test.go CreateSubnet #2 +create tests/e2e/single/account_test.go CreateSubnet #3 +create tests/e2e/single/account_test.go CreateVolume #1 +create tests/e2e/single/account_test.go CreateVolume #2 +create tests/e2e/single/account_test.go CreateVpc #1 +create tests/e2e/single/account_test.go CreateVpc #2 +create tests/e2e/single/account_test.go ImportKeyPair #1 +create tests/e2e/single/account_test.go RunInstances #1 +create tests/e2e/single/account_test.go RunInstances #2 +create tests/e2e/single/baseline_test.go CreateInternetGateway #1 +create tests/e2e/single/baseline_test.go CreateSubnet #1 +create tests/e2e/single/baseline_test.go CreateVpc #1 +create tests/e2e/single/baseline_test.go RunInstances #1 +create tests/e2e/single/datapath_test.go CreateSecurityGroup #1 +create tests/e2e/single/datapath_test.go RunInstances #1 +create tests/e2e/single/iam_test.go CreateKeyPair #1 +create tests/e2e/single/imds_test.go CreateInternetGateway #1 +create tests/e2e/single/imds_test.go CreateRouteTable #1 +create tests/e2e/single/imds_test.go CreateSubnet #1 +create tests/e2e/single/imds_test.go CreateVpc #1 +create tests/e2e/single/imds_test.go RunInstances #1 +create tests/e2e/single/instance_iam_profile_test.go RunInstances #1 +create tests/e2e/single/keypair_test.go ImportKeyPair #1 +create tests/e2e/single/lifecycle_test.go CreateVolume #1 +create tests/e2e/single/lifecycle_test.go RunInstances #1 +create tests/e2e/single/natgw_test.go AllocateAddress #1 +create tests/e2e/single/natgw_test.go CreateNatGateway #1 +create tests/e2e/single/natgw_test.go CreateRouteTable #1 +create tests/e2e/single/natgw_test.go CreateSubnet #1 +create tests/e2e/single/natgw_test.go RunInstances #1 +create tests/e2e/single/natgw_test.go RunInstances #2 +create tests/e2e/single/negative_test.go CreateImage #1 +create tests/e2e/single/negative_test.go CreateKeyPair #1 +create tests/e2e/single/negative_test.go ImportKeyPair #1 +create tests/e2e/single/negative_test.go ImportKeyPair #2 +create tests/e2e/single/negative_test.go RunInstances #1 +create tests/e2e/single/negative_test.go RunInstances #2 +create tests/e2e/single/negative_test.go RunInstances #3 +create tests/e2e/single/negative_test.go RunInstances #4 +create tests/e2e/single/snapshot_test.go CreateSnapshot #1 +create tests/e2e/single/volume_test.go CreateVolume #1 +create tests/e2e/single/vpc_test.go CreateInternetGateway #1 +create tests/e2e/single/vpc_test.go CreateRouteTable #1 +create tests/e2e/single/vpc_test.go CreateRouteTable #2 +create tests/e2e/single/vpc_test.go CreateRouteTable #3 +create tests/e2e/single/vpc_test.go CreateSubnet #1 +create tests/e2e/single/vpc_test.go CreateSubnet #2 +create tests/e2e/single/vpc_test.go CreateVpc #1 +create tests/e2e/single/vpc_test.go RunInstances #1 diff --git a/tests/e2e/manifest-lint/cmd/manifest-lint/main.go b/tests/e2e/manifest-lint/cmd/manifest-lint/main.go new file mode 100644 index 000000000..f4a0d08a5 --- /dev/null +++ b/tests/e2e/manifest-lint/cmd/manifest-lint/main.go @@ -0,0 +1,109 @@ +// Command manifest-lint runs the e2e manifest drift guards (Bead 5): +// fixture-create lint + NATS subject lint, both ratcheted against a checked-in +// baseline. +// +// Usage: +// +// manifest-lint [-repo-root .] [-manifest docs/service-interfaces.yaml] +// [-baseline tests/e2e/manifest-lint/baseline.txt] [-update] +// +// Exits non-zero on any NEW violation beyond the baseline. -update rewrites the +// baseline from the current findings (use after intentional changes). +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" + manifestlint "github.com/mulgadc/spinifex/tests/e2e/manifest-lint" +) + +func main() { + var repoRoot, manifestPath, baselinePath string + var update bool + flag.StringVar(&repoRoot, "repo-root", ".", "spinifex repo root") + flag.StringVar(&manifestPath, "manifest", "docs/service-interfaces.yaml", "path to the manifest YAML") + flag.StringVar(&baselinePath, "baseline", "tests/e2e/manifest-lint/baseline.txt", "path to the ratchet baseline") + flag.BoolVar(&update, "update", false, "rewrite the baseline from current findings") + flag.Parse() + + root, err := filepath.Abs(repoRoot) + if err != nil { + failf("resolve repo-root: %v", err) + } + if !filepath.IsAbs(manifestPath) { + manifestPath = filepath.Join(root, manifestPath) + } + if !filepath.IsAbs(baselinePath) { + baselinePath = filepath.Join(root, baselinePath) + } + + m, err := manifestcheck.Load(manifestPath) + if err != nil { + failf("%v", err) + } + + fix, err := manifestlint.FixtureLint(root) + if err != nil { + failf("fixture lint: %v", err) + } + subj, err := manifestlint.SubjectLint(root, m) + if err != nil { + failf("subject lint: %v", err) + } + + cands := append(append([]manifestlint.Violation{}, fix.Violations...), subj.Violations...) + warnings := append(append([]string{}, fix.Warnings...), subj.Warnings...) + + if update { + if err := manifestlint.WriteBaseline(baselinePath, manifestlint.AllKeys(cands)); err != nil { + failf("write baseline: %v", err) + } + fmt.Printf("manifest-lint: baseline updated (%d entries) -> %s\n", len(cands), rel(root, baselinePath)) + return + } + + baseline, err := manifestlint.LoadBaseline(baselinePath) + if err != nil { + failf("load baseline: %v", err) + } + newViol, stale := manifestlint.Filter(cands, baseline) + + for _, w := range warnings { + fmt.Fprintf(os.Stderr, "warning: %s\n", w) + } + if len(stale) > 0 { + fmt.Fprintf(os.Stderr, "note: %d baseline entr(ies) no longer hit — run `make manifest-lint-update` to slim:\n", len(stale)) + for _, s := range stale { + fmt.Fprintf(os.Stderr, " %s\n", s) + } + } + + if len(newViol) == 0 { + fmt.Printf("manifest-lint: OK (%d baselined, %d warnings)\n", len(baseline), len(warnings)) + return + } + fmt.Fprintf(os.Stderr, "\nmanifest-lint: %d new violation(s):\n", len(newViol)) + for _, v := range newViol { + fmt.Fprintf(os.Stderr, " %s\n", v.Msg) + } + fmt.Fprintln(os.Stderr, "\nFix the call (use a harness.Ensure* fixture / declare the subject), add"+ + "\n// e2e:allow-create for an intentional create-path test, or run"+ + "\n`make manifest-lint-update` if this drift is accepted.") + os.Exit(1) +} + +func failf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(2) +} + +func rel(root, p string) string { + if r, err := filepath.Rel(root, p); err == nil { + return r + } + return p +} diff --git a/tests/e2e/manifest-lint/fixtures.go b/tests/e2e/manifest-lint/fixtures.go new file mode 100644 index 000000000..53a6cca6a --- /dev/null +++ b/tests/e2e/manifest-lint/fixtures.go @@ -0,0 +1,173 @@ +package manifestlint + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +// createAPIs are resource-creating AWS SDK methods that should be reached +// through a harness.Ensure* fixture rather than called directly in a test, so +// setup is shared across the suite. Read/describe/delete/modify calls are not +// listed — only calls that bring a new resource into existence. +var createAPIs = map[string]bool{ + "RunInstances": true, + "CreateImage": true, + "RegisterImage": true, + "CreateVpc": true, + "CreateSubnet": true, + "CreateSecurityGroup": true, + "CreateKeyPair": true, + "ImportKeyPair": true, + "CreateVolume": true, + "CreateSnapshot": true, + "CreateNatGateway": true, + "CreateInternetGateway": true, + "CreateRouteTable": true, + "CreateLoadBalancer": true, + "CreateTargetGroup": true, + "CreateListener": true, + "AllocateAddress": true, + "CreateCluster": true, + "CreateNodegroup": true, + "CreateNetworkInterface": true, +} + +// awsClientFields are the AWSClient struct fields a create call must be made +// through to count (c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...). +// Anchoring on these keeps the lint from flagging unrelated Create* helpers. +var awsClientFields = map[string]bool{ + "EC2": true, "EC2Conf": true, "ELBv2": true, "IAM": true, "STS": true, "EKS": true, +} + +// fixtureExcludeDirs are tests/e2e subdirectories that are not suites: the +// harness (where the Ensure* bodies legitimately live), shared libs, and the +// lint tooling itself. +var fixtureExcludeDirs = map[string]bool{ + "harness": true, "lib": true, "manifest-check": true, + "manifest-lint": true, "select-suites": true, "_bin": true, +} + +const allowCreateMarker = "e2e:allow-create" + +// FixtureLint scans every suite test file under tests/e2e/*/ for direct +// resource-create AWS calls. repoRoot is the spinifex repo root. +func FixtureLint(repoRoot string) (*Result, error) { + res := &Result{} + base := filepath.Join(repoRoot, "tests", "e2e") + entries, err := os.ReadDir(base) + if err != nil { + return nil, fmt.Errorf("read tests/e2e: %w", err) + } + for _, e := range entries { + if !e.IsDir() || fixtureExcludeDirs[e.Name()] { + continue + } + if err := lintSuiteDir(filepath.Join(base, e.Name()), repoRoot, res); err != nil { + return nil, err + } + } + return res, nil +} + +func lintSuiteDir(dir, repoRoot string, res *Result) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), "_test.go") { + continue + } + if err := lintTestFile(filepath.Join(dir, e.Name()), repoRoot, res); err != nil { + return err + } + } + return nil +} + +func lintTestFile(path, repoRoot string, res *Result) error { + src, err := os.ReadFile(path) + if err != nil { + return err + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, src, parser.ParseComments) + if err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + rel, _ := filepath.Rel(repoRoot, path) + rel = filepath.ToSlash(rel) + allowLines := allowMarkerLines(string(src)) + + // Collect occurrences as (api, line) so ordinals are stable and independent + // of which other APIs appear in the file. + type occ struct { + api string + line int + } + var occs []occ + ast.Inspect(f, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + api, ok := createCallAPI(call) + if !ok { + return true + } + line := fset.Position(call.Pos()).Line + if allowLines[line] || allowLines[line-1] { + return true + } + occs = append(occs, occ{api: api, line: line}) + return true + }) + + sort.Slice(occs, func(i, j int) bool { + if occs[i].api != occs[j].api { + return occs[i].api < occs[j].api + } + return occs[i].line < occs[j].line + }) + ordinal := map[string]int{} + for _, o := range occs { + ordinal[o.api]++ + key := tab("create", rel, o.api, fmt.Sprintf("#%d", ordinal[o.api])) + msg := fmt.Sprintf("%s:%d: direct %s — use a harness.Ensure* fixture or add // %s", + rel, o.line, o.api, allowCreateMarker) + res.add(key, msg) + } + return nil +} + +// createCallAPI reports whether call is `..(...)` where +// Field is an AWSClient client field and CreateAPI is a resource-create method. +func createCallAPI(call *ast.CallExpr) (string, bool) { + sel, ok := call.Fun.(*ast.SelectorExpr) // . + if !ok || !createAPIs[sel.Sel.Name] { + return "", false + } + field, ok := sel.X.(*ast.SelectorExpr) // . + if !ok || !awsClientFields[field.Sel.Name] { + return "", false + } + return sel.Sel.Name, true +} + +// allowMarkerLines returns the set of 1-based line numbers containing the +// allow-create marker, so a call on (or one line below) the marker is exempt. +func allowMarkerLines(src string) map[int]bool { + out := map[int]bool{} + for i, line := range strings.Split(src, "\n") { + if strings.Contains(line, allowCreateMarker) { + out[i+1] = true + } + } + return out +} diff --git a/tests/e2e/manifest-lint/lint.go b/tests/e2e/manifest-lint/lint.go new file mode 100644 index 000000000..1dcbd6b44 --- /dev/null +++ b/tests/e2e/manifest-lint/lint.go @@ -0,0 +1,127 @@ +// Package manifestlint implements the e2e manifest drift guards (Bead 5 of +// docs/development/improvements/e2e-targeted-suite-selection.md): +// +// - Fixture lint: every direct resource-create AWS call in a suite test +// (e.g. c.EC2.RunInstances) should go through a harness.Ensure* fixture so +// setup is shared and reusable. Pre-existing call sites are captured in a +// checked-in baseline; the lint fails only on NEW direct creates beyond the +// baseline. Intentional create-path assertions opt out with an inline +// // e2e:allow-create comment. +// +// - Subject lint: NATS subject string literals referenced in service source +// (the natsSub{...} registry and literal Subscribe/Publish/Request args) +// must be declared in docs/service-interfaces.yaml. Code subjects absent +// from the manifest are violations (baseline-ratcheted). Manifest subjects +// with no static reference are reported as warnings, not failures, because +// many subjects are produced dynamically (fmt.Sprintf) and cannot be +// matched statically. +// +// Both checks share one baseline file and one CLI (cmd/manifest-lint), wired +// into `make manifest-lint` and `make preflight`. +package manifestlint + +import ( + "bufio" + "os" + "sort" + "strings" +) + +// Violation is a single drift finding. Key is the stable baseline identity +// (no line numbers, so it survives unrelated edits); Msg is the human-readable +// line shown to the developer. +type Violation struct { + Key string + Msg string +} + +// Result aggregates a lint run before baseline filtering is applied. +type Result struct { + Violations []Violation // candidate findings (pre-baseline) + Warnings []string // advisory only, never fail the build +} + +func (r *Result) add(key, msg string) { + r.Violations = append(r.Violations, Violation{Key: key, Msg: msg}) +} + +func (r *Result) warn(msg string) { r.Warnings = append(r.Warnings, msg) } + +// LoadBaseline reads a baseline file into a set of keys. A missing file is an +// empty baseline (every violation is then "new"). +func LoadBaseline(path string) (map[string]bool, error) { + set := map[string]bool{} + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return set, nil + } + return nil, err + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + set[line] = true + } + return set, sc.Err() +} + +// WriteBaseline writes keys (sorted, deduplicated) to path with a header. +func WriteBaseline(path string, keys []string) error { + uniq := map[string]bool{} + for _, k := range keys { + uniq[k] = true + } + out := make([]string, 0, len(uniq)) + for k := range uniq { + out = append(out, k) + } + sort.Strings(out) + + var b strings.Builder + b.WriteString("# manifest-lint baseline — pre-existing drift accepted by the ratchet.\n") + b.WriteString("# Regenerate with: make manifest-lint-update. Migrate entries away over time.\n") + b.WriteString("# Lines: \"create\\t\\t\\t#\" or \"subject\\t\\t\".\n") + for _, k := range out { + b.WriteString(k) + b.WriteByte('\n') + } + return os.WriteFile(path, []byte(b.String()), 0o600) +} + +// Filter splits candidate violations against a baseline: "new" are findings not +// present in the baseline (these fail the build); "stale" are baseline keys no +// longer observed (advisory — the baseline can be slimmed). +func Filter(cands []Violation, baseline map[string]bool) (added []Violation, stale []string) { + observed := map[string]bool{} + for _, v := range cands { + observed[v.Key] = true + if !baseline[v.Key] { + added = append(added, v) + } + } + for k := range baseline { + if !observed[k] { + stale = append(stale, k) + } + } + sort.Slice(added, func(i, j int) bool { return added[i].Msg < added[j].Msg }) + sort.Strings(stale) + return added, stale +} + +// AllKeys returns the stable keys of every candidate violation, for seeding or +// regenerating the baseline. +func AllKeys(cands []Violation) []string { + keys := make([]string, 0, len(cands)) + for _, v := range cands { + keys = append(keys, v.Key) + } + return keys +} + +func tab(parts ...string) string { return strings.Join(parts, "\t") } diff --git a/tests/e2e/manifest-lint/manifestlint_test.go b/tests/e2e/manifest-lint/manifestlint_test.go new file mode 100644 index 000000000..8aa4557cb --- /dev/null +++ b/tests/e2e/manifest-lint/manifestlint_test.go @@ -0,0 +1,251 @@ +package manifestlint + +import ( + "os" + "path/filepath" + "strings" + "testing" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" +) + +// writeFile creates path (with parents) under root and returns nothing. +func writeFile(t *testing.T, root, rel, body string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestFixtureLint_FlagsDirectCreate(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "tests/e2e/single/x_test.go", `package single +func TestA(t *T) { + c.EC2.RunInstances(nil) + c.ELBv2.CreateLoadBalancer(nil) + harness.EnsureInstance(t, fx, spec) // not a direct create +} +`) + res, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 2 { + t.Fatalf("want 2 violations, got %d: %+v", len(res.Violations), res.Violations) + } + joined := "" + for _, v := range res.Violations { + joined += v.Msg + "\n" + } + for _, want := range []string{"RunInstances", "CreateLoadBalancer"} { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in:\n%s", want, joined) + } + } +} + +func TestFixtureLint_AllowMarkerExempts(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "tests/e2e/single/x_test.go", `package single +func TestA(t *T) { + c.EC2.RunInstances(nil) // e2e:allow-create + // e2e:allow-create + c.EC2.CreateVpc(nil) +} +`) + res, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 0 { + t.Fatalf("allow-create should exempt both, got %d: %+v", len(res.Violations), res.Violations) + } +} + +func TestFixtureLint_IgnoresNonClientCreate(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "tests/e2e/single/x_test.go", `package single +func TestA(t *T) { + os.CreateVpc(nil) // wrong receiver field + createVpc() // local helper, not a client call + c.Other.RunInstances(0) // not an AWS client field +} +`) + res, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 0 { + t.Fatalf("want 0, got %d: %+v", len(res.Violations), res.Violations) + } +} + +func TestFixtureLint_ExcludesHarness(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "tests/e2e/harness/fixtures_test.go", `package harness +func TestEnsure(t *T) { c.EC2.RunInstances(nil) } +`) + res, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 0 { + t.Fatalf("harness dir must be excluded, got %d", len(res.Violations)) + } +} + +func TestFixtureLint_OrdinalKeysStable(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "tests/e2e/single/x_test.go", `package single +func TestA(t *T) { + c.EC2.RunInstances(1) + c.EC2.RunInstances(2) +} +`) + res, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 2 { + t.Fatalf("want 2, got %d", len(res.Violations)) + } + keys := map[string]bool{} + for _, v := range res.Violations { + keys[v.Key] = true + } + if !keys["create\ttests/e2e/single/x_test.go\tRunInstances\t#1"] || + !keys["create\ttests/e2e/single/x_test.go\tRunInstances\t#2"] { + t.Fatalf("ordinal keys wrong: %v", keys) + } +} + +func testManifest() *manifestcheck.Manifest { + return &manifestcheck.Manifest{ + Version: 1, + Services: map[string]manifestcheck.Service{ + "daemon": { + Path: "src/daemon", + Subscribes: []string{"ec2.RunInstances", "vpc.*"}, + Publishes: []string{"ebs.mount"}, + }, + }, + } +} + +func TestSubjectLint_FlagsUndeclared(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "src/daemon/d.go", `package daemon +type natsSub struct{ topic string; h func(); q string } +func reg() { + subs := []natsSub{ + {"ec2.RunInstances", nil, ""}, // declared + {"ec2.SecretSubject", nil, ""}, // NOT declared + } + nc.Publish("vpc.create", nil) // covered by vpc.* wildcard + nc.Request("ec2.Mystery", nil) // undeclared + _ = subs +} +`) + res, err := SubjectLint(root, testManifest()) + if err != nil { + t.Fatal(err) + } + var got []string + for _, v := range res.Violations { + got = append(got, v.Msg) + } + joined := strings.Join(got, "\n") + if len(res.Violations) != 2 { + t.Fatalf("want 2 undeclared, got %d:\n%s", len(res.Violations), joined) + } + for _, want := range []string{"ec2.SecretSubject", "ec2.Mystery"} { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in:\n%s", want, joined) + } + } + if strings.Contains(joined, "vpc.create") { + t.Errorf("vpc.create should be covered by vpc.* wildcard") + } +} + +func TestSubjectLint_SkipsDynamicAndOutOfScope(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "src/daemon/d.go", `package daemon +func reg() { + nc.Subscribe(fmt.Sprintf("ec2.cmd.%s", id), nil) // dynamic, skipped + nc.Subscribe("test.internal.reply", nil) // out-of-scope prefix + nc.Publish("ec2.start: log line not a subject", nil) +} +`) + res, err := SubjectLint(root, testManifest()) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 0 { + t.Fatalf("dynamic/out-of-scope must not violate, got %d: %+v", len(res.Violations), res.Violations) + } +} + +func TestFilter_NewVsStale(t *testing.T) { + cands := []Violation{{Key: "a", Msg: "A"}, {Key: "b", Msg: "B"}} + baseline := map[string]bool{"a": true, "c": true} + newV, stale := Filter(cands, baseline) + if len(newV) != 1 || newV[0].Key != "b" { + t.Fatalf("want new=[b], got %+v", newV) + } + if len(stale) != 1 || stale[0] != "c" { + t.Fatalf("want stale=[c], got %+v", stale) + } +} + +func TestBaselineRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "baseline.txt") + if err := WriteBaseline(path, []string{"b", "a", "a"}); err != nil { + t.Fatal(err) + } + set, err := LoadBaseline(path) + if err != nil { + t.Fatal(err) + } + if len(set) != 2 || !set["a"] || !set["b"] { + t.Fatalf("round-trip wrong: %v", set) + } +} + +// TestRealRepo_NoNewViolations guards the checked-in baseline: the committed +// baseline.txt must keep the live repo green so preflight stays deterministic. +func TestRealRepo_NoNewViolations(t *testing.T) { + root, err := filepath.Abs("../../..") + if err != nil { + t.Fatal(err) + } + m, err := manifestcheck.Load(filepath.Join(root, "docs/service-interfaces.yaml")) + if err != nil { + t.Skipf("manifest unavailable: %v", err) + } + fix, err := FixtureLint(root) + if err != nil { + t.Fatal(err) + } + subj, err := SubjectLint(root, m) + if err != nil { + t.Fatal(err) + } + cands := append(append([]Violation{}, fix.Violations...), subj.Violations...) + base, err := LoadBaseline(filepath.Join(root, "tests/e2e/manifest-lint/baseline.txt")) + if err != nil { + t.Fatal(err) + } + newV, _ := Filter(cands, base) + if len(newV) != 0 { + for _, v := range newV { + t.Errorf("new violation not in baseline: %s", v.Msg) + } + t.Fatalf("%d new violation(s); run `make manifest-lint-update` if intended", len(newV)) + } +} diff --git a/tests/e2e/manifest-lint/subjects.go b/tests/e2e/manifest-lint/subjects.go new file mode 100644 index 000000000..d6479e950 --- /dev/null +++ b/tests/e2e/manifest-lint/subjects.go @@ -0,0 +1,255 @@ +package manifestlint + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" +) + +// natsCallMethods are NATS connection methods whose first string-literal +// argument is a subject. Dynamic args (fmt.Sprintf, vars, concatenation) are +// not statically resolvable and are skipped. +var natsCallMethods = map[string]bool{ + "Subscribe": true, "QueueSubscribe": true, "SubscribeSync": true, + "ChanSubscribe": true, "Publish": true, "PublishRequest": true, "Request": true, +} + +// subjectLitRE matches a clean, fully-static NATS subject literal. Literals +// containing format verbs/spaces (log lines, fmt.Sprintf templates) fail this +// and are treated as dynamic. +var subjectLitRE = regexp.MustCompile(`^[a-z][a-z0-9]*(\.[A-Za-z0-9_-]+)+$`) + +type codeSubject struct { + subject string + file string + line int +} + +// SubjectLint cross-references NATS subject literals in service source against +// the manifest. repoRoot is the spinifex repo root; m is the loaded manifest. +func SubjectLint(repoRoot string, m *manifestcheck.Manifest) (*Result, error) { + res := &Result{} + + exact, wildcards, managed := manifestSubjects(m) + + dirs := serviceDirs(m) + var found []codeSubject + seenExact := map[string]bool{} + for _, d := range dirs { + subs, err := scanSubjectsInDir(filepath.Join(repoRoot, filepath.FromSlash(d)), repoRoot) + if err != nil { + return nil, err + } + for _, cs := range subs { + seenExact[cs.subject] = true + found = append(found, cs) + } + } + + // Code subjects not declared in the manifest (ratcheted failures). + for _, cs := range found { + if !managed[firstSegment(cs.subject)] { + continue // out of scope (test.*, system.* internal topics, ...) + } + if subjectCovered(cs.subject, exact, wildcards) { + continue + } + key := tab("subject", cs.file, cs.subject) + msg := fmt.Sprintf("%s:%d: NATS subject %q not declared in service-interfaces.yaml", + cs.file, cs.line, cs.subject) + res.add(key, msg) + } + + // Orphan manifest subjects: declared, managed-prefix, non-wildcard, with no + // static reference. Advisory only — the real reference may be dynamic. + var orphans []string + for s := range exact { + if !managed[firstSegment(s)] || strings.HasSuffix(s, ".*") { + continue + } + if !seenExact[s] { + orphans = append(orphans, s) + } + } + sort.Strings(orphans) + for _, s := range orphans { + res.warn(fmt.Sprintf("manifest subject %q has no static reference in source (dynamic, or stale entry)", s)) + } + + return res, nil +} + +// manifestSubjects returns the exact (non-wildcard) subject set, the wildcard +// prefixes (".*" stripped), and the managed first-segment set. +func manifestSubjects(m *manifestcheck.Manifest) (exact map[string]bool, wildcards []string, managed map[string]bool) { + exact = map[string]bool{} + managed = map[string]bool{} + wset := map[string]bool{} + for _, svc := range m.Services { + for _, lists := range [][]string{svc.Subscribes, svc.Publishes, svc.DependsOn} { + for _, s := range lists { + managed[firstSegment(s)] = true + if strings.HasSuffix(s, ".*") { + wset[strings.TrimSuffix(s, "*")] = true // "ec2.*" -> "ec2." + } else { + exact[s] = true + } + } + } + } + for w := range wset { + wildcards = append(wildcards, w) + } + sort.Strings(wildcards) + return exact, wildcards, managed +} + +func subjectCovered(s string, exact map[string]bool, wildcards []string) bool { + if exact[s] { + return true + } + for _, w := range wildcards { + if strings.HasPrefix(s, w) { + return true + } + } + return false +} + +// serviceDirs returns the deduplicated, slash-form source roots of every +// service (path + additional_paths). +func serviceDirs(m *manifestcheck.Manifest) []string { + set := map[string]bool{} + for _, svc := range m.Services { + if svc.Path != "" { + set[svc.Path] = true + } + for _, p := range svc.AdditionalPaths { + set[p] = true + } + } + out := make([]string, 0, len(set)) + for d := range set { + out = append(out, d) + } + sort.Strings(out) + return out +} + +// scanSubjectsInDir walks dir recursively, parsing every non-test .go file and +// extracting subject literals from natsSub{...} registry entries and from +// literal first-args to NATS connection methods. +func scanSubjectsInDir(dir, repoRoot string) ([]codeSubject, error) { + var out []codeSubject + fset := token.NewFileSet() + err := filepathWalkGo(dir, func(path string) error { + f, perr := parser.ParseFile(fset, path, nil, 0) + if perr != nil { + return fmt.Errorf("parse %s: %w", path, perr) + } + rel, _ := filepath.Rel(repoRoot, path) + rel = filepath.ToSlash(rel) + ast.Inspect(f, func(n ast.Node) bool { + switch node := n.(type) { + case *ast.CompositeLit: + // Standalone `natsSub{"subject", ...}` (append form). + if isIdent(node.Type, "natsSub") { + addFirstElt(&out, node, rel, fset) + return true + } + // `[]natsSub{ {"subject", ...}, ... }` — inner elements have an + // elided type, so match on the array element type instead. + if at, ok := node.Type.(*ast.ArrayType); ok && isIdent(at.Elt, "natsSub") { + for _, el := range node.Elts { + if cl, ok := el.(*ast.CompositeLit); ok { + addFirstElt(&out, cl, rel, fset) + } + } + } + case *ast.CallExpr: + sel, ok := node.Fun.(*ast.SelectorExpr) + if !ok || !natsCallMethods[sel.Sel.Name] || len(node.Args) == 0 { + return true + } + if s, ok := stringLit(node.Args[0]); ok { + addSubject(&out, s, rel, fset.Position(node.Pos()).Line) + } + } + return true + }) + return nil + }) + return out, err +} + +func isIdent(e ast.Expr, name string) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == name +} + +func addFirstElt(out *[]codeSubject, cl *ast.CompositeLit, file string, fset *token.FileSet) { + if len(cl.Elts) == 0 { + return + } + if s, ok := stringLit(cl.Elts[0]); ok { + addSubject(out, s, file, fset.Position(cl.Pos()).Line) + } +} + +func addSubject(out *[]codeSubject, s, file string, line int) { + if subjectLitRE.MatchString(s) { + *out = append(*out, codeSubject{subject: s, file: file, line: line}) + } +} + +func stringLit(e ast.Expr) (string, bool) { + lit, ok := e.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return "", false + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return "", false + } + return s, true +} + +func firstSegment(s string) string { + if seg, _, ok := strings.Cut(s, "."); ok { + return seg + } + return s +} + +// filepathWalkGo invokes fn for every non-test .go file under root, skipping +// testdata and a missing root (a manifest path may legitimately not exist on a +// given checkout). +func filepathWalkGo(root string, fn func(path string) error) error { + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() { + if d.Name() == "testdata" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + return fn(path) + }) +} diff --git a/tests/e2e/select-suites/ast.go b/tests/e2e/select-suites/ast.go new file mode 100644 index 000000000..61551816c --- /dev/null +++ b/tests/e2e/select-suites/ast.go @@ -0,0 +1,196 @@ +package selectsuites + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +// SuiteCoverage records, per suite, which services each top-level Test* +// function exercises. Coverage is derived by walking the suite's package call +// graph from each Test to the harness.Ensure* helpers it reaches, then mapping +// those fixtures to services via the manifest's fixtures block. +type SuiteCoverage struct { + Path string + Tests map[string][]string // test func name → sorted service names +} + +// ScanSuites builds per-test coverage for every suite in the manifest. repoRoot +// is the spinifex repo root (suite paths are resolved beneath it). Suites whose +// directory is missing or has no test files are skipped silently — the selector +// then runs them in full. +func ScanSuites(repoRoot string, m *Manifest) (map[string]SuiteCoverage, error) { + fixSvc := map[string][]string{} + for name, f := range m.Fixtures { + fixSvc[name] = f.Services + } + out := map[string]SuiteCoverage{} + for name, s := range m.Suites { + dir := filepath.Join(repoRoot, filepath.FromSlash(s.Path)) + cov, err := scanSuiteDir(dir, fixSvc) + if err != nil { + return nil, err + } + if cov == nil { + continue + } + out[name] = SuiteCoverage{Path: s.Path, Tests: cov} + } + return out, nil +} + +// scanSuiteDir parses every *_test.go in dir, builds the local call graph, and +// returns testName → services. Returns nil (not an error) when the dir has no +// Go test files. +func scanSuiteDir(dir string, fixSvc map[string][]string) (map[string][]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + fset := token.NewFileSet() + // directEnsure[func] = set of Ensure* fixture names called directly in func. + directEnsure := map[string]map[string]bool{} + // localCalls[func] = set of package-local func names called in func. + localCalls := map[string]map[string]bool{} + tests := map[string]bool{} + found := false + + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), "_test.go") { + continue + } + found = true + f, perr := parser.ParseFile(fset, filepath.Join(dir, e.Name()), nil, 0) + if perr != nil { + return nil, perr + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil || fn.Recv != nil { + continue + } + name := fn.Name.Name + if isTestFunc(fn) { + tests[name] = true + } + de, lc := scanFuncBody(fn.Body) + directEnsure[name] = de + localCalls[name] = lc + } + } + if !found { + return nil, nil + } + + out := map[string][]string{} + for t := range tests { + ensure := reachableEnsure(t, directEnsure, localCalls) + svc := map[string]bool{} + for fx := range ensure { + for _, s := range fixSvc[fx] { + svc[s] = true + } + } + out[t] = sortedKeys(svc) + } + return out, nil +} + +// scanFuncBody collects the harness.Ensure* fixture calls and package-local +// function calls made directly inside body. +func scanFuncBody(body *ast.BlockStmt) (ensure, local map[string]bool) { + ensure = map[string]bool{} + local = map[string]bool{} + ast.Inspect(body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch fn := call.Fun.(type) { + case *ast.SelectorExpr: + if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "harness" && + strings.HasPrefix(fn.Sel.Name, "Ensure") { + ensure[fn.Sel.Name] = true + } + case *ast.Ident: + local[fn.Name] = true + } + return true + }) + return ensure, local +} + +// reachableEnsure returns every Ensure* fixture reachable from start by +// following package-local calls (BFS over the local call graph). +func reachableEnsure(start string, direct, local map[string]map[string]bool) map[string]bool { + seen := map[string]bool{} + ensure := map[string]bool{} + queue := []string{start} + for len(queue) > 0 { + fn := queue[0] + queue = queue[1:] + if seen[fn] { + continue + } + seen[fn] = true + for e := range direct[fn] { + ensure[e] = true + } + for callee := range local[fn] { + if !seen[callee] { + queue = append(queue, callee) + } + } + } + return ensure +} + +func isTestFunc(fn *ast.FuncDecl) bool { + if !strings.HasPrefix(fn.Name.Name, "Test") { + return false + } + // Exclude TestMain — it is the process entry, not a coverage unit. + return fn.Name.Name != "TestMain" +} + +// runPatterns builds a suite→regex map keeping only tests that touch a covered +// service (tests with no analyzable fixture calls are always kept, so narrowing +// never drops an unanalyzable test). A pattern is emitted only when it is a +// strict, non-empty subset of the suite's tests. +func runPatterns(perTest map[string]SuiteCoverage, suites []string, covered map[string]bool) map[string]string { + out := map[string]string{} + for _, suite := range suites { + cov, ok := perTest[suite] + if !ok || len(cov.Tests) == 0 { + continue + } + var keep []string + for t, svcs := range cov.Tests { + if len(svcs) == 0 || intersects(svcs, covered) { + keep = append(keep, t) + } + } + if len(keep) == 0 || len(keep) >= len(cov.Tests) { + continue // empty or no narrowing — run the whole suite + } + sort.Strings(keep) + out[suite] = "^(" + strings.Join(keep, "|") + ")$" + } + return out +} + +func intersects(svcs []string, set map[string]bool) bool { + for _, s := range svcs { + if set[s] { + return true + } + } + return false +} diff --git a/tests/e2e/select-suites/cmd/select-suites/main.go b/tests/e2e/select-suites/cmd/select-suites/main.go new file mode 100644 index 000000000..507836e90 --- /dev/null +++ b/tests/e2e/select-suites/cmd/select-suites/main.go @@ -0,0 +1,97 @@ +// Command select-suites prints the e2e suites a diff must run. +// +// Usage: +// +// select-suites --manifest docs/service-interfaces.yaml \ +// --base-sha --head-sha [--repo-root .] +// +// Output (stdout, two lines, ready to eval into a workflow): +// +// SUITES=e2e-cert e2e-lb e2e-single +// RUN_PATTERN_e2e-single=^(TestVPCCRUD|TestSubnetCRUD)$ +// +// A trailing comment with the selection reason goes to stderr. Exit non-zero +// only on hard errors (bad manifest, git failure); an empty diff or no match +// resolves to the configured default suites. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "sort" + "strings" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" + selectsuites "github.com/mulgadc/spinifex/tests/e2e/select-suites" +) + +func main() { + manifestPath := flag.String("manifest", "docs/service-interfaces.yaml", "path to service-interfaces.yaml") + baseSHA := flag.String("base-sha", "", "diff base SHA (PR base)") + headSHA := flag.String("head-sha", "", "diff head SHA") + repoRoot := flag.String("repo-root", ".", "repo root for git diff + AST scan") + noAST := flag.Bool("no-ast", false, "skip per-test AST narrowing") + flag.Parse() + + if *baseSHA == "" || *headSHA == "" { + fmt.Fprintln(os.Stderr, "select-suites: --base-sha and --head-sha are required") + os.Exit(2) + } + + m, err := manifestcheck.Load(*manifestPath) + if err != nil { + fmt.Fprintf(os.Stderr, "select-suites: load manifest: %v\n", err) + os.Exit(1) + } + + changed, err := gitDiff(*repoRoot, *baseSHA, *headSHA) + if err != nil { + fmt.Fprintf(os.Stderr, "select-suites: git diff: %v\n", err) + os.Exit(1) + } + + var perTest map[string]selectsuites.SuiteCoverage + if !*noAST { + perTest, err = selectsuites.ScanSuites(*repoRoot, m) + if err != nil { + // AST failure is non-fatal: fall back to whole-suite selection. + fmt.Fprintf(os.Stderr, "select-suites: AST scan failed, running suites in full: %v\n", err) + perTest = nil + } + } + + res := selectsuites.Select(m, selectsuites.DefaultConfig(), changed, perTest) + + fmt.Fprintf(os.Stderr, "# select-suites: %s\n", res.Reason) + fmt.Printf("SUITES=%s\n", strings.Join(res.Suites, " ")) + for _, suite := range sortedMapKeys(res.RunPatterns) { + fmt.Printf("RUN_PATTERN_%s=%s\n", suite, res.RunPatterns[suite]) + } +} + +// gitDiff returns repo-root-relative changed paths between base and head. +func gitDiff(root, base, head string) ([]string, error) { + cmd := exec.Command("git", "-C", root, "diff", "--name-only", base, head) + out, err := cmd.Output() + if err != nil { + return nil, err + } + var files []string + for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") { + if line = strings.TrimSpace(line); line != "" { + files = append(files, line) + } + } + return files, nil +} + +func sortedMapKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/tests/e2e/select-suites/selector.go b/tests/e2e/select-suites/selector.go new file mode 100644 index 000000000..e940c0f5c --- /dev/null +++ b/tests/e2e/select-suites/selector.go @@ -0,0 +1,319 @@ +// Package selectsuites derives the set of e2e suites a diff must exercise. +// +// Inputs: the service-interface manifest (docs/service-interfaces.yaml) and a +// list of repo-root-relative changed paths. Output: the suites to run, plus an +// optional per-suite -test.run regex (RUN_PATTERN) when AST analysis can prove +// a strict subtest subset is sufficient. +// +// Selection model (see docs/development/improvements/ +// e2e-targeted-suite-selection.md): +// +// changed_services = { s | s.path (or additional_paths) prefixes a changed file } +// covered_services = changed_services plus the transitive closure of +// *consumers*: x is covered if x.depends_on overlaps the +// publishes set of an already-covered service. +// must_run_suites = { suite | suite.covers ∩ covered_services ≠ ∅ } +// +// Several special cases short-circuit to "run all" (infra/harness/workflow +// changes, ≥N services touched, submodule bumps) or to a single suite (a diff +// confined to one suite's own test dir). An empty result falls back to the +// default suites rather than skipping e2e entirely. +package selectsuites + +import ( + "path" + "sort" + "strings" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" +) + +// Manifest is re-exported from manifestcheck so callers import one package. +type Manifest = manifestcheck.Manifest + +// Result is the selector's output. AllSuites is true when a special case +// forced the full matrix; Suites then lists every manifest suite. RunPatterns +// maps a suite name to a -test.run regex when a strict subtest subset suffices; +// suites absent from the map run in full. +type Result struct { + Suites []string + RunPatterns map[string]string + AllSuites bool + Reason string +} + +// Config carries the hard-coded special-case rules. Defaults mirror the +// commented rules at the foot of service-interfaces.yaml; exposed as a struct +// so tests can pin them without depending on repo layout. +type Config struct { + // InfraGlobs force "run all" when any changed file matches. Globs use + // path.Match semantics per segment, with a trailing "/**" meaning + // "this dir and everything under it". + InfraGlobs []string + // SubmodulePaths force "run all" when a changed file is at or under one + // (a submodule pointer bump can hide a contract change). + SubmodulePaths []string + // ManyServicesThreshold: ≥ this many distinct changed services → run all. + ManyServicesThreshold int + // DefaultSuites run when selection is otherwise empty. + DefaultSuites []string +} + +// DefaultConfig returns the production special-case rules. +func DefaultConfig() Config { + return Config{ + InfraGlobs: []string{ + "spinifex/services/nats/**", + "tests/e2e/harness/**", + "tests/e2e/lib/**", + "tests/e2e/Makefile", + "Makefile", + ".github/workflows/e2e.yml", + ".github/scripts/**", + }, + SubmodulePaths: []string{ + "predastore", + "viperblock", + }, + ManyServicesThreshold: 3, + DefaultSuites: []string{"e2e-cert", "e2e-lb"}, + } +} + +// Select computes the suites to run for changedFiles. changedFiles are +// repo-root-relative (as `git diff --name-only` emits inside the spinifex +// repo). runPatterns is an optional suite→(testName set) map from AST analysis; +// pass nil to skip subtest narrowing. +func Select(m *Manifest, cfg Config, changedFiles []string, perTest map[string]SuiteCoverage) Result { + all := func(reason string) Result { + names := suiteNames(m) + return Result{Suites: names, AllSuites: true, Reason: reason} + } + + if len(changedFiles) == 0 { + return Result{Suites: append([]string(nil), cfg.DefaultSuites...), Reason: "empty diff → default suites"} + } + + // 1. Force-all special cases take precedence over everything. + for _, f := range changedFiles { + if g := matchInfra(f, cfg.InfraGlobs); g != "" { + return all("infra path changed: " + f + " (" + g + ")") + } + if s := matchSubmodule(f, cfg.SubmodulePaths); s != "" { + return all("submodule pointer changed: " + s) + } + } + + // 2. A diff confined to a single suite's own test dir runs only that suite. + if name, ok := loneSuiteDir(m, changedFiles); ok { + return Result{Suites: []string{name}, Reason: "diff confined to suite dir " + name} + } + + // 3. Map changed files to changed services, then take the consumer closure. + changedSvc := changedServices(m, changedFiles) + if len(changedSvc) >= cfg.ManyServicesThreshold { + return all("≥3 services touched") + } + covered := consumerClosure(m, changedSvc) + + // 4. Suites whose covers intersect the covered service set. + suites := suitesForServices(m, covered) + if len(suites) == 0 { + return Result{Suites: append([]string(nil), cfg.DefaultSuites...), Reason: "no suite matched → default suites"} + } + + res := Result{Suites: suites, Reason: "service-mapped: " + strings.Join(sortedKeys(covered), ",")} + + // 5. Best-effort subtest narrowing: for each selected suite with per-test + // coverage data, keep only the tests touching a covered service. Emit a + // RUN_PATTERN only when that is a strict, non-empty subset. + if len(perTest) > 0 { + res.RunPatterns = runPatterns(perTest, suites, covered) + } + return res +} + +// changedServices returns the set of service names whose path or +// additional_paths prefix at least one changed file. +func changedServices(m *Manifest, changed []string) map[string]bool { + out := map[string]bool{} + for name, svc := range m.Services { + for _, p := range append([]string{svc.Path}, svc.AdditionalPaths...) { + if p == "" { + continue + } + for _, f := range changed { + if underDir(f, p) { + out[name] = true + } + } + } + } + return out +} + +// consumerClosure grows seed with every service that depends_on a subject an +// already-covered service handles — whether that service *publishes* the +// subject (produces data the dependant consumes) or *subscribes* to it (handles +// requests the dependant sends). Both mean "if the covered service changes, the +// dependant may break". Iterates to a fixpoint. +func consumerClosure(m *Manifest, seed map[string]bool) map[string]bool { + covered := map[string]bool{} + for k := range seed { + covered[k] = true + } + for { + added := false + for name, svc := range m.Services { + if covered[name] { + continue + } + for cname := range covered { + c := m.Services[cname] + if subjectsOverlap(svc.DependsOn, c.Publishes) || + subjectsOverlap(svc.DependsOn, c.Subscribes) { + covered[name] = true + added = true + break + } + } + } + if !added { + return covered + } + } +} + +// suitesForServices returns the sorted suite names whose covers list intersects +// the covered service set. A suite with covers [all] always matches. +func suitesForServices(m *Manifest, covered map[string]bool) []string { + var out []string + for name, s := range m.Suites { + match := false + for _, c := range s.Covers { + if c == "all" || covered[c] { + match = true + break + } + } + if match { + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +// loneSuiteDir reports the suite when every changed file lives under exactly +// one suite's path and that path is not a shared dir (harness/lib are infra, +// handled earlier). +func loneSuiteDir(m *Manifest, changed []string) (string, bool) { + hit := "" + for _, f := range changed { + owner := "" + for name, s := range m.Suites { + if s.Path != "" && underDir(f, s.Path) { + owner = name + break + } + } + if owner == "" { + return "", false // a non-suite file is in the diff + } + if hit == "" { + hit = owner + } else if hit != owner { + return "", false // spans ≥2 suites + } + } + return hit, hit != "" +} + +func matchInfra(f string, globs []string) string { + for _, g := range globs { + if globMatch(g, f) { + return g + } + } + return "" +} + +func matchSubmodule(f string, subs []string) string { + for _, s := range subs { + if f == s || underDir(f, s) { + return s + } + } + return "" +} + +// underDir reports whether file f is dir itself or sits beneath it. Both are +// slash-separated repo-relative paths. +func underDir(f, dir string) bool { + dir = strings.TrimSuffix(dir, "/") + return f == dir || strings.HasPrefix(f, dir+"/") +} + +// globMatch supports a trailing "/**" (this dir and all descendants) plus +// per-segment path.Match for the rest. Without "/**" it is an exact or +// path.Match comparison. +func globMatch(pattern, f string) bool { + if before, ok := strings.CutSuffix(pattern, "/**"); ok { + return underDir(f, before) + } + if ok, _ := path.Match(pattern, f); ok { + return true + } + return pattern == f +} + +// subjectsOverlap reports whether any subject in a matches any in b under +// trailing-wildcard semantics (either side may end in ".*"). +func subjectsOverlap(a, b []string) bool { + for _, x := range a { + for _, y := range b { + if subjectMatch(x, y) { + return true + } + } + } + return false +} + +// subjectMatch compares two subjects, each optionally ending ".*". A wildcard +// matches anything sharing its prefix; two wildcards match if one prefix +// contains the other. +func subjectMatch(x, y string) bool { + xw := strings.HasSuffix(x, ".*") + yw := strings.HasSuffix(y, ".*") + xp := strings.TrimSuffix(x, ".*") + yp := strings.TrimSuffix(y, ".*") + switch { + case xw && yw: + return xp == yp || strings.HasPrefix(xp, yp+".") || strings.HasPrefix(yp, xp+".") + case xw: + return y == xp || strings.HasPrefix(y, xp+".") + case yw: + return x == yp || strings.HasPrefix(x, yp+".") + default: + return x == y + } +} + +func suiteNames(m *Manifest) []string { + out := make([]string, 0, len(m.Suites)) + for k := range m.Suites { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedKeys(s map[string]bool) []string { + out := make([]string, 0, len(s)) + for k := range s { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/tests/e2e/select-suites/selector_test.go b/tests/e2e/select-suites/selector_test.go new file mode 100644 index 000000000..98ea37a7e --- /dev/null +++ b/tests/e2e/select-suites/selector_test.go @@ -0,0 +1,305 @@ +package selectsuites + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "testing" + + manifestcheck "github.com/mulgadc/spinifex/tests/e2e/manifest-check" +) + +// testManifest mirrors the essentials of docs/service-interfaces.yaml: the five +// daemon services plus cert/lb/single/ddil/reboot suites and the fixtures +// block. Kept inline so the unit tests do not depend on the real manifest +// drifting. +func testManifest() *Manifest { + return &manifestcheck.Manifest{ + Version: 1, + Services: map[string]manifestcheck.Service{ + "awsgw": { + Path: "spinifex/services/awsgw", + Publishes: []string{"ec2.*", "elbv2.*", "acm.*", "iam.*"}, + DependsOn: []string{"ec2.*", "elbv2.*", "acm.*", "iam.*"}, + Subscribes: []string{}, + }, + "spinifex_daemon": { + Path: "spinifex/services/spinifex", + AdditionalPaths: []string{"spinifex/daemon", "spinifex/handlers/ec2"}, + Subscribes: []string{"ec2.RunInstances", "elbv2.CreateLoadBalancer"}, + Publishes: []string{"vpc.create", "ebs.mount", "s3.*", "iam.account.created"}, + DependsOn: []string{"vpc.*", "ebs.*", "s3.*"}, + }, + "vpcd": { + Path: "spinifex/vpcd", + Subscribes: []string{"vpc.create", "vpc.delete"}, + Publishes: []string{"vpc.port-status"}, + DependsOn: []string{}, + }, + "viperblockd": { + Path: "spinifex/services/viperblockd", + Subscribes: []string{"ebs.mount", "ebs.unmount"}, + Publishes: []string{"ebs.mount.response"}, + DependsOn: []string{}, + }, + "predastore": { + Path: "spinifex/services/predastore", + Subscribes: []string{"s3.*"}, + Publishes: []string{}, + DependsOn: []string{}, + }, + }, + Fixtures: map[string]manifestcheck.Fixture{ + "EnsureDefaultVPC": {Services: []string{"vpcd", "spinifex_daemon"}}, + "EnsureInstance": {Services: []string{"spinifex_daemon", "viperblockd", "vpcd"}}, + "EnsureVolume": {Services: []string{"spinifex_daemon", "viperblockd"}}, + }, + Suites: map[string]manifestcheck.Suite{ + "e2e-cert": {Path: "tests/e2e/cert", Covers: []string{"awsgw", "spinifex_daemon", "vpcd", "predastore"}}, + "e2e-lb": {Path: "tests/e2e/lb", Covers: []string{"awsgw", "spinifex_daemon", "vpcd", "viperblockd", "predastore"}}, + "e2e-single": {Path: "tests/e2e/single", Covers: []string{"awsgw", "spinifex_daemon", "vpcd", "viperblockd", "predastore"}}, + "e2e-ddil": {Path: "tests/e2e/ddil", Covers: []string{"awsgw", "spinifex_daemon", "vpcd", "viperblockd", "predastore"}}, + "e2e-reboot": {Path: "tests/e2e/reboot", Covers: []string{"spinifex_daemon", "viperblockd"}}, + }, + } +} + +func selectSuites(t *testing.T, changed []string) Result { + t.Helper() + return Select(testManifest(), DefaultConfig(), changed, nil) +} + +func wantSuites(t *testing.T, got Result, want ...string) { + t.Helper() + g := append([]string(nil), got.Suites...) + sort.Strings(g) + w := append([]string(nil), want...) + sort.Strings(w) + if !reflect.DeepEqual(g, w) { + t.Fatalf("suites = %v, want %v (reason: %s)", g, w, got.Reason) + } +} + +func TestSelect_EmptyDiff_Default(t *testing.T) { + got := selectSuites(t, nil) + wantSuites(t, got, "e2e-cert", "e2e-lb") +} + +func TestSelect_VPCDChange_ClosureToAWSGW(t *testing.T) { + // vpcd → spinifex_daemon (depends_on vpc.*) → awsgw (depends_on iam.* of + // daemon's iam.account.created). reboot does not cover awsgw/vpcd but does + // cover spinifex_daemon, so it is included; every suite covers the daemon. + got := selectSuites(t, []string{"spinifex/vpcd/topology.go"}) + wantSuites(t, got, "e2e-cert", "e2e-lb", "e2e-single", "e2e-ddil", "e2e-reboot") +} + +func TestSelect_PredastoreChange_DiscriminatesReboot(t *testing.T) { + // predastore publishes nothing, but spinifex_daemon depends_on s3.* which + // predastore subscribes → daemon + awsgw covered. reboot covers neither + // predastore nor awsgw, only daemon/viperblockd → still matches via daemon. + // Use a service nobody depends on to truly discriminate: see vpcd-isolated. + got := selectSuites(t, []string{"spinifex/services/predastore/adapter.go"}) + wantSuites(t, got, "e2e-cert", "e2e-lb", "e2e-single", "e2e-ddil", "e2e-reboot") +} + +func TestSelect_AdditionalPaths(t *testing.T) { + // handlers/ec2 is an additional_path of spinifex_daemon. + got := selectSuites(t, []string{"spinifex/handlers/ec2/runinstances.go"}) + if len(got.Suites) == 0 || got.AllSuites { + t.Fatalf("expected service-mapped selection, got %+v", got) + } +} + +func TestSelect_HarnessChange_ForcesAll(t *testing.T) { + got := selectSuites(t, []string{"tests/e2e/harness/fixtures.go"}) + if !got.AllSuites { + t.Fatalf("harness change must force all; got %+v", got) + } + wantSuites(t, got, "e2e-cert", "e2e-lb", "e2e-single", "e2e-ddil", "e2e-reboot") +} + +func TestSelect_WorkflowChange_ForcesAll(t *testing.T) { + got := selectSuites(t, []string{".github/workflows/e2e.yml"}) + if !got.AllSuites { + t.Fatalf("workflow change must force all; got %+v", got) + } +} + +func TestSelect_NatsInfra_ForcesAll(t *testing.T) { + got := selectSuites(t, []string{"spinifex/services/nats/cluster.go"}) + if !got.AllSuites { + t.Fatalf("nats change must force all; got %+v", got) + } +} + +func TestSelect_ManyServices_ForcesAll(t *testing.T) { + got := selectSuites(t, []string{ + "spinifex/vpcd/x.go", + "spinifex/services/viperblockd/y.go", + "spinifex/services/awsgw/z.go", + }) + if !got.AllSuites { + t.Fatalf("≥3 services must force all; got %+v", got) + } +} + +func TestSelect_LoneSuiteDir(t *testing.T) { + got := selectSuites(t, []string{"tests/e2e/cert/cert_test.go"}) + wantSuites(t, got, "e2e-cert") + if got.AllSuites { + t.Fatalf("lone suite dir must not force all") + } +} + +func TestSelect_SuiteDirPlusSource_NotLone(t *testing.T) { + // A suite test edit alongside a service edit is not a lone-suite diff; it + // resolves through the service map (and may add suites). + got := selectSuites(t, []string{"tests/e2e/cert/cert_test.go", "spinifex/vpcd/x.go"}) + if len(got.Suites) < 2 { + t.Fatalf("expected service-mapped multi-suite, got %+v", got) + } +} + +func TestSelect_Submodule_ForcesAll(t *testing.T) { + got := selectSuites(t, []string{"viperblock"}) + if !got.AllSuites { + t.Fatalf("submodule pointer change must force all; got %+v", got) + } +} + +func TestSelect_UnmappedDocOnly_Default(t *testing.T) { + // A docs-only change maps to no service and no suite dir → default suites. + got := selectSuites(t, []string{"docs/DESIGN.md"}) + wantSuites(t, got, "e2e-cert", "e2e-lb") +} + +func TestSubjectMatch(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"vpc.*", "vpc.port-status", true}, + {"vpc.create", "vpc.*", true}, + {"vpc.*", "vpc.*", true}, + {"vpc.*", "vpcd.create", false}, // prefix must hit a segment boundary + {"ebs.*", "ebs.mount.response", true}, + {"s3.*", "s3.*", true}, + {"ec2.RunInstances", "ec2.RunInstances", true}, + {"ec2.RunInstances", "ec2.Terminate", false}, + {"vpc.create.*", "vpc.*", true}, + } + for _, c := range cases { + if got := subjectMatch(c.a, c.b); got != c.want { + t.Errorf("subjectMatch(%q,%q) = %v, want %v", c.a, c.b, got, c.want) + } + } +} + +// --- AST scan ------------------------------------------------------------- + +func TestScanSuites_PerTestCoverage(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "tests", "e2e", "faux") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + src := `//go:build e2e + +package faux + +import "testing" + +func helperVPC(t *testing.T) string { return harness.EnsureDefaultVPC(t, nil) } + +func TestVPCOnly(t *testing.T) { + _ = helperVPC(t) // transitive Ensure via local helper +} + +func TestInstance(t *testing.T) { + _ = harness.EnsureInstance(t, nil, spec) +} + +func TestReadOnly(t *testing.T) { + // no Ensure* calls → unknown coverage, always kept +} +` + if err := os.WriteFile(filepath.Join(dir, "faux_test.go"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + + m := testManifest() + m.Suites = map[string]manifestcheck.Suite{ + "e2e-faux": {Path: "tests/e2e/faux", Covers: []string{"vpcd"}}, + } + + cov, err := ScanSuites(root, m) + if err != nil { + t.Fatalf("ScanSuites: %v", err) + } + faux, ok := cov["e2e-faux"] + if !ok { + t.Fatalf("no coverage for e2e-faux: %+v", cov) + } + // TestVPCOnly reaches EnsureDefaultVPC via helperVPC → {vpcd, spinifex_daemon}. + if !reflect.DeepEqual(faux.Tests["TestVPCOnly"], []string{"spinifex_daemon", "vpcd"}) { + t.Errorf("TestVPCOnly services = %v", faux.Tests["TestVPCOnly"]) + } + // TestInstance → EnsureInstance → {spinifex_daemon, viperblockd, vpcd}. + if !reflect.DeepEqual(faux.Tests["TestInstance"], []string{"spinifex_daemon", "viperblockd", "vpcd"}) { + t.Errorf("TestInstance services = %v", faux.Tests["TestInstance"]) + } + // TestReadOnly has no fixtures. + if len(faux.Tests["TestReadOnly"]) != 0 { + t.Errorf("TestReadOnly should have no services, got %v", faux.Tests["TestReadOnly"]) + } +} + +func TestRunPatterns_StrictSubset(t *testing.T) { + perTest := map[string]SuiteCoverage{ + "e2e-single": {Tests: map[string][]string{ + "TestVPCCRUD": {"vpcd", "spinifex_daemon"}, + "TestVolume": {"viperblockd", "spinifex_daemon"}, + "TestReadOnly": {}, // always kept + "TestSnapshotBak": {"viperblockd", "spinifex_daemon", "predastore"}, + }}, + } + // covered excludes spinifex_daemon: only a vpcd-confined closure narrows, + // since almost every test transitively touches the daemon. + covered := map[string]bool{"vpcd": true} + got := runPatterns(perTest, []string{"e2e-single"}, covered) + // Keeps TestVPCCRUD (vpcd) + TestReadOnly (unknown). Drops Volume/Snapshot. + want := "^(TestReadOnly|TestVPCCRUD)$" + if got["e2e-single"] != want { + t.Fatalf("RUN_PATTERN = %q, want %q", got["e2e-single"], want) + } +} + +func TestRunPatterns_NoNarrowingWhenAllMatch(t *testing.T) { + perTest := map[string]SuiteCoverage{ + "e2e-single": {Tests: map[string][]string{ + "TestA": {"vpcd"}, + "TestB": {"vpcd"}, + }}, + } + covered := map[string]bool{"vpcd": true} + got := runPatterns(perTest, []string{"e2e-single"}, covered) + if _, ok := got["e2e-single"]; ok { + t.Fatalf("expected no pattern when all tests match, got %v", got) + } +} + +// Guard: the real manifest loads and Select runs against it without panicking. +func TestSelect_RealManifest_Smoke(t *testing.T) { + _, file, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(file), "..", "..", "..") + m, err := manifestcheck.Load(filepath.Join(root, "docs", "service-interfaces.yaml")) + if err != nil { + t.Fatalf("load real manifest: %v", err) + } + res := Select(m, DefaultConfig(), []string{"spinifex/vpcd/topology.go"}, nil) + if len(res.Suites) == 0 { + t.Fatalf("real manifest vpcd change selected no suites") + } +} diff --git a/tests/e2e/teardown/teardown_test.go b/tests/e2e/teardown/teardown_test.go new file mode 100644 index 000000000..367b7d725 --- /dev/null +++ b/tests/e2e/teardown/teardown_test.go @@ -0,0 +1,47 @@ +//go:build e2e + +// Package teardown is the run-scoped cleanup sweep (mulga-siv-277). It runs as +// the final suite of an e2e run: every resource the Ensure* fixtures created +// carries the e2e:run= tag, and this suite reclaims anything still +// tagged — covering leaks from a crashed suite, an interrupted run, or a +// persistent (non-tofu-destroyed) cluster. +// +// It is intentionally tolerant: a sweep error is logged, never fatal. Most +// resources are already gone (each Ensure* registers its own teardown), so the +// common case deletes nothing. The value is the long tail of leaks. +package teardown + +import ( + "os" + "testing" + + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// TestSweepRunResources deletes every EC2/ELBv2 resource tagged with the +// current run id. Skips when SPINIFEX_E2E is unset (no cluster) or +// SPINIFEX_E2E_RUN_ID is unset (nothing to scope the sweep to — without a run +// id the sweep would no-op anyway, so skipping is clearer than running it). +func TestSweepRunResources(t *testing.T) { + if os.Getenv("SPINIFEX_E2E") == "" { + t.Skip("SPINIFEX_E2E unset; no cluster to sweep") + } + runID := os.Getenv(harness.RunIDEnv) + if runID == "" { + t.Skipf("%s unset; nothing to sweep", harness.RunIDEnv) + } + + env := harness.LoadEnv(t) + awsCli := harness.NewAWSClient(t, env) + + rep := harness.SweepRunResources(awsCli.EC2, awsCli.ELBv2, runID) + + for kind, ids := range rep.Deleted { + t.Logf("swept %d %s: %v", len(ids), kind, ids) + } + for _, err := range rep.Errors { + // Non-fatal: a delete failure usually means the resource is already + // gone or still settling. Surface it for diagnostics, don't fail. + t.Logf("sweep error (non-fatal): %v", err) + } +}