From 044d35737e0f049708e1eec0ce2c16013179b3bf Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:32:30 +1000 Subject: [PATCH] feat: remove bash e2e for e2e-go --- .github/workflows/e2e-go.yml | 1173 --------------------- .github/workflows/e2e.yml | 1187 +++++++++++++++------ scripts/create-multi-node.sh | 122 --- scripts/start-multi-node.sh | 12 - tests/e2e/run-pseudo-multinode-e2e.sh | 1382 ------------------------- 5 files changed, 866 insertions(+), 3010 deletions(-) delete mode 100644 .github/workflows/e2e-go.yml delete mode 100755 scripts/create-multi-node.sh delete mode 100755 scripts/start-multi-node.sh delete mode 100755 tests/e2e/run-pseudo-multinode-e2e.sh diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml deleted file mode 100644 index 9236d7382..000000000 --- a/.github/workflows/e2e-go.yml +++ /dev/null @@ -1,1173 +0,0 @@ -name: E2E (Go) - -# Standalone workflow for the Go E2E harness (tests/e2e/{cert,lb,...}). -# Mirrors e2e.yml's single-node provisioning chain but runs `go test` via -# gotestsum so we get JUnit XML + per-test artifact directories surfaced to -# GitHub Actions. Manual-dispatch only while the conversion lands — push -# trigger lifts once the single-node Go port is green and replaces -# run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model. - -on: - workflow_dispatch: - inputs: - sha: - description: "PR head SHA to report status on" - required: false - suites: - description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')" - required: false - default: "e2e-single e2e-multinode e2e-cert e2e-lb" - test_run: - description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite." - required: false - default: "" - -permissions: - contents: read - statuses: write - -# Pushes to the same ref stack on the single `ci-single` self-hosted runner, -# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that -# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref. -concurrency: - group: e2e-go-${{ github.ref }} - cancel-in-progress: true - -env: - GIT_BRANCH: ${{ github.ref_name }} - SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }} - TEST_RUN: ${{ inputs.test_run || '' }} - -jobs: - e2e_go_single: - name: Single-Node E2E (Go) - # Skip when the dispatch input requests no in-VM suites (e.g. - # suites='e2e-reboot' only). Push triggers and the default SUITES list - # both include single/cert/lb so the inputs.suites null-check keeps the - # job running on plain PR pushes. - if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') - runs-on: [ self-hosted, ci-single ] - timeout-minutes: 45 - env: - TOFU_LOG: /tmp/e2e-go-tofu-single-${{ github.run_id }}.log - ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }} - steps: - - name: Reap Orphan Procs - # A cancelled prior run can leave `tofu`/`terraform` mid-apply or a - # libvirt `qemu+ssh` connection orphaned. Kill only procs whose cwd is - # under THIS runner's workspace — every self-hosted runner on this host - # shares the `tf-user` UID, so a UID-wide pkill would SIGKILL sibling - # workflows (e2e.yml/e2e-ddil.yml/...) mid-apply. SIGTERM first to let - # tofu release its state lock, then SIGKILL. - run: | - set +e - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 3 - reap KILL - # Drop only this workspace's stale tofu state locks from a SIGKILL'd apply. - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - true - - - name: Clean Workspace - run: sudo rm -rf "$GITHUB_WORKSPACE"/* - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - repositories: mulga - - - uses: actions/checkout@v6 - with: - repository: mulgadc/mulga - token: ${{ steps.app-token.outputs.token }} - sparse-checkout: scripts/tofu-cluster - path: mulga - - - uses: actions/checkout@v6 - with: - path: spinifex - - - uses: actions/setup-go@v6 - with: - go-version-file: spinifex/go.mod - # Self-hosted runner; the cache upload to GitHub blob storage - # dominates total job time and the local Go module cache already - # warms across runs. - cache: false - - - name: Re-enable Go toolchain auto-download - # setup-go@v6 exports GOTOOLCHAIN=local to $GITHUB_ENV (v5 left it - # "auto"). build-install-artifacts.sh prepends /usr/local/go/bin to - # PATH on the self-hosted runner, and that Go is older than - # spinifex/go.mod's `go` directive; under "local" the build fails - # with `requires go >= X.Y.Z`. Restoring "auto" lets the resolved - # Go self-heal by fetching the named minor on demand. - run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - - name: Prune rootless docker cache - # Self-hosted runner uses rootless docker; build-microvm-image.sh - # leaks ~100 MiB overlay2 layer per failed build. Prune before each - # run keeps /home//.local/share/docker bounded. - run: | - docker container prune -f || true - docker image prune -af || true - docker buildx prune -af --filter "until=24h" || true - docker system df || true - - - name: Build Install Artifacts - working-directory: mulga/scripts/tofu-cluster - run: | - ./build-install-artifacts.sh "${GIT_BRANCH}" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Provision VM - working-directory: mulga/scripts/tofu-cluster - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - export TF_VAR_git_branch="${GIT_BRANCH}" - quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true - quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true - quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - ok "VM provisioned" - - - name: Bootstrap via Binary Install - working-directory: mulga/scripts/tofu-cluster - run: | - SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ - ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Resolve VM SSH details - id: vm - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT" - echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" - - - name: Build E2E test binaries on runner - working-directory: spinifex/tests/e2e - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - # SUITES is in make-target form (e.g. "e2e-cert e2e-lb"); strip the - # "e2e-" prefix to get package directory names for `go test -c`. - # 'reboot' is filtered out here — that suite runs runner-resident - # via the e2e_go_reboot job, not in-VM, so this build step doesn't - # need its binary. - # Filter out reboot (runs on e2e_go_reboot) and multinode (runs on - # e2e_go_multinode). Single VM has no 3-node cluster — building + - # shipping multinode.test here would just produce a binary that - # skips on its own SPINIFEX_MODE check. - SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ') - if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then - echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build" - exit 0 - fi - banner "build ${SUITES_BUILD// /, }" - group_open "go test -c" - make e2e-build SUITES_BUILD="${SUITES_BUILD}" - group_close - for f in _bin/*.test; do - ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" - done - - - name: Install go-junit-report on runner - run: | - # Converts `go test -v` text output (from the pre-built binaries on - # the VM) into the junit-*.xml that "Publish test summary" expects. - go install github.com/jstemmer/go-junit-report/v2@latest - echo "$HOME/go/bin" >> "$GITHUB_PATH" - - - name: Ship test binaries to VM - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" - tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" - ok "shipped binaries to $VM" - - - name: Start packet capture on VM br-wan - run: | - set +e - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - scp "${SSH_OPTS[@]}" \ - "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ - "$VM":/tmp/start-net-capture.sh - ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true - - - name: Run Go E2E suites - id: e2e - run: | - set +e - # SUITES are make-target names ("e2e-cert" → cert.test). Each - # pre-built binary is self-contained (testdata //go:embed'd) so the - # VM needs no go toolchain, no source tree, no gotestsum. Each - # suite gets its own ::group:: + cyan banner inside so the Actions - # UI collapses suite output by default; expand only what failed. - # 'reboot' is filtered: runs via e2e_go_reboot on the runner, not here. - SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ') - if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then - echo "no in-VM suites requested (SUITES=${SUITES}); skipping" - echo "rc=0" >> "$GITHUB_OUTPUT" - exit 0 - fi - # Optional regex passed through to `go test -test.run`. Empty input - # means run all tests in the suite (default behaviour). - if [ -n "${TEST_RUN}" ]; then - TEST_RUN_FLAG="-test.run=${TEST_RUN}" - else - TEST_RUN_FLAG="" - fi - VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}" - SSH_KEY="${{ steps.vm.outputs.ssh_key }}" - # On cancel / runner SIGTERM / step timeout, force-kill any test - # binaries left running on the VM. Without this, the remote `.test` - # process keeps holding the suite's NATS connections + tmp dirs and - # the subsequent Teardown's tofu-destroy races with its file - # handles. -o ConnectTimeout=5 so a wedged VM doesn't block cleanup. - cleanup_remote() { - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ - 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ - 2>/dev/null || true - } - trap cleanup_remote EXIT INT TERM - # `ssh -tt` forces PTY allocation. When the runner cancels this - # step, the local ssh dies and the remote sshd sends SIGHUP through - # the PTY to its children — without -tt the remote `.test` would - # orphan and keep running until its own 30m timeout. - ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \ - "$VM_HOST" \ - "set -u; mkdir -p '${ARTIFACT_DIR}'; \ - cd \$HOME/spinifex-e2e-bin; \ - R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ - FAIL=0; \ - for SUITE in ${SUITES_RUN}; do \ - echo \"::group::suite \$SUITE\"; \ - printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \ - # Record suite wall-clock start so the analyzer can locate - # the right journal window. junit-report's timestamp attr - # marks conversion time, not test start, so we cannot use it. - date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ - GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ - ARTIFACT_DIR='${ARTIFACT_DIR}' \ - ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ - 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ - | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ - rc=\${PIPESTATUS[0]}; \ - echo \"::endgroup::\"; \ - if [ \$rc -ne 0 ]; then \ - printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \ - echo \"::error::suite \$SUITE failed (exit \$rc)\"; \ - echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \ - FAIL=1; \ - else \ - printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \ - fi; \ - done; \ - exit \$FAIL" - E2E_RC=$? - echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" - exit 0 # don't fail the step — artifact upload + final gate happen below - - - name: Capture network/IPsec diagnostics on VM - if: always() - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the - # output for the next step's tar to pull down. Captures OVN/OVS/IPsec - # state at failure time — before teardown destroys the VM. - scp "${SSH_OPTS[@]}" \ - "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ - "$VM":/tmp/capture-net-diag.sh - ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" - ok "net diagnostics captured" - - - name: Collect artifacts from VM - if: always() - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - mkdir -p "${ARTIFACT_DIR}" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - # `|| true` so a missing dir on an early failure doesn't mask the - # original error. - ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ - | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null - ok "artifacts pulled to ${ARTIFACT_DIR}" - # `-o short-iso` writes RFC3339-ish timestamps (e.g. - # `2026-05-19T12:32:34+1000 host …`) so the analyzer can window-slice - # the journal around each failure's wall-clock span. - ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ - > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null - ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" - - - name: Convert test logs to JUnit XML - if: always() - run: | - # VM runs the pre-built test binaries with `-test.v` and tees plain - # text logs (no Go toolchain on the VM to emit test2json). Convert - # each log to the junit-.xml shape "Publish test summary" - # already parses. - shopt -s nullglob - for LOG in "${ARTIFACT_DIR}"/test-*.log; do - BASE=$(basename "$LOG" .log) - SUITE=${BASE#test-} - go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true - done - - - name: Publish test summary - if: always() - uses: ./spinifex/.github/actions/junit-report - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - title: "Go E2E results" - - - name: Analyze e2e failures - if: always() - uses: ./spinifex/.github/actions/e2e-analyze - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - log-dir: ${{ env.ARTIFACT_DIR }} - title: "E2E failure analysis" - - - uses: actions/upload-artifact@v7 - if: always() - with: - name: e2e-go-artifacts-${{ github.run_id }} - path: ${{ env.ARTIFACT_DIR }} - if-no-files-found: warn - retention-days: 14 - - - name: Teardown - if: always() - working-directory: mulga/scripts/tofu-cluster - # `set -e` off — teardown must run scrub-env even if tofu destroy - # fails; scrub is the libvirt-level fallback when tofu state is - # inconsistent with the remote Proxmox node. - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - # Kill any test binaries still running on the VM (cancelled mid-suite - # paths) BEFORE tofu destroy yanks the qcow2 from underneath them. - # Scoped to this run's VM by wan_ip — safe with sibling agents on - # the same host. ConnectTimeout=5 so an already-wedged VM doesn't - # block teardown. - if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ - -i "${{ steps.vm.outputs.ssh_key }}" \ - tf-user@${{ steps.vm.outputs.wan_ip }} \ - 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ - 2>/dev/null || true - fi - # Cancelled jobs from this point on can orphan an in-flight - # tofu/libvirt SSH; clear them before destroy so the next run's - # pre-clean isn't doing it. Scoped to this runner's workspace via - # /proc/$pid/cwd — sibling runners share the tf-user UID, so a - # UID-wide pkill would nuke their tofu mid-apply. - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 2 - reap KILL - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" - ok "teardown complete" - - - name: Fail job if any suite failed - if: always() - run: | - if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then - echo "::error::Go E2E suites failed (rc=${{ steps.e2e.outputs.rc }})" - cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true - exit 1 - fi - - e2e_go_multinode: - name: Multi-Node E2E (Go) - # Opt-in via e2e-multinode in the SUITES list. Runs the Go multinode - # suite (port of run-multinode-e2e.sh) plus cert + lb against a 3-node - # tofu cluster. Mirrors e2e_go_single's binary-ship + per-suite flow, - # adapted for the ci-multi runner pool and SPINIFEX_NODES env wiring. - if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') - runs-on: [ self-hosted, ci-multi ] - timeout-minutes: 60 - env: - TOFU_LOG: /tmp/e2e-go-tofu-multi-${{ github.run_id }}.log - ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }} - steps: - - name: Reap Orphan Procs - run: | - set +e - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 3 - reap KILL - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - true - - - name: Clean Workspace - run: sudo rm -rf "$GITHUB_WORKSPACE"/* - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - repositories: mulga - - - uses: actions/checkout@v6 - with: - repository: mulgadc/mulga - token: ${{ steps.app-token.outputs.token }} - sparse-checkout: scripts/tofu-cluster - path: mulga - - - uses: actions/checkout@v6 - with: - path: spinifex - - - uses: actions/setup-go@v6 - with: - go-version-file: spinifex/go.mod - cache: false - - - name: Re-enable Go toolchain auto-download - run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - - name: Prune rootless docker cache - # Self-hosted runner uses rootless docker; build-microvm-image.sh - # leaks ~100 MiB overlay2 layer per failed build. Prune before each - # run keeps /home//.local/share/docker bounded. - run: | - docker container prune -f || true - docker image prune -af || true - docker buildx prune -af --filter "until=24h" || true - docker system df || true - - - name: Build Install Artifacts - working-directory: mulga/scripts/tofu-cluster - run: | - ./build-install-artifacts.sh "${GIT_BRANCH}" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Provision VMs - working-directory: mulga/scripts/tofu-cluster - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - export TF_VAR_git_branch="${GIT_BRANCH}" - quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true - quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true - quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - ok "3-node cluster provisioned" - - - name: Bootstrap via Binary Install - working-directory: mulga/scripts/tofu-cluster - run: | - SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ - ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Resolve cluster SSH details - id: cluster - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - # Comma-separated node list for SPINIFEX_NODES (ClusterFromEnv) - # and SPINIFEX_NODE_IPS (Env.NodeIPs). Both consume the same csv. - NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') - NODES_CSV="" - for i in $(seq 0 $((NODE_COUNT - 1))); do - ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") - if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi - done - echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT" - echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" - echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT" - - - name: Build E2E test binaries on runner - working-directory: spinifex/tests/e2e - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - # Strip e2e- prefix, drop reboot (runner-resident) and single - # (single-VM job only) so the multinode job builds only what it - # ships: multinode, cert, lb. - SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ') - if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then - echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build" - exit 0 - fi - banner "build ${SUITES_BUILD// /, }" - group_open "go test -c" - make e2e-build SUITES_BUILD="${SUITES_BUILD}" - group_close - for f in _bin/*.test; do - ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" - done - - - name: Install go-junit-report on runner - run: | - go install github.com/jstemmer/go-junit-report/v2@latest - echo "$HOME/go/bin" >> "$GITHUB_PATH" - - - name: Ship test binaries to node1 - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") - VM=tf-user@${{ steps.cluster.outputs.node1_ip }} - ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" - tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" - ok "shipped binaries to $VM" - - - name: Start packet capture on all nodes br-wan - run: | - set +e - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") - IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" - for ip in "${NODE_LIST[@]}"; do - [ -z "$ip" ] && continue - scp "${SSH_OPTS[@]}" \ - "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ - tf-user@"$ip":/tmp/start-net-capture.sh - ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true - done - - - name: Pre-flight diagnostics on node1 - # Captures cluster + tf-user state directly via SSH so the post-mortem - # in run 26163994815-style spx-NATS failures has structured data - # before the test binary runs. Best-effort (|| true on everything) — - # this step never fails the job, only feeds the artifact bundle. - run: | - set +e - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" 2>/dev/null || true - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") - VM=tf-user@${{ steps.cluster.outputs.node1_ip }} - mkdir -p "${ARTIFACT_DIR}" - DIAG="${ARTIFACT_DIR}/preflight-diag.log" - { - echo "=== id (tf-user groups) ===" - ssh "${SSH_OPTS[@]}" "$VM" 'id; echo; getent group spinifex || true' - echo - echo "=== ls -la /etc/spinifex/ ===" - ssh "${SSH_OPTS[@]}" "$VM" 'ls -la /etc/spinifex/ 2>&1; echo; ls -la /etc/spinifex/nats/ 2>&1' - echo - echo "=== spinifex.toml NATS section ===" - ssh "${SSH_OPTS[@]}" "$VM" 'grep -A4 "\[nodes\\..*\\.nats" /etc/spinifex/spinifex.toml 2>&1 | head -40' - echo - echo "=== systemctl is-active spinifex-* ===" - ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done' - echo - echo "=== which spx + version ===" - ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true' - echo - echo "=== spx get nodes (direct ssh, tf-user) ===" - ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"' - echo - echo "=== spx top nodes (direct ssh, tf-user) ===" - ssh "${SSH_OPTS[@]}" "$VM" 'spx top nodes 2>&1 | head -30; echo "exit=$?"' - echo - echo "=== NATS routez (curl 127.0.0.1:8222) ===" - ssh "${SSH_OPTS[@]}" "$VM" 'curl -fsS http://127.0.0.1:8222/routez 2>&1 | head -40' - echo - echo "=== aws describe-instance-types via gateway (tf-user) ===" - ssh "${SSH_OPTS[@]}" "$VM" 'AWS_PROFILE=spinifex AWS_CA_BUNDLE=/etc/spinifex/ca.pem aws --endpoint-url https://${{ steps.cluster.outputs.node1_ip }}:9999 ec2 describe-instance-types --query "InstanceTypes[*].InstanceType" --output text 2>&1 | head -5' - } 2>&1 | tee "$DIAG" - true - - - name: Run Go E2E suites - id: e2e - run: | - set +e - # multinode + cert + lb only — see SUITES_BUILD comment above. - SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ') - if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then - echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping" - echo "rc=0" >> "$GITHUB_OUTPUT" - exit 0 - fi - if [ -n "${TEST_RUN}" ]; then - TEST_RUN_FLAG="-test.run=${TEST_RUN}" - else - TEST_RUN_FLAG="" - fi - VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}" - SSH_KEY="${{ steps.cluster.outputs.ssh_key }}" - NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}" - cleanup_remote() { - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ - 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ - 2>/dev/null || true - } - trap cleanup_remote EXIT INT TERM - # SPINIFEX_NODES + SPINIFEX_NODE_IPS share the csv; ClusterFromEnv - # uses the first; cert/lb use Env.NodeIPs for SAN + endpoint maps. - # SPINIFEX_SSH_KEY default ~/.ssh/tf-user-ap-southeast-2 lines up - # with the bash peer_ssh path — bootstrap-install.sh writes it. - ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \ - "$VM_HOST" \ - "set -u; mkdir -p '${ARTIFACT_DIR}'; \ - cd \$HOME/spinifex-e2e-bin; \ - R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ - FAIL=0; \ - for SUITE in ${SUITES_RUN}; do \ - echo \"::group::suite \$SUITE\"; \ - printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \ - date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ - GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \ - SPINIFEX_NODES='${NODES_CSV}' \ - SPINIFEX_NODE_IPS='${NODES_CSV}' \ - SPINIFEX_SSH_USER=tf-user \ - SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \ - ARTIFACT_DIR='${ARTIFACT_DIR}' \ - ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ - 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ - | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ - rc=\${PIPESTATUS[0]}; \ - echo \"::endgroup::\"; \ - if [ \$rc -ne 0 ]; then \ - printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \ - echo \"::error::suite \$SUITE failed (exit \$rc)\"; \ - echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \ - FAIL=1; \ - else \ - printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \ - fi; \ - done; \ - exit \$FAIL" - E2E_RC=$? - echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" - exit 0 - - - name: Capture network/IPsec diagnostics on all nodes - if: always() - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") - # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls - # from). One subdir per node so xfrm SA state is comparable across - # peers — critical for diagnosing the IPsec wiring. - IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" - NODE1_IP="${NODE_LIST[0]}" - scp "${SSH_OPTS[@]}" \ - "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ - tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh - for i in "${!NODE_LIST[@]}"; do - idx=$((i + 1)) - ip="${NODE_LIST[$i]}" - if [ "$idx" -ne 1 ]; then - scp "${SSH_OPTS[@]}" \ - "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ - tf-user@"$ip":/tmp/capture-net-diag.sh - fi - ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ - "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'" - # node1 keeps its diag locally; pull node2..N back into node1's - # ARTIFACT_DIR so the existing tar-pull below catches everything. - if [ "$idx" -ne 1 ]; then - ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ - "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \ - | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \ - "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null" - fi - ok "net diagnostics node${idx} captured" - done - - - name: Collect artifacts from node1 - if: always() - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - mkdir -p "${ARTIFACT_DIR}" - SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") - VM=tf-user@${{ steps.cluster.outputs.node1_ip }} - ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ - | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null - ok "artifacts pulled to ${ARTIFACT_DIR}" - # Fan out journal collection across all 3 nodes so a failure - # signal on node2/3 isn't blind. Naming matches DumpAllNodeLogs. - IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" - for i in "${!NODE_LIST[@]}"; do - idx=$((i + 1)) - ip="${NODE_LIST[$i]}" - ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ - > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null - ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines" - done - # openvswitch-ipsec journal per node (separate file). - # Debian unit name; the binary inside is /usr/share/openvswitch/scripts/ovs-ctl - # which spawns ovs-monitor-ipsec as a child. - for i in "${!NODE_LIST[@]}"; do - idx=$((i + 1)) - ip="${NODE_LIST[$i]}" - ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \ - > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null - done - - - name: Convert test logs to JUnit XML - if: always() - run: | - shopt -s nullglob - for LOG in "${ARTIFACT_DIR}"/test-*.log; do - BASE=$(basename "$LOG" .log) - SUITE=${BASE#test-} - go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true - done - - - name: Publish test summary - if: always() - uses: ./spinifex/.github/actions/junit-report - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - title: "Go Multi-Node E2E results" - - - name: Analyze e2e failures - if: always() - uses: ./spinifex/.github/actions/e2e-analyze - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - log-dir: ${{ env.ARTIFACT_DIR }} - title: "Multi-Node E2E failure analysis" - - - uses: actions/upload-artifact@v7 - if: always() - with: - name: e2e-go-multi-artifacts-${{ github.run_id }} - path: ${{ env.ARTIFACT_DIR }} - if-no-files-found: warn - retention-days: 14 - - - name: Teardown - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then - IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" - for ip in "${NODE_LIST[@]}"; do - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ - -i "${{ steps.cluster.outputs.ssh_key }}" \ - tf-user@"$ip" \ - 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ - 2>/dev/null || true - done - fi - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 2 - reap KILL - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" - ok "teardown complete" - - - name: Fail job if any suite failed - if: always() - run: | - if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then - echo "::error::Go multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})" - cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true - exit 1 - fi - - e2e_go_reboot: - name: Reboot Resilience E2E (Go) - # Opt-in: only fires when the SUITES dispatch input lists e2e-reboot. - # The reboot test reboots the spinifex VM, so it must run on the runner - # (not shipped to the VM like the cert/lb/single suites). Runs in - # parallel with e2e_go_single — both target [self-hosted, ci-single] - # and GitHub Actions serialises on the underlying runner allocation, - # so no explicit needs: ordering is required. - if: contains(github.event.inputs.suites, 'e2e-reboot') - runs-on: [ self-hosted, ci-single ] - timeout-minutes: 60 - env: - TOFU_LOG: /tmp/e2e-go-tofu-reboot-${{ github.run_id }}.log - ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }} - steps: - - name: Reap Orphan Procs - run: | - set +e - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 3 - reap KILL - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - true - - - name: Clean Workspace - run: sudo rm -rf "$GITHUB_WORKSPACE"/* - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - repositories: mulga - - - uses: actions/checkout@v6 - with: - repository: mulgadc/mulga - token: ${{ steps.app-token.outputs.token }} - sparse-checkout: scripts/tofu-cluster - path: mulga - - - uses: actions/checkout@v6 - with: - path: spinifex - - - uses: actions/setup-go@v6 - with: - go-version-file: spinifex/go.mod - cache: false - - - name: Re-enable Go toolchain auto-download - run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - - name: Prune rootless docker cache - # Self-hosted runner uses rootless docker; build-microvm-image.sh - # leaks ~100 MiB overlay2 layer per failed build. Prune before each - # run keeps /home//.local/share/docker bounded. - run: | - docker container prune -f || true - docker image prune -af || true - docker buildx prune -af --filter "until=24h" || true - docker system df || true - - - name: Build Install Artifacts - working-directory: mulga/scripts/tofu-cluster - run: | - ./build-install-artifacts.sh "${GIT_BRANCH}" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Provision VM - working-directory: mulga/scripts/tofu-cluster - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - export TF_VAR_git_branch="${GIT_BRANCH}" - quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true - quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true - quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - ok "VM provisioned" - - - name: Bootstrap via Binary Install - working-directory: mulga/scripts/tofu-cluster - run: | - SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ - ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ - "${RUNNER_TEMP}/setup.sh" \ - "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ - --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - - - name: Resolve VM SSH details - id: vm - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT" - echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" - - - name: Build reboot.test on runner - working-directory: spinifex/tests/e2e - run: | - set -euo pipefail - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - banner "build reboot" - group_open "go test -c" - make e2e-build SUITES_BUILD="reboot" - group_close - ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)" - - - name: Fetch AWS credentials from VM - id: creds - run: | - set -euo pipefail - # The spinifex CLI on the VM (via `spx admin init` during bootstrap) - # writes the [spinifex] profile to tf-user's ~/.aws/credentials. - # Pull it over SSH and expose as SPINIFEX_AWS_* env vars (the same - # static-cred path used by harness.NewAWSClient). Mirrors the cred - # hop already proven by e2e-ddil.yml. - SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials') - AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}') - AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}') - if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then - echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}" - exit 1 - fi - echo "::add-mask::${AWS_AK}" - echo "::add-mask::${AWS_SK}" - echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT" - echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT" - - - name: Run reboot suite on runner - id: e2e - run: | - set +e - mkdir -p "${ARTIFACT_DIR}" - # Banner + capture wall-clock start so the e2e-analyze action can - # window-slice the journal around the test span (matches the - # convention in e2e_go_single). - date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start" - R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m' - printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R" - - if [ -n "${TEST_RUN}" ]; then - TEST_RUN_FLAG="-test.run=${TEST_RUN}" - else - TEST_RUN_FLAG="" - fi - GITHUB_ACTIONS=true FORCE_COLOR=1 \ - SPINIFEX_E2E=1 SPINIFEX_MODE=single \ - SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \ - SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \ - SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \ - SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \ - SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \ - ARTIFACT_DIR="${ARTIFACT_DIR}" \ - ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ - 2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \ - | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//' - rc=${PIPESTATUS[0]} - echo "rc=${rc}" >> "$GITHUB_OUTPUT" - exit 0 - - - name: Collect VM journals - if: always() - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - mkdir -p "${ARTIFACT_DIR}" - SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \ - -i "${{ steps.vm.outputs.ssh_key }}") - VM=tf-user@${{ steps.vm.outputs.wan_ip }} - ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ - > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true - ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" - - - name: Install go-junit-report - if: always() - run: | - go install github.com/jstemmer/go-junit-report/v2@latest - echo "$HOME/go/bin" >> "$GITHUB_PATH" - - - name: Convert test log to JUnit XML - if: always() - run: | - shopt -s nullglob - for LOG in "${ARTIFACT_DIR}"/test-*.log; do - BASE=$(basename "$LOG" .log) - SUITE=${BASE#test-} - go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true - done - - - name: Publish test summary - if: always() - uses: ./spinifex/.github/actions/junit-report - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - title: "Go E2E reboot results" - - - name: Analyze e2e failures - if: always() - uses: ./spinifex/.github/actions/e2e-analyze - with: - junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml - log-dir: ${{ env.ARTIFACT_DIR }} - title: "Reboot E2E failure analysis" - - - uses: actions/upload-artifact@v7 - if: always() - with: - name: e2e-go-reboot-artifacts-${{ github.run_id }} - path: ${{ env.ARTIFACT_DIR }} - if-no-files-found: warn - retention-days: 14 - - - name: Teardown - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - set +e - # shellcheck source=/dev/null - source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" - UID_SELF=$(id -u) - WS="${GITHUB_WORKSPACE}" - reap() { - local sig="$1" - for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do - cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue - case "$cwd" in - "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; - esac - done - } - reap TERM - sleep 2 - reap KILL - find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null - quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" - ok "teardown complete" - - - name: Fail job if reboot suite failed - if: always() - run: | - if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then - echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})" - exit 1 - fi - - e2e_go_status: - name: Report E2E (Go) Status - # Always post status. When inputs.sha is unset (manual dispatch without - # a PR head), fall back to github.sha — the resolved commit SHA of the - # dispatched ref, always present. The previous `&& inputs.sha` gate - # plus `statuses/${{ inputs.sha }}` produced an 'invalid hash' error - # whenever the workflow was dispatched without the optional input - # (tests passed, status step silently failed). - if: always() - needs: [ e2e_go_single, e2e_go_multinode, e2e_go_reboot ] - runs-on: [ self-hosted, ci-single ] - timeout-minutes: 5 - steps: - - name: Report Status to PR - env: - GH_TOKEN: ${{ github.token }} - REPORT_SHA: ${{ inputs.sha || github.sha }} - run: | - if [[ -z "${REPORT_SHA}" ]]; then - echo "::error::no SHA available (inputs.sha and github.sha both empty)" - exit 1 - fi - # Treat skipped as success — e2e_go_multinode and e2e_go_reboot - # only run when opted in via the SUITES input, so a single-only - # dispatch should still post a green status when the single job - # passes and the others skipped. - single_ok=true; multi_ok=true; reboot_ok=true - case "${{ needs.e2e_go_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac - case "${{ needs.e2e_go_multinode.result }}" in success|skipped) ;; *) multi_ok=false ;; esac - case "${{ needs.e2e_go_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac - if $single_ok && $multi_ok && $reboot_ok; then - STATE="success" - DESC="Go E2E suites passed" - else - STATE="failure" - DESC="Go E2E failed: single=${{ needs.e2e_go_single.result }} multi=${{ needs.e2e_go_multinode.result }} reboot=${{ needs.e2e_go_reboot.result }}" - fi - gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \ - -f state="$STATE" \ - -f context=e2e-go \ - -f description="$DESC" \ - -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5c546aad1..9f745a773 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -17,22 +17,65 @@ on: sha: description: "PR head SHA to report status on" required: false + suites: + description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')" + required: false + default: "e2e-single e2e-multinode e2e-cert e2e-lb" + test_run: + description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite." + required: false + default: "" permissions: contents: read statuses: write +# Pushes to the same ref stack on the single `ci-single` self-hosted runner, +# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that +# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref. +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + env: GIT_BRANCH: ${{ github.ref_name }} + SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }} + TEST_RUN: ${{ inputs.test_run || '' }} jobs: e2e_single: name: Single-Node E2E + # Skip when the dispatch input requests no in-VM suites (e.g. + # suites='e2e-reboot' only). Push triggers and the default SUITES list + # both include single/cert/lb so the inputs.suites null-check keeps the + # job running on plain PR pushes. + if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') runs-on: [ self-hosted, ci-single ] - timeout-minutes: 35 + timeout-minutes: 45 env: TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log + ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }} steps: + - name: Reap Orphan Procs + run: | + set +e + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 3 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + true + - name: Clean Workspace run: sudo rm -rf "$GITHUB_WORKSPACE"/* @@ -50,6 +93,25 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Re-enable Go toolchain auto-download + run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" + + - name: Prune rootless docker cache + run: | + docker container prune -f || true + docker image prune -af || true + docker buildx prune -af --filter "until=24h" || true + docker system df || true + - name: Build Install Artifacts working-directory: mulga/scripts/tofu-cluster run: | @@ -61,12 +123,15 @@ jobs: - name: Provision VM working-directory: mulga/scripts/tofu-cluster run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" export TF_VAR_git_branch="${GIT_BRANCH}" - tofu init -input=false >> "$TOFU_LOG" 2>&1 - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "VM provisioned" - name: Bootstrap via Binary Install working-directory: mulga/scripts/tofu-cluster @@ -77,253 +142,246 @@ jobs: "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - - name: Start Packet Capture (siv-167) + - name: Resolve VM SSH details + id: vm working-directory: mulga/scripts/tofu-cluster run: | INVENTORY=$(tofu output -json inventory) WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE' - set -x - sudo mkdir -p /tmp/spinifex-diag - sudo chmod 0777 /tmp/spinifex-diag - sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid /tmp/spinifex-diag/tcpdump.log - sudo bash -c ' - nohup tcpdump -i br-wan -nn -s 0 -U \ - -w /tmp/spinifex-diag/br-wan.pcap \ - "(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp" \ - /tmp/spinifex-diag/tcpdump.log 2>&1 & - echo $! > /tmp/spinifex-diag/tcpdump.pid - ' - sleep 2 - PID=$(cat /tmp/spinifex-diag/tcpdump.pid 2>/dev/null || echo "") - if [ -n "$PID" ] && sudo kill -0 "$PID" 2>/dev/null; then - echo "tcpdump running PID=$PID" - ls -la /tmp/spinifex-diag/ + echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT" + echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" + + - name: Build E2E test binaries on runner + working-directory: spinifex/tests/e2e + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ') + if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then + echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build" + exit 0 + fi + banner "build ${SUITES_BUILD// /, }" + group_open "go test -c" + make e2e-build SUITES_BUILD="${SUITES_BUILD}" + group_close + for f in _bin/*.test; do + ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" + done + + - name: Install go-junit-report on runner + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Ship test binaries to VM + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" + tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" + ok "shipped binaries to $VM" + + - name: Start packet capture on VM br-wan + run: | + set +e + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ + "$VM":/tmp/start-net-capture.sh + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true + + - name: Run E2E suites + id: e2e + run: | + set +e + SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ') + if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then + echo "no in-VM suites requested (SUITES=${SUITES}); skipping" + echo "rc=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" else - echo "tcpdump failed to start" - cat /tmp/spinifex-diag/tcpdump.log 2>&1 || true + TEST_RUN_FLAG="" + fi + VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}" + SSH_KEY="${{ steps.vm.outputs.ssh_key }}" + cleanup_remote() { + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + } + trap cleanup_remote EXIT INT TERM + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \ + "$VM_HOST" \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; \ + cd \$HOME/spinifex-e2e-bin; \ + R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ + FAIL=0; \ + for SUITE in ${SUITES_RUN}; do \ + echo \"::group::suite \$SUITE\"; \ + printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ + 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ + | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ + rc=\${PIPESTATUS[0]}; \ + echo \"::endgroup::\"; \ + if [ \$rc -ne 0 ]; then \ + printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \ + echo \"::error::suite \$SUITE failed (exit \$rc)\"; \ + echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \ + FAIL=1; \ + else \ + printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \ + fi; \ + done; \ + exit \$FAIL" + E2E_RC=$? + echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Capture network/IPsec diagnostics on VM + if: always() + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + "$VM":/tmp/capture-net-diag.sh + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" + ok "net diagnostics captured" + + - name: Collect artifacts from VM + if: always() + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + ok "artifacts pulled to ${ARTIFACT_DIR}" + ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null + ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" + + - name: Convert test logs to JUnit XML + if: always() + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log) + SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Publish test summary + if: always() + uses: ./spinifex/.github/actions/junit-report + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + title: "E2E results" + + - name: Analyze e2e failures + if: always() + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "E2E failure analysis" + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-artifacts-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: warn + retention-days: 14 + + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + -i "${{ steps.vm.outputs.ssh_key }}" \ + tf-user@${{ steps.vm.outputs.wan_ip }} \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + fi + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 2 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" + ok "teardown complete" + + - name: Fail job if any suite failed + if: always() + run: | + if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then + echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})" + cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true exit 1 fi - REMOTE - - - name: Run E2E Tests - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ - "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh" - - - name: Collect Diagnostics - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - DIAG_DIR="${RUNNER_TEMP}/diag-single" - mkdir -p "$DIAG_DIR" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true - set -x - DIAG=/tmp/spinifex-diag - sudo mkdir -p "$DIAG" && cd "$DIAG" - if [ -f tcpdump.pid ]; then - PID=$(cat tcpdump.pid) - sudo kill -INT "$PID" 2>/dev/null || true - for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done - sudo kill -KILL "$PID" 2>/dev/null || true - fi - sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true - sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true - sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true - sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true - sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true - sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true - sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true - sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true - sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true - sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true - sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true - sudo ip -d link > ip-link.txt 2>&1 || true - sudo ip addr > ip-addr.txt 2>&1 || true - sudo ip route > ip-route.txt 2>&1 || true - sudo ip neigh > ip-neigh.txt 2>&1 || true - sudo bridge fdb show > bridge-fdb.txt 2>&1 || true - sudo bridge link > bridge-link.txt 2>&1 || true - for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do - [ -e "$f" ] && echo "$f $(cat $f)" - done > hairpin-modes.txt 2>&1 || true - sudo iptables-save > iptables.txt 2>&1 || true - sudo nft list ruleset > nft.txt 2>&1 || true - sudo dmesg --ctime > dmesg.txt 2>&1 || true - sudo virsh list --all > virsh-list.txt 2>&1 || true - ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true - sudo chmod -R a+r /tmp/spinifex-diag - tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag - REMOTE - scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \ - tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true - - - name: Upload Diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: e2e-diag-single-${{ github.run_id }} - path: ${{ runner.temp }}/diag-single/ - if-no-files-found: warn - retention-days: 14 - - - name: Teardown - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - - e2e_pseudo_multi: - name: Pseudo Multi-Node E2E - runs-on: [ self-hosted, ci-pseudo ] - timeout-minutes: 25 - env: - TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log - steps: - - name: Clean Workspace - run: sudo rm -rf "$GITHUB_WORKSPACE"/* - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - repositories: mulga - - - uses: actions/checkout@v6 - with: - repository: mulgadc/mulga - token: ${{ steps.app-token.outputs.token }} - sparse-checkout: scripts/tofu-cluster - path: mulga - - - name: Clone and Build - working-directory: mulga/scripts/tofu-cluster - run: ./build-artifacts.sh "${GIT_BRANCH}" - "${RUNNER_TEMP}/spinifex-artifacts.tar" - - - name: Provision VM - working-directory: mulga/scripts/tofu-cluster - run: | - export TF_VAR_git_branch="${GIT_BRANCH}" - tofu init -input=false >> "$TOFU_LOG" 2>&1 - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - - - name: Bootstrap Cluster - working-directory: mulga/scripts/tofu-cluster - run: | - SPX_PREBUILT=true \ - SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \ - ./bootstrap.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - - - name: Run E2E Tests - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ - "cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL=\"${SPINIFEX_CI_POOL}\" ./tests/e2e/run-pseudo-multinode-e2e.sh" - - - name: Collect Diagnostics - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - INVENTORY=$(tofu output -json inventory) - WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - DIAG_DIR="${RUNNER_TEMP}/diag-pseudo" - mkdir -p "$DIAG_DIR" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true - set -x - DIAG=/tmp/spinifex-diag - sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" - sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true - sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true - sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true - sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true - sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true - sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true - sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true - sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true - sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true - sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true - sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true - sudo ip -d link > ip-link.txt 2>&1 || true - sudo ip addr > ip-addr.txt 2>&1 || true - sudo ip route > ip-route.txt 2>&1 || true - sudo ip neigh > ip-neigh.txt 2>&1 || true - sudo bridge fdb show > bridge-fdb.txt 2>&1 || true - sudo bridge link > bridge-link.txt 2>&1 || true - for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do - [ -e "$f" ] && echo "$f $(cat $f)" - done > hairpin-modes.txt 2>&1 || true - sudo iptables-save > iptables.txt 2>&1 || true - sudo nft list ruleset > nft.txt 2>&1 || true - sudo dmesg --ctime > dmesg.txt 2>&1 || true - sudo virsh list --all > virsh-list.txt 2>&1 || true - ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true - tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag - REMOTE - scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \ - tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true - - - name: Upload Diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: e2e-diag-pseudo-${{ github.run_id }} - path: ${{ runner.temp }}/diag-pseudo/ - if-no-files-found: warn - retention-days: 14 - - - name: Teardown - if: always() - working-directory: mulga/scripts/tofu-cluster - run: | - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true e2e_real_multi: name: Real Multi-Node E2E + if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') runs-on: [ self-hosted, ci-multi ] - timeout-minutes: 30 + timeout-minutes: 60 env: TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log + ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }} steps: + - name: Reap Orphan Procs + run: | + set +e + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 3 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + true + - name: Clean Workspace run: sudo rm -rf "$GITHUB_WORKSPACE"/* @@ -341,6 +399,25 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Re-enable Go toolchain auto-download + run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" + + - name: Prune rootless docker cache + run: | + docker container prune -f || true + docker image prune -af || true + docker buildx prune -af --filter "until=24h" || true + docker system df || true + - name: Build Install Artifacts working-directory: mulga/scripts/tofu-cluster run: | @@ -352,12 +429,15 @@ jobs: - name: Provision VMs working-directory: mulga/scripts/tofu-cluster run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" export TF_VAR_git_branch="${GIT_BRANCH}" - tofu init -input=false >> "$TOFU_LOG" 2>&1 - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "3-node cluster provisioned" - name: Bootstrap via Binary Install working-directory: mulga/scripts/tofu-cluster @@ -368,89 +448,253 @@ jobs: "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - - name: Run E2E Tests + - name: Resolve cluster SSH details + id: cluster working-directory: mulga/scripts/tofu-cluster run: | INVENTORY=$(tofu output -json inventory) NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" - - # Build node IP list - NODE_IPS="" NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + NODES_CSV="" for i in $(seq 0 $((NODE_COUNT - 1))); do - NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")" + ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") + if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi done + echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT" + echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" + echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT" - NODE2_IP=$(echo "$INVENTORY" | jq -r '.nodes[1].wan_ip') - - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ - "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}" - - - name: Collect Diagnostics - if: always() - working-directory: mulga/scripts/tofu-cluster + - name: Build E2E test binaries on runner + working-directory: spinifex/tests/e2e run: | - INVENTORY=$(tofu output -json inventory) - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" - DIAG_DIR="${RUNNER_TEMP}/diag-multi" - mkdir -p "$DIAG_DIR" - cat > "${RUNNER_TEMP}/collect-diag.sh" <<'REMOTE' - set -x - DIAG=/tmp/spinifex-diag - sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" - sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true - sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true - sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true - sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true - sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true - sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true - sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true - sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true - sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true - sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true - sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true - sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true - sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true - sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true - sudo ip -d link > ip-link.txt 2>&1 || true - sudo ip addr > ip-addr.txt 2>&1 || true - sudo ip route > ip-route.txt 2>&1 || true - sudo ip neigh > ip-neigh.txt 2>&1 || true - sudo bridge fdb show > bridge-fdb.txt 2>&1 || true - sudo bridge link > bridge-link.txt 2>&1 || true - for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do - [ -e "$f" ] && echo "$f $(cat $f)" - done > hairpin-modes.txt 2>&1 || true - sudo iptables-save > iptables.txt 2>&1 || true - sudo nft list ruleset > nft.txt 2>&1 || true - sudo dmesg --ctime > dmesg.txt 2>&1 || true - sudo virsh list --all > virsh-list.txt 2>&1 || true - ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true - tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag - REMOTE - NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') - for i in $(seq 0 $((NODE_COUNT - 1))); do - NODE_IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") - NODE_DIR="$DIAG_DIR/node${i}-${NODE_IP}" - mkdir -p "$NODE_DIR" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${NODE_IP} 'bash -s' < "${RUNNER_TEMP}/collect-diag.sh" > "$NODE_DIR/collect.log" 2>&1 || true - scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \ - tf-user@${NODE_IP}:/tmp/spinifex-diag.tar.gz "$NODE_DIR/spinifex-diag.tar.gz" || true + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ') + if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then + echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build" + exit 0 + fi + banner "build ${SUITES_BUILD// /, }" + group_open "go test -c" + make e2e-build SUITES_BUILD="${SUITES_BUILD}" + group_close + for f in _bin/*.test; do + ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" done - - name: Upload Diagnostics + - name: Install go-junit-report on runner + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Ship test binaries to node1 + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + VM=tf-user@${{ steps.cluster.outputs.node1_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" + tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" + ok "shipped binaries to $VM" + + - name: Start packet capture on all nodes br-wan + run: | + set +e + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + for ip in "${NODE_LIST[@]}"; do + [ -z "$ip" ] && continue + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \ + tf-user@"$ip":/tmp/start-net-capture.sh + ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true + done + + - name: Pre-flight diagnostics on node1 + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" 2>/dev/null || true + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + VM=tf-user@${{ steps.cluster.outputs.node1_ip }} + mkdir -p "${ARTIFACT_DIR}" + DIAG="${ARTIFACT_DIR}/preflight-diag.log" + { + echo "=== id (tf-user groups) ===" + ssh "${SSH_OPTS[@]}" "$VM" 'id; echo; getent group spinifex || true' + echo + echo "=== ls -la /etc/spinifex/ ===" + ssh "${SSH_OPTS[@]}" "$VM" 'ls -la /etc/spinifex/ 2>&1; echo; ls -la /etc/spinifex/nats/ 2>&1' + echo + echo "=== spinifex.toml NATS section ===" + ssh "${SSH_OPTS[@]}" "$VM" 'grep -A4 "\[nodes\\..*\\.nats" /etc/spinifex/spinifex.toml 2>&1 | head -40' + echo + echo "=== systemctl is-active spinifex-* ===" + ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done' + echo + echo "=== which spx + version ===" + ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true' + echo + echo "=== spx get nodes (direct ssh, tf-user) ===" + ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"' + echo + echo "=== spx top nodes (direct ssh, tf-user) ===" + ssh "${SSH_OPTS[@]}" "$VM" 'spx top nodes 2>&1 | head -30; echo "exit=$?"' + echo + echo "=== NATS routez (curl 127.0.0.1:8222) ===" + ssh "${SSH_OPTS[@]}" "$VM" 'curl -fsS http://127.0.0.1:8222/routez 2>&1 | head -40' + echo + echo "=== aws describe-instance-types via gateway (tf-user) ===" + ssh "${SSH_OPTS[@]}" "$VM" 'AWS_PROFILE=spinifex AWS_CA_BUNDLE=/etc/spinifex/ca.pem aws --endpoint-url https://${{ steps.cluster.outputs.node1_ip }}:9999 ec2 describe-instance-types --query "InstanceTypes[*].InstanceType" --output text 2>&1 | head -5' + } 2>&1 | tee "$DIAG" + true + + - name: Run E2E suites + id: e2e + run: | + set +e + SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ') + if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then + echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping" + echo "rc=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" + else + TEST_RUN_FLAG="" + fi + VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}" + SSH_KEY="${{ steps.cluster.outputs.ssh_key }}" + NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}" + cleanup_remote() { + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + } + trap cleanup_remote EXIT INT TERM + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \ + "$VM_HOST" \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; \ + cd \$HOME/spinifex-e2e-bin; \ + R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ + FAIL=0; \ + for SUITE in ${SUITES_RUN}; do \ + echo \"::group::suite \$SUITE\"; \ + printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \ + SPINIFEX_NODES='${NODES_CSV}' \ + SPINIFEX_NODE_IPS='${NODES_CSV}' \ + SPINIFEX_SSH_USER=tf-user \ + SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ + 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ + | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ + rc=\${PIPESTATUS[0]}; \ + echo \"::endgroup::\"; \ + if [ \$rc -ne 0 ]; then \ + printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \ + echo \"::error::suite \$SUITE failed (exit \$rc)\"; \ + echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \ + FAIL=1; \ + else \ + printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \ + fi; \ + done; \ + exit \$FAIL" + E2E_RC=$? + echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Capture network/IPsec diagnostics on all nodes if: always() - uses: actions/upload-artifact@v4 + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + NODE1_IP="${NODE_LIST[0]}" + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + if [ "$idx" -ne 1 ]; then + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$ip":/tmp/capture-net-diag.sh + fi + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'" + if [ "$idx" -ne 1 ]; then + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \ + | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \ + "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null" + fi + ok "net diagnostics node${idx} captured" + done + + - name: Collect artifacts from node1 + if: always() + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + VM=tf-user@${{ steps.cluster.outputs.node1_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + ok "artifacts pulled to ${ARTIFACT_DIR}" + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null + ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines" + done + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null + done + + - name: Convert test logs to JUnit XML + if: always() + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log) + SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Publish test summary + if: always() + uses: ./spinifex/.github/actions/junit-report with: - name: e2e-diag-multi-${{ github.run_id }} - path: ${{ runner.temp }}/diag-multi/ + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + title: "Multi-Node E2E results" + + - name: Analyze e2e failures + if: always() + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "Multi-Node E2E failure analysis" + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-multi-artifacts-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} if-no-files-found: warn retention-days: 14 @@ -458,31 +702,332 @@ jobs: if: always() working-directory: mulga/scripts/tofu-cluster run: | - tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 - tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + for ip in "${NODE_LIST[@]}"; do + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + -i "${{ steps.cluster.outputs.ssh_key }}" \ + tf-user@"$ip" \ + 'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \ + 2>/dev/null || true + done + fi + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 2 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" + ok "teardown complete" + + - name: Fail job if any suite failed + if: always() + run: | + if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then + echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})" + cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true + exit 1 + fi + + e2e_reboot: + name: Reboot Resilience E2E + # Opt-in: only fires when the SUITES dispatch input lists e2e-reboot. + # The reboot test reboots the spinifex VM, so it must run on the runner + # (not shipped to the VM like the cert/lb/single suites). Runs in + # parallel with e2e_single — both target [self-hosted, ci-single] + # and GitHub Actions serialises on the underlying runner allocation, + # so no explicit needs: ordering is required. + if: contains(github.event.inputs.suites, 'e2e-reboot') + runs-on: [ self-hosted, ci-single ] + timeout-minutes: 60 + env: + TOFU_LOG: /tmp/e2e-tofu-reboot-${{ github.run_id }}.log + ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }} + steps: + - name: Reap Orphan Procs + run: | + set +e + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 3 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + true + + - name: Clean Workspace + run: sudo rm -rf "$GITHUB_WORKSPACE"/* + + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - uses: actions/checkout@v6 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/tofu-cluster + path: mulga + + - uses: actions/checkout@v6 + with: + path: spinifex + + - uses: actions/setup-go@v6 + with: + go-version-file: spinifex/go.mod + cache: false + + - name: Re-enable Go toolchain auto-download + run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" + + - name: Prune rootless docker cache + run: | + docker container prune -f || true + docker image prune -af || true + docker buildx prune -af --filter "until=24h" || true + docker system df || true + + - name: Build Install Artifacts + working-directory: mulga/scripts/tofu-cluster + run: | + ./build-install-artifacts.sh "${GIT_BRANCH}" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Provision VM + working-directory: mulga/scripts/tofu-cluster + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + export TF_VAR_git_branch="${GIT_BRANCH}" + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "VM provisioned" + + - name: Bootstrap via Binary Install + working-directory: mulga/scripts/tofu-cluster + run: | + SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \ + ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + + - name: Resolve VM SSH details + id: vm + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT" + echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT" + + - name: Build reboot.test on runner + working-directory: spinifex/tests/e2e + run: | + set -euo pipefail + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + banner "build reboot" + group_open "go test -c" + make e2e-build SUITES_BUILD="reboot" + group_close + ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)" + + - name: Fetch AWS credentials from VM + id: creds + run: | + set -euo pipefail + SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials') + AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}') + AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}') + if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then + echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}" + exit 1 + fi + echo "::add-mask::${AWS_AK}" + echo "::add-mask::${AWS_SK}" + echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT" + echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT" + + - name: Run reboot suite on runner + id: e2e + run: | + set +e + mkdir -p "${ARTIFACT_DIR}" + date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start" + R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m' + printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R" + + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" + else + TEST_RUN_FLAG="" + fi + GITHUB_ACTIONS=true FORCE_COLOR=1 \ + SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \ + SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \ + SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \ + SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \ + SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \ + ARTIFACT_DIR="${ARTIFACT_DIR}" \ + ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ + 2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \ + | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//' + rc=${PIPESTATUS[0]} + echo "rc=${rc}" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Collect VM journals + if: always() + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + mkdir -p "${ARTIFACT_DIR}" + SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true + ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" + + - name: Install go-junit-report + if: always() + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Convert test log to JUnit XML + if: always() + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log) + SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Publish test summary + if: always() + uses: ./spinifex/.github/actions/junit-report + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + title: "E2E reboot results" + + - name: Analyze e2e failures + if: always() + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "Reboot E2E failure analysis" + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: e2e-reboot-artifacts-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: warn + retention-days: 14 + + - name: Teardown + if: always() + working-directory: mulga/scripts/tofu-cluster + run: | + set +e + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + UID_SELF=$(id -u) + WS="${GITHUB_WORKSPACE}" + reap() { + local sig="$1" + for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do + cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + case "$cwd" in + "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; + esac + done + } + reap TERM + sleep 2 + reap KILL + find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" + ok "teardown complete" + + - name: Fail job if reboot suite failed + if: always() + run: | + if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then + echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})" + exit 1 + fi e2e_status: name: Report E2E Status - if: always() && inputs.sha - needs: [ e2e_single, e2e_pseudo_multi, e2e_real_multi ] + if: always() + needs: [ e2e_single, e2e_real_multi, e2e_reboot ] runs-on: [ self-hosted, ci-single ] timeout-minutes: 5 steps: - name: Report Status to PR env: GH_TOKEN: ${{ github.token }} + REPORT_SHA: ${{ inputs.sha || github.sha }} run: | - if [[ "${{ needs.e2e_single.result }}" == "success" && \ - "${{ needs.e2e_pseudo_multi.result }}" == "success" && \ - "${{ needs.e2e_real_multi.result }}" == "success" ]]; then + if [[ -z "${REPORT_SHA}" ]]; then + echo "::error::no SHA available (inputs.sha and github.sha both empty)" + exit 1 + fi + single_ok=true; multi_ok=true; reboot_ok=true + case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac + case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac + case "${{ needs.e2e_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac + if $single_ok && $multi_ok && $reboot_ok; then STATE="success" - DESC="All E2E suites passed" + DESC="E2E suites passed" else STATE="failure" - DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}" + DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }} reboot=${{ needs.e2e_reboot.result }}" fi - gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \ + gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \ -f state="$STATE" \ -f context=e2e \ -f description="$DESC" \ diff --git a/scripts/create-multi-node.sh b/scripts/create-multi-node.sh deleted file mode 100755 index c1ae2ca3a..000000000 --- a/scripts/create-multi-node.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash - -# Simulated multi-node cluster on a single machine using loopback addresses. -# Uses ~/node{1,2,3}/ for separate data directories. -# Usage: ./scripts/create-multi-node.sh - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -cd "$PROJECT_ROOT" - -echo "Multi-node simulation environment" -echo "=================================" - -# Build first -make build - -echo "" -echo "Removing old node directories..." -rm -rf ~/node1/ ~/node2/ ~/node3/ - -echo "" -echo "Step 1: Cluster Formation" -echo "=========================" -echo "Starting init node (127.0.0.1) and joining nodes (127.0.0.2, 127.0.0.3)..." - -# Init blocks until all nodes join, so run in background -./bin/spx admin init \ - --node node1 \ - --nodes 3 \ - --bind 127.0.0.1 \ - --cluster-bind 127.0.0.1 \ - --port 4432 \ - --spinifex-dir ~/node1/ \ - --config-dir ~/node1/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a & -INIT_PID=$! - -# Wait for formation server to start -echo "Waiting for formation server..." -for i in $(seq 1 30); do - if curl -sk "https://127.0.0.1:4432/formation/health" > /dev/null 2>&1; then - echo "Formation server ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "ERROR: Formation server failed to start" - kill $INIT_PID 2>/dev/null || true - exit 1 - fi - sleep 1 -done - -# Read join token -JOIN_TOKEN=$(cat ~/node1/config/join-token) -if [ -z "$JOIN_TOKEN" ]; then - echo "ERROR: Join token file is empty" - exit 1 -fi - -# Join nodes concurrently -./bin/spx admin join \ - --node node2 \ - --bind 127.0.0.2 \ - --cluster-bind 127.0.0.2 \ - --host 127.0.0.1:4432 \ - --token "$JOIN_TOKEN" \ - --data-dir ~/node2/ \ - --config-dir ~/node2/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a & -JOIN2_PID=$! - -./bin/spx admin join \ - --node node3 \ - --bind 127.0.0.3 \ - --cluster-bind 127.0.0.3 \ - --host 127.0.0.1:4432 \ - --token "$JOIN_TOKEN" \ - --data-dir ~/node3/ \ - --config-dir ~/node3/config/ \ - --region ap-southeast-2 \ - --az ap-southeast-2a & -JOIN3_PID=$! - -# Wait for all formation processes to complete -wait $INIT_PID $JOIN2_PID $JOIN3_PID -echo "" -echo "Cluster formation complete!" - -echo "" -echo "Step 2: Starting Services" -echo "=========================" - -# Start node 1 first (Predastore needs quorum) -UI=false ./scripts/start-dev.sh ~/node1/ & -sleep 5 - -# Start nodes 2 and 3 -UI=false ./scripts/start-dev.sh ~/node2/ & -sleep 2 -UI=false ./scripts/start-dev.sh ~/node3/ & - -# Wait for all start scripts to finish -wait - -echo "" -echo "Step 3: Verification" -echo "====================" -sleep 5 - -echo "Daemon health:" -for IP in 127.0.0.1 127.0.0.2 127.0.0.3; do - echo " $IP: $(curl -sk https://$IP:4432/health | head -c 100)" -done - -echo "" -echo "Multi-node cluster is running!" -echo " Stop with: ./scripts/stop-dev.sh" -echo " Logs: ~/node{1,2,3}/logs/" diff --git a/scripts/start-multi-node.sh b/scripts/start-multi-node.sh deleted file mode 100755 index 8f1c19df4..000000000 --- a/scripts/start-multi-node.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Start node1 -./scripts/start-dev.sh ~/node1/ - -# Start node2 -./scripts/start-dev.sh ~/node2/ - -# Start node3 -./scripts/start-dev.sh ~/node3/ diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh deleted file mode 100755 index 4cfc601e1..000000000 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ /dev/null @@ -1,1382 +0,0 @@ -#!/bin/bash -set -e - -# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc) -export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" - -# Pseudo Multi-Node E2E test runner -# This script sets up a 3-node Spinifex cluster using simulated IPs on the loopback interface -# and runs distributed instance tests on a single VM. - -# Ensure we are in the project root -cd "$(dirname "$0")/../.." - -# Source helper functions -source ./tests/e2e/lib/multinode-helpers.sh - -# Cleanup function - ensure resources are cleaned up on exit -cleanup() { - local exit_code=$? - - echo "" - echo "Cleanup triggered (exit code: $exit_code)..." - - if [ $exit_code -ne 0 ]; then - dump_all_node_logs - fi - - # Kill any lingering formation background processes - [ -n "$LEADER_INIT_PID" ] && kill "$LEADER_INIT_PID" 2>/dev/null || true - [ -n "$JOIN2_PID" ] && kill "$JOIN2_PID" 2>/dev/null || true - [ -n "$JOIN3_PID" ] && kill "$JOIN3_PID" 2>/dev/null || true - - # Try coordinated shutdown first (only if NATS is likely still up) - if [ "$CLUSTER_SERVICES_STARTED" = "true" ]; then - echo "Attempting coordinated cluster shutdown..." - if timeout 60 ./bin/spx admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/spinifex.toml" 2>/dev/null; then - echo "Coordinated shutdown succeeded" - else - echo "Coordinated shutdown failed, falling back to per-node stop..." - stop_all_nodes || true - fi - else - stop_all_nodes || true - fi - - # Force-kill anything that survived and clean up stale locks - force_cleanup_all_nodes || true - - # Remove simulated IPs - remove_simulated_ips || true - - echo "Cleanup complete" -} -trap cleanup EXIT - -# PIDs for background formation processes (used in cleanup) -JOIN2_PID="" -JOIN3_PID="" - -# Track whether cluster services have been started (for cleanup trap) -CLUSTER_SERVICES_STARTED="false" - -# Use Spinifex profile for AWS CLI -export AWS_PROFILE=spinifex -# Trust Spinifex CA for all profiles (AWS CLI v2 bundles its own Python/certifi, ignores system CA store) -export AWS_CA_BUNDLE="$HOME/node1/config/ca.pem" - - -echo "========================================" -echo "Multi-Node E2E Test Suite" -echo "========================================" -echo "" - -# Phase 1: Environment Setup -echo "Phase 1: Environment Setup" -echo "========================================" - -# Check for KVM support -echo "Checking for KVM support..." -if [ -e /dev/kvm ]; then - echo " /dev/kvm exists" - if [ -w /dev/kvm ]; then - echo " /dev/kvm is writable" - else - echo " ERROR: /dev/kvm is NOT writable" - exit 1 - fi -else - echo " ERROR: /dev/kvm does NOT exist" - exit 1 -fi - -# Check for ip command (iproute2) -if ! command -v ip &> /dev/null; then - echo " ERROR: 'ip' command not found. Install iproute2." - exit 1 -fi - - -# Setup simulated network -echo "" -echo "Setting up simulated network..." -add_simulated_ips - -echo "" - -# Phase 2: Cluster Initialization -echo "Phase 2: Cluster Initialization" -echo "========================================" - -# Background init — starts formation server, generates certs first -echo "" -init_leader_node - -# Trust CA cert (exists before formation completes — cert generation is the first step) -echo "" -echo "Adding Spinifex CA certificate to system trust store..." -sudo cp ~/node1/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt -sudo update-ca-certificates - -# Background BOTH joins — they poll /formation/status until all 3 nodes have joined. -# Must be concurrent: each join blocks until formation is complete. -echo "" -echo "Joining follower nodes concurrently..." -join_follower_node 2 & -JOIN2_PID=$! -join_follower_node 3 & -JOIN3_PID=$! - -# Wait for formation to complete (all processes generate their configs) -echo "Waiting for cluster formation to complete..." -wait $JOIN2_PID || { echo "ERROR: Node 2 join failed"; exit 1; } -wait $JOIN3_PID || { echo "ERROR: Node 3 join failed"; exit 1; } -wait $LEADER_INIT_PID || { echo "ERROR: Leader init failed"; exit 1; } -echo "Cluster formation complete — all configs generated" - -# Disable monitoring on non-primary nodes to avoid port conflict on 127.0.0.1:8222 -for i in 2 3; do - sed -i '/^http:\s*127\.0\.0\.1:8222/d' "$HOME/node${i}/config/nats/nats.conf" - echo " Stripped NATS monitoring from node$i (avoids 127.0.0.1:8222 conflict)" -done - -# Now start services (configs exist for all nodes) -echo "" -echo "Starting node services..." -start_node_services 1 "$HOME/node1" -start_node_services 2 "$HOME/node2" -start_node_services 3 "$HOME/node3" -CLUSTER_SERVICES_STARTED="true" - -# Wait for all services to stabilize -echo "" -echo "Waiting for cluster to stabilize..." -sleep 5 - -# Phase 3: Cluster Health Verification -echo "" -echo "Phase 3: Cluster Health Verification" -echo "========================================" - -# Verify NATS cluster -echo "" -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed, continuing anyway..." -} - -# Verify Predastore cluster -echo "" -verify_predastore_cluster 3 || { - echo "ERROR: Predastore cluster verification failed" - dump_all_node_logs - exit 1 -} - -# Wait for gateway on node1 (primary gateway) -echo "" -wait_for_gateway "${NODE1_IP}" 15 - -# Wait for daemon NATS subscriptions to be active -wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" - -# Define AWS CLI args pointing to node1's gateway -AWS_EC2="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} ec2" -AWS_IAM="aws --endpoint-url https://${NODE1_IP}:${AWSGW_PORT} iam" - -# Discover the cluster's availability zone and region dynamically -SPINIFEX_AZ=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text) -SPINIFEX_REGION=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].RegionName' --output text) -echo "Discovered AZ: $SPINIFEX_AZ, Region: $SPINIFEX_REGION" - -# Phase 3b: Cluster Stats CLI (Multi-Node) -echo "" -echo "Phase 3b: Cluster Stats CLI (Multi-Node)" -echo "========================================" - -# Test spx get nodes — should show all 3 nodes as Ready -echo "Testing spx get nodes..." -GET_NODES_OUTPUT=$(./bin/spx get nodes --config "$HOME/node1/config/spinifex.toml" --timeout 5s 2>/dev/null) -echo "$GET_NODES_OUTPUT" -READY_COUNT=$(echo "$GET_NODES_OUTPUT" | grep -c "Ready" || true) -if [ "$READY_COUNT" -lt 3 ]; then - echo "WARNING: spx get nodes shows $READY_COUNT Ready nodes (expected 3)" -fi -echo "spx get nodes passed ($READY_COUNT Ready nodes)" - -# Test spx top nodes — should show resource stats for all nodes -echo "Testing spx top nodes..." -TOP_NODES_OUTPUT=$(./bin/spx top nodes --config "$HOME/node1/config/spinifex.toml" --timeout 5s 2>/dev/null) -echo "$TOP_NODES_OUTPUT" -if ! echo "$TOP_NODES_OUTPUT" | grep -q "INSTANCE TYPE"; then - echo "WARNING: spx top nodes did not show instance type capacity table" -fi -echo "spx top nodes passed" - -# Test spx get vms — should show no VMs yet -echo "Testing spx get vms (empty)..." -GET_VMS_OUTPUT=$(./bin/spx get vms --config "$HOME/node1/config/spinifex.toml" --timeout 5s 2>/dev/null) -echo "$GET_VMS_OUTPUT" -echo "spx get vms (empty) passed" - -# Verify gateway responds -echo "" -echo "Testing gateway connectivity..." -$AWS_EC2 describe-regions | jq -e '.Regions | length > 0' || { - echo "ERROR: Gateway not responding correctly" - exit 1 -} -echo " Gateway is responding" - -# Phase 4: Image and Key Setup -echo "" -echo "Phase 4: Image and Key Setup" -echo "========================================" - -# Discover instance types -echo "Discovering available instance types..." -AVAILABLE_TYPES=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text) -echo " Available: $AVAILABLE_TYPES" - -# Pick nano instance type -INSTANCE_TYPE=$(echo $AVAILABLE_TYPES | tr ' ' '\n' | grep -m1 'nano') -if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" == "None" ]; then - echo "ERROR: No instance types found" - exit 1 -fi -echo " Selected: $INSTANCE_TYPE" - -# Get architecture -ARCH=$($AWS_EC2 describe-instance-types --instance-types "$INSTANCE_TYPE" \ - --query 'InstanceTypes[0].ProcessorInfo.SupportedArchitectures[0]' --output text) -echo " Architecture: $ARCH" - -# Create test key -echo "" -echo "Creating test key pair..." -KEY_MATERIAL=$($AWS_EC2 create-key-pair --key-name multinode-test-key --query 'KeyMaterial' --output text) -echo "$KEY_MATERIAL" > multinode-test-key.pem -chmod 600 multinode-test-key.pem -echo " Key created: multinode-test-key" - -# Default SG is egress-only on this branch; create an explicit SG that allows -# SSH + ICMP so subsequent run-instances calls produce reachable VMs. -echo "" -echo "Creating test security group..." -DEFAULT_VPC_ID=$($AWS_EC2 describe-vpcs --query 'Vpcs[?IsDefault==`true`].VpcId | [0]' --output text) -MULTINODE_SG=$($AWS_EC2 create-security-group \ - --group-name multinode-test-sg \ - --description "Pseudo multi-node e2e (SSH + ICMP ingress)" \ - --vpc-id "$DEFAULT_VPC_ID" \ - --query 'GroupId' --output text) -$AWS_EC2 authorize-security-group-ingress \ - --group-id "$MULTINODE_SG" --protocol tcp --port 22 --cidr 0.0.0.0/0 > /dev/null -$AWS_EC2 authorize-security-group-ingress \ - --group-id "$MULTINODE_SG" --protocol icmp --port -1 --cidr 0.0.0.0/0 > /dev/null -echo " SG created: $MULTINODE_SG (tcp/22 + icmp from 0.0.0.0/0)" - -# Import Ubuntu image (use node1's config and spinifex-dir). v6+ gold image -# stages ubuntu-26.04 (resolute); v3 staged ubuntu-24.04 (noble). Detect -# which is present so we can keep running both during the v3→v6 transition. -echo "" -echo "Importing Ubuntu image..." -if [ -f "$HOME/images/ubuntu-26.04.img" ]; then - UBUNTU_IMG="$HOME/images/ubuntu-26.04.img" - UBUNTU_VER="26.04" -elif [ -f "$HOME/images/ubuntu-24.04.img" ]; then - UBUNTU_IMG="$HOME/images/ubuntu-24.04.img" - UBUNTU_VER="24.04" -else - echo "ERROR: no ubuntu cloud image at ~/images/ubuntu-{24,26}.04.img" >&2 - exit 1 -fi -echo " Using: $UBUNTU_IMG (version $UBUNTU_VER)" -IMPORT_LOG=$(./bin/spx admin images import \ - --file "$UBUNTU_IMG" \ - --arch "$ARCH" \ - --distro ubuntu \ - --version "$UBUNTU_VER" \ - --boot-mode uefi \ - --config "$HOME/node1/config/spinifex.toml" \ - --spinifex-dir "$HOME/node1/" \ - --force 2>/dev/null) -echo "Import output: $IMPORT_LOG" -AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+') - -if [ -z "$AMI_ID" ]; then - echo "ERROR: Failed to capture AMI ID" - exit 1 -fi -echo " AMI ID: $AMI_ID" - -# Verify AMI -$AWS_EC2 describe-images --image-ids "$AMI_ID" | jq -e ".Images[0] | select(.ImageId==\"$AMI_ID\")" > /dev/null -echo " AMI verified" - - -# Phase 5: Multi-Node Instance Tests -echo "" -echo "Phase 5: Multi-Node Instance Tests" -echo "========================================" - -# Test 1: Instance Distribution -echo "" -echo "Test 1: Instance Distribution" -echo "----------------------------------------" -echo "Launching 3 instances to test distribution across nodes..." - -INSTANCE_IDS=() -for i in 1 2 3; do - echo " Launching instance $i..." - RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG") - - INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') - if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then - echo " ERROR: Failed to launch instance $i" - echo " Output: $RUN_OUTPUT" - exit 1 - fi - echo " Launched: $INSTANCE_ID" - INSTANCE_IDS+=("$INSTANCE_ID") - - # Small delay between launches to encourage distribution - sleep 1 -done - -# Wait for all instances to be running -echo "" -echo "Waiting for instances to reach running state..." -for instance_id in "${INSTANCE_IDS[@]}"; do - wait_for_instance_state "$instance_id" "running" 30 || { - echo "ERROR: Instance $instance_id failed to start" - exit 1 - } -done - -# Check distribution — enforce all 3 nodes received an instance -echo "" -check_instance_distribution 3 || { - echo "ERROR: Default spread did not distribute instances across all 3 nodes" - exit 1 -} -echo " Default spread distribution verified" - -# Verify spx get vms shows all running instances -echo "" -echo "Verifying spx get vms (with running VMs)..." -GET_VMS_OUTPUT=$(./bin/spx get vms --config "$HOME/node1/config/spinifex.toml" --timeout 5s 2>/dev/null) -echo "$GET_VMS_OUTPUT" -for instance_id in "${INSTANCE_IDS[@]}"; do - if ! echo "$GET_VMS_OUTPUT" | grep -q "$instance_id"; then - echo "WARNING: spx get vms did not show instance $instance_id" - fi -done -echo "spx get vms shows launched instances" - -# Test 1a-ii: SSH Connectivity & Volume Verification -echo "" -echo "Test 1a-ii: SSH Connectivity & Volume Verification" -echo "----------------------------------------" -echo "Testing SSH into all 3 instances..." - -# Arrays to store SSH details for post-termination verification -SSH_PORTS=() -SSH_HOSTS=() - -for idx in "${!INSTANCE_IDS[@]}"; do - instance_id="${INSTANCE_IDS[$idx]}" - echo "" - echo " Instance $((idx + 1)): $instance_id" - - # Determine SSH connection method: - # - If instance has a public IP (external networking), SSH via public IP on port 22 - # - Otherwise, fall back to QEMU hostfwd (dev_networking mode) - INST_PUBLIC_IP=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || echo "None") - - if [ -n "$INST_PUBLIC_IP" ] && [ "$INST_PUBLIC_IP" != "None" ] && [ "$INST_PUBLIC_IP" != "null" ]; then - SSH_HOST="$INST_PUBLIC_IP" - SSH_PORT=22 - echo " SSH via public IP: $SSH_HOST:$SSH_PORT" - else - echo " No public IP — using QEMU hostfwd for SSH" - # set -e would abort on $() returning non-zero, hiding the diagnostic below - if ! SSH_PORT=$(get_ssh_port "$instance_id"); then - SSH_PORT="" - fi - if [ -z "$SSH_PORT" ]; then - echo " ERROR: Failed to get SSH port for instance $instance_id" - exit 1 - fi - SSH_HOST=$(get_ssh_host "$instance_id") - echo " SSH endpoint: $SSH_HOST:$SSH_PORT" - fi - - SSH_PORTS+=("$SSH_PORT") - SSH_HOSTS+=("$SSH_HOST") - - # Wait for SSH to become ready (VM boot + cloud-init) - wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 60 - - # Test basic SSH connectivity - test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" - - # Check root volume size via lsblk - echo " Verifying root volume size from inside the VM..." - ROOT_VOL_ID_SSH=$($AWS_EC2 describe-instances --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId' --output text) - ROOT_VOL_SIZE_API=$($AWS_EC2 describe-volumes --volume-ids "$ROOT_VOL_ID_SSH" \ - --query 'Volumes[0].Size' --output text) - # Find the disk backing the root filesystem (avoids picking up floppy/cdrom devices) - # 1. findmnt gets the source device for / (e.g. /dev/vda1) - # 2. lsblk PKNAME resolves to parent disk name (e.g. vda) - # 3. lsblk -b -d gets that disk's byte size - ROOT_DISK_BYTES=$(ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o ConnectTimeout=5 \ - -o BatchMode=yes \ - -p "$SSH_PORT" \ - -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'SRC=$(findmnt -n -o SOURCE /); PKN=$(lsblk -n -o PKNAME "$SRC" 2>/dev/null | head -1); DEV=${PKN:-$(basename "$SRC")}; lsblk -b -d -n -o SIZE "/dev/$DEV"' | tr -d '[:space:]') - if [ -z "$ROOT_DISK_BYTES" ] || [ "$ROOT_DISK_BYTES" = "0" ]; then - echo " ERROR: Failed to get root disk size from VM (got: '$ROOT_DISK_BYTES')" - echo " lsblk debug output:" - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ - -o ConnectTimeout=5 -o BatchMode=yes -p "$SSH_PORT" -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'lsblk -b -d; echo "---"; findmnt -n -o SOURCE /; cat /proc/partitions' || true - exit 1 - fi - ROOT_DISK_GIB=$((ROOT_DISK_BYTES / 1073741824)) - echo " Root disk size from VM: ${ROOT_DISK_GIB}GiB (API reports: ${ROOT_VOL_SIZE_API}GiB)" - if [ "$ROOT_DISK_GIB" -ne "$ROOT_VOL_SIZE_API" ]; then - echo " ERROR: Root volume size mismatch: VM reports ${ROOT_DISK_GIB}GiB, API reports ${ROOT_VOL_SIZE_API}GiB" - exit 1 - fi - echo " Root volume size verified" - - # Verify hostname contains instance ID - echo " Verifying hostname inside the VM..." - VM_HOSTNAME=$(ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=5 \ - -o BatchMode=yes \ - -p "$SSH_PORT" \ - -i "multinode-test-key.pem" \ - ec2-user@"$SSH_HOST" 'hostname' 2>/dev/null) - echo " VM hostname: $VM_HOSTNAME" - # Hostname uses truncated ID: spinifex-vm- - SHORT_ID=$(echo "$instance_id" | sed 's/^i-//' | cut -c1-8) - if echo "$VM_HOSTNAME" | grep -q "$SHORT_ID"; then - echo " Hostname contains instance ID prefix ($SHORT_ID)" - else - echo " WARNING: Hostname '$VM_HOSTNAME' does not contain instance ID prefix '$SHORT_ID' (non-fatal)" - fi -done - -echo "" -echo " SSH connectivity and volume verification passed for all instances" - -# Test 2: DescribeInstances Aggregation -echo "" -echo "Test 2: DescribeInstances Aggregation" -echo "----------------------------------------" -echo "Verifying all instances are returned via fan-out query..." - -DESCRIBE_OUTPUT=$($AWS_EC2 describe-instances --query 'Reservations[*].Instances[*].InstanceId' --output text) -DESCRIBED_COUNT=$(echo "$DESCRIBE_OUTPUT" | wc -w) - -echo " Launched: ${#INSTANCE_IDS[@]} instances" -echo " Described: $DESCRIBED_COUNT instances" - -if [ "$DESCRIBED_COUNT" -lt "${#INSTANCE_IDS[@]}" ]; then - echo "ERROR: DescribeInstances did not return all instances" - echo " Expected: ${#INSTANCE_IDS[@]}, Got: $DESCRIBED_COUNT" - exit 1 -fi -echo " Aggregation test passed" - -# Test 3: Cross-Node Operations -echo "" -echo "Test 3: Cross-Node Operations" -echo "----------------------------------------" -echo "Testing stop/start/terminate via gateway regardless of instance location..." - -# Pick first instance for cross-node operations -TEST_INSTANCE="${INSTANCE_IDS[0]}" -echo " Test instance: $TEST_INSTANCE" - -# Stop instance -echo " Stopping instance..." -$AWS_EC2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "stopped" 30 - -# Start instance -echo " Starting instance..." -$AWS_EC2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null -wait_for_instance_state "$TEST_INSTANCE" "running" 30 - -echo " Cross-node operations test passed" - -# Test 4: NATS Cluster Health (Post-Operations) -echo "" -echo "Test 4: NATS Cluster Health (Post-Operations)" -echo "----------------------------------------" -echo "Verifying NATS cluster is still healthy after operations..." - -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed after operations" -} - -# Phase 5b: Multi-Node Batch Distribution Tests -# Tests batch launches with --count N (placement groups phase 1 routing) -echo "" -echo "Phase 5b: Multi-Node Batch Distribution Tests" -echo "========================================" - -# Clean up instances from Phase 5 before batch tests -echo "Cleaning up Phase 5 instances..." -if ! terminate_and_wait "${INSTANCE_IDS[@]}"; then - echo "WARNING: Some Phase 5 instances failed to terminate cleanly" -fi -unset INSTANCE_IDS - -# Wait for clean state -sleep 2 - -# Test 5b-1: Batch launch spreads across nodes -echo "" -echo "Test 5b-1: Batch Launch Spread (3 instances on 3 nodes)" -echo "----------------------------------------" -echo "Launching 3 instances in a single API call..." - -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 3) - -BATCH_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$BATCH_COUNT" -ne 3 ]; then - echo "FAIL: Expected 3 instances, got $BATCH_COUNT" - echo "Output: $RUN_OUTPUT" - exit 1 -fi - -# Verify single reservation in response -RESERVATION_ID=$(echo "$RUN_OUTPUT" | jq -r '.ReservationId // empty') -if [ -z "$RESERVATION_ID" ]; then - echo "FAIL: No ReservationId in response" - exit 1 -fi -echo " ReservationId: $RESERVATION_ID" -echo " Instances: $BATCH_COUNT" - -BATCH_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -echo " Launched: ${BATCH_IDS[*]}" - -# Wait for all to reach running -for id in "${BATCH_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || { - echo "ERROR: Instance $id failed to reach running state" - exit 1 - } -done - -# Verify distribution: check which node hosts each instance via nbdkit process -echo " Checking distribution..." -NODES_USED=0 -for i in 1 2 3; do - NODE_IP="${SIMULATED_NETWORK}.$i" - NODE_COUNT=0 - for id in "${BATCH_IDS[@]}"; do - local_node=$(get_instance_node "$id") || continue - if [ "$local_node" = "$i" ]; then - NODE_COUNT=$((NODE_COUNT + 1)) - fi - done - echo " Node$i ($NODE_IP): $NODE_COUNT instances" - if [ "$NODE_COUNT" -gt 0 ]; then - NODES_USED=$((NODES_USED + 1)) - fi -done - -if [ "$NODES_USED" -lt 3 ]; then - echo "FAIL: Expected instances on 3 nodes, only used $NODES_USED" - exit 1 -fi -echo " PASS: Instances distributed across $NODES_USED nodes" - -# Verify DescribeInstances from each node sees all batch-launched instances -echo " Verifying DescribeInstances aggregation from all gateways..." -for i in 1 2 3; do - NODE_GW_IP="${SIMULATED_NETWORK}.$i" - DESCRIBE=$(aws --endpoint-url "https://${NODE_GW_IP}:${AWSGW_PORT}" \ - ec2 describe-instances \ - --query 'Reservations[*].Instances[?State.Name!=`terminated`].InstanceId' \ - --output text 2>/dev/null) || { - echo " WARNING: Could not query gateway on node$i" - continue - } - - MISSING=0 - for id in "${BATCH_IDS[@]}"; do - if ! echo "$DESCRIBE" | grep -q "$id"; then - echo " Node$i missing instance $id" - MISSING=$((MISSING + 1)) - fi - done - - if [ "$MISSING" -gt 0 ]; then - echo "FAIL: Node$i gateway missing $MISSING instances" - exit 1 - fi - echo " Node$i gateway: all ${#BATCH_IDS[@]} instances visible" -done -echo " PASS: All gateways see all batch-launched instances" - -# Cleanup -echo " Cleaning up..." -if ! terminate_and_wait "${BATCH_IDS[@]}"; then - echo "WARNING: Some batch instances failed to terminate" -fi -sleep 2 - -# Test 5b-2: Batch launch with overflow packing (5 on 3 nodes) -echo "" -echo "Test 5b-2: Batch Launch Overflow (5 instances on 3 nodes)" -echo "----------------------------------------" -echo "Launching 5 instances in a single API call..." - -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 5) - -BATCH_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$BATCH_COUNT" -ne 5 ]; then - echo "FAIL: Expected 5 instances, got $BATCH_COUNT" - exit 1 -fi - -BATCH_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -echo " Launched: ${BATCH_IDS[*]}" - -for id in "${BATCH_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || { - echo "ERROR: Instance $id failed to reach running state" - exit 1 - } -done - -# All 3 nodes should have at least 1 instance (spread), some get 2 (pack) -echo " Checking distribution..." -ALL_NODES_USED=true -for i in 1 2 3; do - NODE_IP="${SIMULATED_NETWORK}.$i" - NODE_COUNT=0 - for id in "${BATCH_IDS[@]}"; do - local_node=$(get_instance_node "$id") || continue - if [ "$local_node" = "$i" ]; then - NODE_COUNT=$((NODE_COUNT + 1)) - fi - done - echo " Node$i ($NODE_IP): $NODE_COUNT instances" - if [ "$NODE_COUNT" -eq 0 ]; then - ALL_NODES_USED=false - fi -done - -if [ "$ALL_NODES_USED" = false ]; then - echo "FAIL: Not all nodes received instances" - exit 1 -fi -echo " PASS: All nodes received at least 1 instance, 5 total distributed" - -# Cleanup -echo " Cleaning up..." -if ! terminate_and_wait "${BATCH_IDS[@]}"; then - echo "WARNING: Some batch instances failed to terminate" -fi -sleep 2 - -# Test 5b-3: Insufficient capacity error -echo "" -echo "Test 5b-3: Insufficient Capacity Error" -echo "----------------------------------------" -echo "Requesting more instances than cluster can provide..." - -# Request an absurdly high count that definitely exceeds capacity -if RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 100 2>&1); then - echo "FAIL: Expected error but got success" - echo "Output: $RUN_OUTPUT" - exit 1 -fi - -if echo "$RUN_OUTPUT" | grep -q "InsufficientInstanceCapacity"; then - echo " PASS: Got InsufficientInstanceCapacity as expected" -else - echo "FAIL: Wrong error: $RUN_OUTPUT" - exit 1 -fi - -# Verify no orphaned instances were left behind -ORPHAN_CHECK=$($AWS_EC2 describe-instances \ - --query 'Reservations[*].Instances[?State.Name!=`terminated`].InstanceId' --output text) -if [ -n "$ORPHAN_CHECK" ] && [ "$ORPHAN_CHECK" != "None" ]; then - echo " WARNING: Found non-terminated instances after capacity error: $ORPHAN_CHECK" -else - echo " PASS: No orphaned instances after capacity error" -fi - -# Test 5b-4: MinCount/MaxCount capping -echo "" -echo "Test 5b-4: MinCount/MaxCount Capping" -echo "----------------------------------------" -echo "Launching with --count 2..." - -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 2) - -BATCH_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$BATCH_COUNT" -ne 2 ]; then - echo "FAIL: Expected exactly 2 instances, got $BATCH_COUNT" - exit 1 -fi -echo " PASS: Launched exactly 2 instances (capped to MaxCount)" - -# Cleanup -BATCH_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -for id in "${BATCH_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || true -done -if ! terminate_and_wait "${BATCH_IDS[@]}"; then - echo "WARNING: Some batch instances failed to terminate" -fi -sleep 2 - -# Test 5b-5: MinCount failure threshold -echo "" -echo "Test 5b-5: MinCount Failure Threshold" -echo "----------------------------------------" -echo "Requesting --count 100 (exceeds cluster capacity)..." - -if RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 100 2>&1); then - echo "FAIL: Expected InsufficientInstanceCapacity" - exit 1 -fi - -if echo "$RUN_OUTPUT" | grep -q "InsufficientInstanceCapacity"; then - echo " PASS: Got InsufficientInstanceCapacity for MinCount > capacity" -else - echo "FAIL: Wrong error: $RUN_OUTPUT" - exit 1 -fi - -echo "" -echo "Phase 5b: All batch distribution tests passed" - -# Phase 5c: Placement Group E2E Tests -echo "" -echo "Phase 5c: Placement Group E2E Tests (CRUD + Spread + Cluster)" -echo "==============================================================" - -# Test 5c-1: Create spread group -echo "" -echo "Test 5c-1: CreatePlacementGroup (spread)" -echo "----------------------------------------" -CREATE_OUTPUT=$($AWS_EC2 create-placement-group \ - --group-name test-spread --strategy spread) -SPREAD_GROUP_ID=$(echo "$CREATE_OUTPUT" | jq -r '.PlacementGroup.GroupId') -SPREAD_STATE=$(echo "$CREATE_OUTPUT" | jq -r '.PlacementGroup.State') -if [ -z "$SPREAD_GROUP_ID" ] || [ "$SPREAD_GROUP_ID" = "null" ]; then - echo "FAIL: No GroupId returned"; exit 1 -fi -if [ "$SPREAD_STATE" != "available" ]; then - echo "FAIL: Expected state=available, got $SPREAD_STATE"; exit 1 -fi -echo " PASS: Created spread group $SPREAD_GROUP_ID (state=$SPREAD_STATE)" - -# Test 5c-2: Create cluster group -echo "" -echo "Test 5c-2: CreatePlacementGroup (cluster)" -echo "----------------------------------------" -CREATE_OUTPUT=$($AWS_EC2 create-placement-group \ - --group-name test-cluster --strategy cluster) -CLUSTER_GROUP_ID=$(echo "$CREATE_OUTPUT" | jq -r '.PlacementGroup.GroupId') -CLUSTER_STATE=$(echo "$CREATE_OUTPUT" | jq -r '.PlacementGroup.State') -if [ -z "$CLUSTER_GROUP_ID" ] || [ "$CLUSTER_GROUP_ID" = "null" ]; then - echo "FAIL: No GroupId returned"; exit 1 -fi -if [ "$CLUSTER_STATE" != "available" ]; then - echo "FAIL: Expected state=available, got $CLUSTER_STATE"; exit 1 -fi -echo " PASS: Created cluster group $CLUSTER_GROUP_ID (state=$CLUSTER_STATE)" - -# Test 5c-3: Partition strategy rejected -echo "" -echo "Test 5c-3: CreatePlacementGroup (partition rejected)" -echo "----------------------------------------" -if PARTITION_OUTPUT=$($AWS_EC2 create-placement-group \ - --group-name test-partition --strategy partition 2>&1); then - echo "FAIL: Expected error for partition strategy"; exit 1 -fi -if echo "$PARTITION_OUTPUT" | grep -q "InvalidParameterValue"; then - echo " PASS: partition strategy correctly rejected" -else - echo "FAIL: Wrong error: $PARTITION_OUTPUT"; exit 1 -fi - -# Test 5c-4: Duplicate name rejected -echo "" -echo "Test 5c-4: CreatePlacementGroup (duplicate name)" -echo "----------------------------------------" -if DUP_OUTPUT=$($AWS_EC2 create-placement-group \ - --group-name test-spread --strategy spread 2>&1); then - echo "FAIL: Expected error for duplicate name"; exit 1 -fi -if echo "$DUP_OUTPUT" | grep -q "InvalidPlacementGroup.Duplicate"; then - echo " PASS: Duplicate name correctly rejected" -else - echo "FAIL: Wrong error: $DUP_OUTPUT"; exit 1 -fi - -# Test 5c-5: DescribePlacementGroups -echo "" -echo "Test 5c-5: DescribePlacementGroups" -echo "----------------------------------------" -DESCRIBE_OUTPUT=$($AWS_EC2 describe-placement-groups) -PG_COUNT=$(echo "$DESCRIBE_OUTPUT" | jq '.PlacementGroups | length') -if [ "$PG_COUNT" -lt 2 ]; then - echo "FAIL: Expected at least 2 placement groups, got $PG_COUNT"; exit 1 -fi - -SPREAD_STRATEGY=$(echo "$DESCRIBE_OUTPUT" | jq -r '.PlacementGroups[] | select(.GroupName=="test-spread") | .Strategy') -SPREAD_LEVEL=$(echo "$DESCRIBE_OUTPUT" | jq -r '.PlacementGroups[] | select(.GroupName=="test-spread") | .SpreadLevel') -if [ "$SPREAD_STRATEGY" != "spread" ]; then - echo "FAIL: Spread group strategy=$SPREAD_STRATEGY"; exit 1 -fi -if [ "$SPREAD_LEVEL" != "host" ]; then - echo "FAIL: Spread group SpreadLevel=$SPREAD_LEVEL, expected host"; exit 1 -fi - -CLUSTER_STRATEGY=$(echo "$DESCRIBE_OUTPUT" | jq -r '.PlacementGroups[] | select(.GroupName=="test-cluster") | .Strategy') -if [ "$CLUSTER_STRATEGY" != "cluster" ]; then - echo "FAIL: Cluster group strategy=$CLUSTER_STRATEGY"; exit 1 -fi -echo " PASS: Both groups described with correct fields" - -FILTER_OUTPUT=$($AWS_EC2 describe-placement-groups --group-names test-spread) -FILTER_COUNT=$(echo "$FILTER_OUTPUT" | jq '.PlacementGroups | length') -if [ "$FILTER_COUNT" -ne 1 ]; then - echo "FAIL: Name filter returned $FILTER_COUNT groups, expected 1"; exit 1 -fi -echo " PASS: Name filter works correctly" - -# Test 5c-6: DescribeInstanceTypes PlacementGroupInfo -echo "" -echo "Test 5c-6: DescribeInstanceTypes PlacementGroupInfo" -echo "----------------------------------------" -DIT_OUTPUT=$($AWS_EC2 describe-instance-types --instance-types "$INSTANCE_TYPE") -STRATEGIES=$(echo "$DIT_OUTPUT" | jq -r '.InstanceTypes[0].PlacementGroupInfo.SupportedStrategies[]' 2>/dev/null | sort | tr '\n' ',') -if ! echo "$STRATEGIES" | grep -q "cluster" || ! echo "$STRATEGIES" | grep -q "spread"; then - echo "FAIL: PlacementGroupInfo.SupportedStrategies missing spread/cluster: $STRATEGIES"; exit 1 -fi -echo " PASS: $INSTANCE_TYPE supports strategies: $STRATEGIES" - -# Test 5c-7: Spread — 3 instances on 3 nodes (strict 1-per-node) -echo "" -echo "Test 5c-7: Spread — Launch 3 Instances (1 per node)" -echo "----------------------------------------" -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 3 \ - --placement GroupName=test-spread) - -SPREAD_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$SPREAD_COUNT" -ne 3 ]; then - echo "FAIL: Expected 3 instances, got $SPREAD_COUNT"; exit 1 -fi - -SPREAD_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -echo " Launched: ${SPREAD_IDS[*]}" - -for id in "${SPREAD_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || { - echo "ERROR: Instance $id failed to reach running state"; exit 1 - } -done - -echo " Checking strict 1-per-node distribution..." -count_instances_per_node "${SPREAD_IDS[@]}" -if [ "$NODE1_COUNT" -ne 1 ] || [ "$NODE2_COUNT" -ne 1 ] || [ "$NODE3_COUNT" -ne 1 ]; then - echo "FAIL: Spread group did not enforce strict 1-per-node"; exit 1 -fi -echo " PASS: Strict 1-per-node spread enforced" - -# Test 5c-8: DescribeInstances shows Placement field -echo "" -echo "Test 5c-8: Spread — DescribeInstances Placement Field" -echo "----------------------------------------" -verify_placement "test-spread" "${SPREAD_IDS[@]}" || exit 1 -echo " PASS: Placement field correctly populated" - -# Test 5c-9: 4th instance fails (all nodes occupied) -echo "" -echo "Test 5c-9: Spread — 4th Instance Fails (all nodes used)" -echo "----------------------------------------" -if FOURTH_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 1 \ - --placement GroupName=test-spread 2>&1); then - echo "FAIL: Expected InsufficientInstanceCapacity for 4th spread instance" - ACCIDENTAL_ID=$(echo "$FOURTH_OUTPUT" | jq -r '.Instances[0].InstanceId // empty') - [ -n "$ACCIDENTAL_ID" ] && $AWS_EC2 terminate-instances --instance-ids "$ACCIDENTAL_ID" > /dev/null - exit 1 -fi -if echo "$FOURTH_OUTPUT" | grep -q "InsufficientInstanceCapacity"; then - echo " PASS: 4th instance correctly rejected (all nodes occupied)" -else - echo "FAIL: Wrong error: $FOURTH_OUTPUT"; exit 1 -fi - -# Test 5c-10: Terminate frees node slot, replacement succeeds -echo "" -echo "Test 5c-10: Spread — Terminate Frees Node, Replacement Succeeds" -echo "----------------------------------------" -TERMINATED_ID="${SPREAD_IDS[0]}" -echo " Terminating $TERMINATED_ID to free a node slot..." -$AWS_EC2 terminate-instances --instance-ids "$TERMINATED_ID" > /dev/null -wait_for_instance_state "$TERMINATED_ID" "terminated" 30 || true -sleep 2 - -echo " Launching replacement instance..." -REPLACE_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 1 \ - --placement GroupName=test-spread) -REPLACE_ID=$(echo "$REPLACE_OUTPUT" | jq -r '.Instances[0].InstanceId') -if [ -z "$REPLACE_ID" ] || [ "$REPLACE_ID" = "null" ]; then - echo "FAIL: Replacement launch failed"; exit 1 -fi -wait_for_instance_state "$REPLACE_ID" "running" 60 || { - echo "ERROR: Replacement instance failed to reach running state"; exit 1 -} -echo " PASS: Replacement launched successfully after terminate freed slot" - -# Cleanup spread instances -echo " Cleaning up spread instances..." -SPREAD_CLEANUP_IDS=("${SPREAD_IDS[@]:1}" "$REPLACE_ID") -if ! terminate_and_wait "${SPREAD_CLEANUP_IDS[@]}"; then - echo "WARNING: Some spread instances failed to terminate" -fi -sleep 2 - -# Test 5c-11: Cluster — all instances on same node -echo "" -echo "Test 5c-11: Cluster — Launch 3 Instances (all on one node)" -echo "----------------------------------------" -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 3 \ - --placement GroupName=test-cluster) - -CLUSTER_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$CLUSTER_COUNT" -ne 3 ]; then - echo "FAIL: Expected 3 instances, got $CLUSTER_COUNT"; exit 1 -fi - -CLUSTER_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -echo " Launched: ${CLUSTER_IDS[*]}" - -for id in "${CLUSTER_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || { - echo "ERROR: Instance $id failed to reach running state"; exit 1 - } -done - -echo " Checking cluster pinning..." -count_instances_per_node "${CLUSTER_IDS[@]}" -NODES_WITH_INSTANCES=0 -PINNED_NODE="" -for i in 1 2 3; do - count_var="NODE${i}_COUNT" - if [ "${!count_var}" -gt 0 ]; then - NODES_WITH_INSTANCES=$((NODES_WITH_INSTANCES + 1)) - PINNED_NODE=$i - fi -done -if [ "$NODES_WITH_INSTANCES" -ne 1 ]; then - echo "FAIL: Cluster instances spread across $NODES_WITH_INSTANCES nodes (expected 1)"; exit 1 -fi -echo " PASS: All 3 instances pinned to Node$PINNED_NODE" - -# Test 5c-12: DescribeInstances shows Placement field -echo "" -echo "Test 5c-12: Cluster — DescribeInstances Placement Field" -echo "----------------------------------------" -verify_placement "test-cluster" "${CLUSTER_IDS[@]}" || exit 1 -echo " PASS: Placement field correctly populated" - -# Test 5c-13: Subsequent launch pins to same node -echo "" -echo "Test 5c-13: Cluster — Subsequent Launch Pins to Same Node" -echo "----------------------------------------" -echo " Launching 1 more instance into cluster group..." -EXTRA_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 1 \ - --placement GroupName=test-cluster) -EXTRA_ID=$(echo "$EXTRA_OUTPUT" | jq -r '.Instances[0].InstanceId') -if [ -z "$EXTRA_ID" ] || [ "$EXTRA_ID" = "null" ]; then - echo "FAIL: Subsequent launch failed"; exit 1 -fi -wait_for_instance_state "$EXTRA_ID" "running" 60 || { - echo "ERROR: Extra instance failed to reach running state"; exit 1 -} - -echo " Checking pinning..." -ALL_CLUSTER_IDS=("${CLUSTER_IDS[@]}" "$EXTRA_ID") -count_instances_per_node "${ALL_CLUSTER_IDS[@]}" -NODES_WITH_INSTANCES=0 -for i in 1 2 3; do - count_var="NODE${i}_COUNT" - if [ "${!count_var}" -gt 0 ]; then - NODES_WITH_INSTANCES=$((NODES_WITH_INSTANCES + 1)) - fi -done -if [ "$NODES_WITH_INSTANCES" -ne 1 ]; then - echo "FAIL: Extra instance not pinned to same node"; exit 1 -fi -echo " PASS: Subsequent launch pinned to same Node$PINNED_NODE" - -# Cleanup cluster instances -echo " Cleaning up cluster instances..." -if ! terminate_and_wait "${ALL_CLUSTER_IDS[@]}"; then - echo "WARNING: Some cluster instances failed to terminate" -fi -sleep 2 - -# Test 5c-14: Cluster capacity exhausted on pinned node -echo "" -echo "Test 5c-14: Cluster — Capacity Exhausted on Pinned Node" -echo "----------------------------------------" -FILL_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 1 \ - --placement GroupName=test-cluster) -FILL_ID=$(echo "$FILL_OUTPUT" | jq -r '.Instances[0].InstanceId') -wait_for_instance_state "$FILL_ID" "running" 60 || true - -if CAP_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 100 \ - --placement GroupName=test-cluster 2>&1); then - echo "FAIL: Expected InsufficientInstanceCapacity"; exit 1 -fi -if echo "$CAP_OUTPUT" | grep -q "InsufficientInstanceCapacity"; then - echo " PASS: Capacity exhaustion correctly detected on pinned node" -else - echo "FAIL: Wrong error: $CAP_OUTPUT"; exit 1 -fi - -if [ -n "$FILL_ID" ] && [ "$FILL_ID" != "null" ]; then - terminate_and_wait "$FILL_ID" || true -fi -sleep 2 - -# Test 5c-15: DeletePlacementGroup with running instances fails -echo "" -echo "Test 5c-15: DeletePlacementGroup with Instances (InUse)" -echo "----------------------------------------" -INUSE_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 1 \ - --placement GroupName=test-spread) -INUSE_ID=$(echo "$INUSE_OUTPUT" | jq -r '.Instances[0].InstanceId') -wait_for_instance_state "$INUSE_ID" "running" 60 || true - -if DEL_OUTPUT=$($AWS_EC2 delete-placement-group \ - --group-name test-spread 2>&1); then - echo "FAIL: Expected InvalidPlacementGroup.InUse"; exit 1 -fi -if echo "$DEL_OUTPUT" | grep -q "InvalidPlacementGroup.InUse"; then - echo " PASS: Delete correctly blocked while instances running" -else - echo "FAIL: Wrong error: $DEL_OUTPUT"; exit 1 -fi - -# Test 5c-16: Terminate all + delete succeeds -echo "" -echo "Test 5c-16: Terminate All + DeletePlacementGroup" -echo "----------------------------------------" -echo " Terminating instance $INUSE_ID..." -terminate_and_wait "$INUSE_ID" || true -sleep 2 - -$AWS_EC2 delete-placement-group --group-name test-spread -echo " PASS: Spread group deleted after terminating instances" - -$AWS_EC2 delete-placement-group --group-name test-cluster -echo " PASS: Cluster group deleted" - -REMAINING=$($AWS_EC2 describe-placement-groups \ - --query 'PlacementGroups[?GroupName==`test-spread` || GroupName==`test-cluster`] | length(@)' \ - --output text 2>/dev/null || echo "0") -if [ "$REMAINING" != "0" ] && [ "$REMAINING" != "None" ]; then - echo "FAIL: Groups still exist after deletion"; exit 1 -fi -echo " PASS: Both groups deleted and verified gone" - -# Test 5c-17: Default spread unchanged (regression) -echo "" -echo "Test 5c-17: Default Spread (No Placement Group) Unchanged" -echo "----------------------------------------" -RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG" \ - --count 3) -REG_COUNT=$(echo "$RUN_OUTPUT" | jq '.Instances | length') -if [ "$REG_COUNT" -ne 3 ]; then - echo "FAIL: Default spread launch failed, got $REG_COUNT instances"; exit 1 -fi -REG_IDS=($(echo "$RUN_OUTPUT" | jq -r '.Instances[].InstanceId')) -for id in "${REG_IDS[@]}"; do - wait_for_instance_state "$id" "running" 60 || true -done - -PLACEMENT=$($AWS_EC2 describe-instances --instance-ids "${REG_IDS[0]}" \ - --query 'Reservations[0].Instances[0].Placement.GroupName' --output text) -if [ -n "$PLACEMENT" ] && [ "$PLACEMENT" != "None" ] && [ "$PLACEMENT" != "null" ]; then - echo "FAIL: Default spread instance has Placement.GroupName='$PLACEMENT' (should be empty)"; exit 1 -fi -echo " PASS: Default spread works, no Placement.GroupName set" - -if ! terminate_and_wait "${REG_IDS[@]}"; then - echo "WARNING: Some regression test instances failed to terminate" -fi -sleep 2 - -echo "" -echo "Phase 5c: All placement group E2E tests passed" - -# Re-launch 3 instances for Phase 6 shutdown/restart tests -echo "" -echo "Launching 3 instances for Phase 6 shutdown/restart tests..." -INSTANCE_IDS=() -for i in 1 2 3; do - RUN_OUTPUT=$($AWS_EC2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name multinode-test-key \ - --security-group-ids "$MULTINODE_SG") - INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId') - if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then - echo " ERROR: Failed to launch instance $i for Phase 6 setup" - exit 1 - fi - echo " Launched: $INSTANCE_ID" - INSTANCE_IDS+=("$INSTANCE_ID") - sleep 1 -done -for id in "${INSTANCE_IDS[@]}"; do - wait_for_instance_state "$id" "running" 30 || { - echo "ERROR: Instance $id failed to start for Phase 6 setup" - exit 1 - } -done -echo " Phase 6 setup: ${#INSTANCE_IDS[@]} instances running" -echo "========================================" - -# Phase 6: Cluster Shutdown + Restart -echo "" -echo "Phase 6: Cluster Shutdown + Restart" -echo "========================================" -echo "Testing spx admin cluster shutdown command..." - -# Test 6a: Dry-run shutdown -echo "" -echo "Test 6a: Dry-Run Shutdown" -echo "----------------------------------------" -echo "Running cluster shutdown in dry-run mode..." - -DRY_RUN_OUTPUT=$(./bin/spx admin cluster shutdown --dry-run --config "$HOME/node1/config/spinifex.toml" 2>&1) -echo "$DRY_RUN_OUTPUT" - -# Validate dry-run output contains expected phases -for phase in GATE DRAIN STORAGE PERSIST INFRA; do - if echo "$DRY_RUN_OUTPUT" | grep -qi "$phase"; then - echo " Phase $phase found in shutdown plan" - else - echo " WARNING: Phase $phase not found in dry-run output" - fi -done -echo " Dry-run shutdown test passed" - -# Test 6b: Real coordinated shutdown -echo "" -echo "Test 6b: Coordinated Cluster Shutdown" -echo "----------------------------------------" -echo "Running cluster shutdown..." - -./bin/spx admin cluster shutdown --force --timeout 30s --config "$HOME/node1/config/spinifex.toml" 2>&1 || { - echo " WARNING: Cluster shutdown command returned non-zero exit code" -} -CLUSTER_SERVICES_STARTED="false" - -# Verify all services are down -echo " Waiting for services to stop..." -sleep 1 -if ! verify_all_services_down; then - echo " Some services still running, force-cleaning..." - force_cleanup_all_nodes -fi - -# Test 6c: Restart and recovery -echo "" -echo "Test 6c: Cluster Restart + Recovery" -echo "----------------------------------------" -echo "Restarting all node services concurrently..." - -# Cluster restart requires concurrent startup: NATS needs route peers to form, -# Predastore needs Raft quorum (2/3), and the daemon needs JetStream. -# Sequential start would leave node1 waiting for quorum that never arrives. -start_node_services 1 "$HOME/node1" & -start_node_services 2 "$HOME/node2" & -start_node_services 3 "$HOME/node3" & -wait -CLUSTER_SERVICES_STARTED="true" - -echo "" -echo "Waiting for cluster to stabilize..." -sleep 5 - -# Verify NATS cluster reformed -echo "" -verify_nats_cluster 3 || { - echo "WARNING: NATS cluster verification failed after restart" -} - -# Wait for gateway -echo "" -wait_for_gateway "${NODE1_IP}" 15 - -# Wait for daemon readiness -wait_for_daemon_ready "https://${NODE1_IP}:${AWSGW_PORT}" - -# Smoke test: describe-instance-types -echo "" -echo "Running post-restart smoke test..." -SMOKE_OUTPUT=$($AWS_EC2 describe-instance-types --query 'InstanceTypes[*].InstanceType' --output text 2>/dev/null) -if [ -n "$SMOKE_OUTPUT" ] && [ "$SMOKE_OUTPUT" != "None" ]; then - echo " Smoke test passed: describe-instance-types returned: $SMOKE_OUTPUT" -else - echo " ERROR: Smoke test failed: describe-instance-types returned empty/None" - exit 1 -fi - -echo " Cluster shutdown + restart test passed" - -# Test 6d: Restart instances after cluster restart, then terminate -echo "" -echo "Test 6d: Instance Restart + Terminate" -echo "----------------------------------------" - -# DRAIN persists Running VMs as Stopped and migrates them to the cluster-shared -# stopped KV bucket, so Restore does not auto-relaunch them. Verify the new -# contract: instances come back as stopped, then explicitly start them. -echo "Verifying instances are in stopped state after restart..." -for instance_id in "${INSTANCE_IDS[@]}"; do - wait_for_instance_state "$instance_id" "stopped" 30 || { - echo " ERROR: $instance_id did not surface as stopped after restart" - dump_all_node_logs - exit 1 - } -done - -echo "" -echo "Starting instances..." -$AWS_EC2 start-instances --instance-ids "${INSTANCE_IDS[@]}" > /dev/null || { - echo " ERROR: start-instances failed" - dump_all_node_logs - exit 1 -} -for instance_id in "${INSTANCE_IDS[@]}"; do - wait_for_instance_state "$instance_id" "running" 60 || { - echo " ERROR: $instance_id did not reach running after start-instances" - dump_all_node_logs - exit 1 - } -done - -# Terminate all instances -echo "" -echo "Terminating all instances..." -if ! terminate_and_wait "${INSTANCE_IDS[@]}"; then - echo "" - echo "ERROR: Some instances failed to terminate properly after restart" - dump_all_node_logs - exit 1 -fi - -echo " Instance restart + terminate after cluster restart passed" - -# Run certificate validation E2E while services are still up (before cleanup trap). -echo "" -echo "========================================" -echo "Running Certificate E2E Tests" -echo "========================================" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -"${SCRIPT_DIR}/run-cert-e2e.sh" --pseudo-multinode - -echo "" -echo "========================================" -echo "Multi-Node E2E Tests Completed Successfully" -echo "========================================" -exit 0