diff --git a/.github/workflows/e2e-baremetal.yml b/.github/workflows/e2e-baremetal.yml index d593420af..2f0329760 100644 --- a/.github/workflows/e2e-baremetal.yml +++ b/.github/workflows/e2e-baremetal.yml @@ -31,21 +31,12 @@ jobs: steps: - name: Clean Workspace run: | - # Self-heal a host whose /dev was clobbered by a leaked iso-builder - # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover - # rootfs mounts and recreate /dev/null before anything runs `go build`, - # which otherwise dies "open /dev/null: no such file or directory". sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi - # --one-file-system stops rm descending into a still-mounted fs (e.g. a - # leaked devtmpfs) and deleting host device nodes. sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* - uses: actions/checkout@v6 with: - # When called (nightly/release) the caller picks the ref (dev on - # schedule, the tag on release); standalone dispatch defaults to the - # triggering ref. ref: ${{ inputs.ref || github.ref }} - uses: actions/create-github-app-token@v3 @@ -113,7 +104,6 @@ jobs: echo "Ubuntu 26.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))" fi - - name: Install go-junit-report on runner run: go install github.com/jstemmer/go-junit-report/v2@latest @@ -126,10 +116,6 @@ jobs: SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }} run: | mkdir -p "${ARTIFACT_DIR}" - # Capture combined bootstrap + e2e output so failures produce a - # downloadable, analyzable artifact (the suites run go test -v on the - # init node and stream back over SSH). pipefail keeps the script's - # real exit code through the tee. set -o pipefail sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \ --vmlinuz "${BM_ISO_CACHE}/vmlinuz" \ diff --git a/.github/workflows/e2e-ddil.yml b/.github/workflows/e2e-ddil.yml index a7fde3c24..5e4049101 100644 --- a/.github/workflows/e2e-ddil.yml +++ b/.github/workflows/e2e-ddil.yml @@ -21,10 +21,6 @@ jobs: ddil: name: DDIL Real Multi-Node runs-on: [ self-hosted, ci-multi ] - # Budget: ~45m go test + tofu provision/bootstrap/teardown on ci-multi. - # In Phase 1 the scenarios all t.Skip, so the real-world runtime is - # dominated by provisioning; the 60m ceiling leaves headroom for - # Phase 2/3 when assertions come online. timeout-minutes: 60 env: TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log @@ -39,10 +35,6 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} repositories: mulga - # Full spinifex checkout: the Go harness must run on the runner - # (outside the cluster) so its SSH path survives scenario partitions - # that drop peer-to-peer cluster traffic. Running inside the cluster - # would self-sever when Scenario C isolates a node. - uses: actions/checkout@v6 name: Checkout Spinifex with: @@ -88,7 +80,6 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail - # shellcheck source=/dev/null source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" export TF_VAR_git_branch="${GIT_BRANCH}" quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false @@ -140,10 +131,6 @@ jobs: exit 1 fi - # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2; - # the matching private material was generated on the primary node at - # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH - # the witness via its MapPublicIpOnLaunch-assigned EIP. GUEST_KEY="${RUNNER_TEMP}/spinifex-key" scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}" @@ -151,11 +138,6 @@ jobs: mkdir -p "${RUNNER_TEMP}/ddil-results" cd "${GITHUB_WORKSPACE}/spinifex" - # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout, - # plus machine-readable jsonfile + JUnit XML artifacts. The JUnit - # file feeds the junit-report composite action below so failures - # become summary table rows + workflow annotations. - # shellcheck source=/dev/null source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" banner "DDIL scenarios" DDIL_NODES="${NODE_IPS}" \ @@ -185,10 +167,6 @@ jobs: if: failure() working-directory: mulga/scripts/tofu-cluster run: | - # Pull spinifex-* journals + QEMU process listings from every - # node into the artifact bundle so failures can be triaged - # without re-provisioning the cluster. Best-effort — a missing - # node should not mask the original test failure. set +e INVENTORY=$(tofu output -json inventory) SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') @@ -197,7 +175,6 @@ jobs: LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs" mkdir -p "${LOG_DIR}" - # shellcheck source=/dev/null source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip') for ip in "${NODE_IPS[@]}"; do @@ -221,7 +198,6 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail - # shellcheck source=/dev/null source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 83261571d..abb02642c 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -62,14 +62,8 @@ jobs: - name: Clean Workspace if: steps.gate.outputs.selected == 'true' run: | - # Self-heal a host whose /dev was clobbered by a leaked iso-builder - # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover - # rootfs mounts and recreate /dev/null before anything runs `go build`, - # which otherwise dies "open /dev/null: no such file or directory". sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi - # --one-file-system stops rm descending into a still-mounted fs (e.g. a - # leaked devtmpfs) and deleting host device nodes. sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* - uses: actions/create-github-app-token@v3 @@ -88,8 +82,6 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga - # spinifex tree on the runner for the e2e-analyze / junit-report composite - # actions and .github/scripts used by the collect/analyze steps below. - uses: actions/checkout@v6 if: steps.gate.outputs.selected == 'true' with: @@ -149,9 +141,6 @@ jobs: WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" - # Go harness (single/cert/lb) run in-VM; the suites use the node's - # spinifex AWS profile. Per-suite logs land in ARTIFACT_DIR on the VM - # and are pulled back by the Collect step. ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \ for SUITE in single cert lb; do \ @@ -181,8 +170,6 @@ jobs: NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip" done - # Go harness (multinode/cert/lb). The multinode suite drives peer nodes - # from node1 over $HOME/.ssh/tf-user-ap-southeast-2 (matches e2e.yml). ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \ for SUITE in multinode cert lb; do \ @@ -267,15 +254,9 @@ jobs: NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY") - # In-VM suite artifacts (single/multi write ARTIFACT_DIR on node1; no-op for reboot). ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null - # All VM serial consoles (LB/system microVMs + guest instances) — the - # lb-agent logs nginx/HAProxy activation + health probes here, and guest - # cloud-init/sshd bring-up lands here too (the cell-18 LB-datapath + - # cell-9/10 guest-ssh blind spots). Pulled per node since any VM may be - # hosted on any node. for i in $(seq 0 $((NODE_COUNT - 1))); do IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1)) mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}" @@ -285,11 +266,6 @@ jobs: > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null done - # Canonical journal the e2e-analyze bundler slices into each failure - # bundle as journal.log — it reads "${ARTIFACT_DIR}/spinifex-journal.log". - # Concat every node: SliceJournal filters by each line's short-iso - # timestamp, so cross-node interleaving is fine and the failing node's - # elbv2/vpcd/daemon lines land in the failure window regardless of host. cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \ > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries" @@ -339,8 +315,7 @@ jobs: tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - # Bare-metal single-node suite (PXE-reimaged physical hardware). Doesn't fit - # the tofu matrix shape (different runner/provisioning/120m timeout), so it + # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09d6c3c43..64b5a00e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.26.4' + go-version: "1.26.4" - name: Build microVM artifacts (lb-agent + kernel + initramfs) run: cd spinifex && make build-microvm-image @@ -132,12 +132,6 @@ jobs: steps: - name: Clean Workspace run: | - # This runner (ci-single) is shared with the iso job, which mounts a - # kernel-global devtmpfs at /mulga/scripts/iso-builder/.cache/rootfs/dev. - # If that job is killed its EXIT-trap unmount never runs and the bind - # leaks; a plain rm -rf then traverses it and deletes the host's /dev - # (wipes /dev/null, wedges the box). Detach any leaked mounts under the - # workspace first, then forbid rm from crossing a mount boundary. awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \ | sort -r | xargs -r -I{} sudo umount -l -R {} || true sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* @@ -186,10 +180,6 @@ jobs: tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true - # Bare-metal single-node suite on the real ISO/squashfs + PXE-boot path — - # the only release check that exercises hardware install. Runs in parallel - # with `verify` (both need: release). Promotion and ISO upload both hard-gate - # on this passing. baremetal: needs: release permissions: @@ -198,8 +188,6 @@ jobs: uses: ./.github/workflows/e2e-baremetal.yml secrets: inherit - # Promote the draft only after BOTH the tofu install (`verify`) and the - # bare-metal hardware boot (`baremetal`) pass — runs only if both succeed. promote: needs: [ verify, baremetal ] runs-on: ubuntu-24.04 @@ -248,8 +236,6 @@ jobs: - name: Clean Workspace run: | ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs" - # Mirror make clean's umount sequence unconditionally — lazy unmount - # detaches immediately even if the mount is busy, so rm -rf is safe. sudo umount -l -R "${ROOTFS}" 2>/dev/null || true sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true @@ -335,9 +321,6 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - # Gate the R2 upload on BOTH the tofu install (`verify`) and the - # real-hardware boot (`baremetal`, a reusable workflow whose job shows - # as "baremetal / ..."). Upload only if both conclude success. echo "Waiting for verify + baremetal jobs to complete..." while true; do JOBS=$(gh api \