From 90369371ee09c9e825482dc7ccf859067bccd143 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Wed, 10 Jun 2026 07:52:09 +1000 Subject: [PATCH] Feat/nightly e2e fix (#330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(e2e-nightly): run Go harness suites instead of bash run-*.sh Swap nightly cells 1-3 (single), 8-10 (real-multi) and 18 (reboot) to the compiled .test binaries, mirroring e2e.yml: SPINIFEX_E2E=1 with SPINIFEX_MODE single|multinode, per-suite logs to ARTIFACT_DIR, then go-junit-report + e2e-analyze + upload-artifact. Adds a spinifex checkout for the composite actions and pulls /run/spinifex/console-*.log so LB microVM (lb-agent) state is captured on failure. Cells 17 (tofu) and 20 (baremetal) stay on bash — no Go suite exists. * ci(e2e-nightly): cell-17 RCA analytics + cell-20 cosmetic alignment run-tofu-examples-e2e.sh now emits go test -v markers per workbook so the cell-17 step's tee'd test-tofu.log flows through the same go-junit-report -> e2e-analyze -> upload-artifact chain as the Go suites, producing junit-tofu.xml and an analysis bundle for RCA. Align e2e-baremetal.yml (cell 20) ARTIFACT_DIR, analyze title, and artifact name with the matrix cells so CI-RCA discovers its bundle uniformly. * ci(e2e-nightly): wire host journal + VM consoles into RCA bundles Collect now emits the canonical spinifex-journal.log (concat of all nodes) that the e2e-analyze bundler slices into each failure's journal.log, so elbv2-service/vpcd/daemon lines in the failure window travel with the bundle (previously the journal uploaded as spinifex-journal-node*.log, a name the bundler never reads). Rename the console capture to vm-console-node* and note it also carries guest cloud-init/sshd consoles, covering the cell-3 internal-LB, cell-18 reboot-datapath, and cell-9/10 guest-ssh blind spots. * ci(e2e): self-heal runner /dev + --one-file-system on workspace clean Clean Workspace now detaches any leaked iso-builder rootfs mount, recreates /dev/null if a prior build clobbered it, and uses rm --one-file-system so the clean can't descend into a still-mounted devtmpfs and delete host device nodes. Guards both the matrix cells and the bare-metal job against the host-wide /dev/null wipe that fails go build with 'open /dev/null: no such file or directory'. --- .github/workflows/e2e-baremetal.yml | 17 ++- .github/workflows/e2e-nightly.yml | 172 ++++++++++++++++++++++++---- tests/e2e/run-tofu-examples-e2e.sh | 27 ++++- 3 files changed, 184 insertions(+), 32 deletions(-) diff --git a/.github/workflows/e2e-baremetal.yml b/.github/workflows/e2e-baremetal.yml index fdfc56208..d593420af 100644 --- a/.github/workflows/e2e-baremetal.yml +++ b/.github/workflows/e2e-baremetal.yml @@ -27,10 +27,19 @@ jobs: cancel-in-progress: false env: BM_ISO_CACHE: /var/cache/spinifex-iso-builder - ARTIFACT_DIR: /tmp/spinifex-e2e-baremetal-artifacts-${{ github.run_id }} + ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }} steps: - name: Clean Workspace - run: sudo rm -rf "$GITHUB_WORKSPACE"/* + run: | + # Self-heal a host whose /dev was clobbered by a leaked iso-builder + # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover + # rootfs mounts and recreate /dev/null before anything runs `go build`, + # which otherwise dies "open /dev/null: no such file or directory". + sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true + if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi + # --one-file-system stops rm descending into a still-mounted fs (e.g. a + # leaked devtmpfs) and deleting host device nodes. + sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* - uses: actions/checkout@v6 with: @@ -142,12 +151,12 @@ jobs: with: junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml log-dir: ${{ env.ARTIFACT_DIR }} - title: "Bare-Metal E2E failure analysis" + title: "E2E failure analysis (cell 20)" - uses: actions/upload-artifact@v7 if: always() with: - name: e2e-baremetal-artifacts-${{ github.run_id }} + name: e2e-artifacts-cell-20-${{ github.run_id }} path: ${{ env.ARTIFACT_DIR }} if-no-files-found: warn retention-days: 14 diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index ed9385c8b..83261571d 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -40,6 +40,7 @@ jobs: timeout-minutes: 35 env: TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log + ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }} steps: # blank or scheduled run = full matrix. @@ -60,7 +61,16 @@ jobs: - name: Clean Workspace if: steps.gate.outputs.selected == 'true' - run: sudo rm -rf "$GITHUB_WORKSPACE"/* + run: | + # Self-heal a host whose /dev was clobbered by a leaked iso-builder + # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover + # rootfs mounts and recreate /dev/null before anything runs `go build`, + # which otherwise dies "open /dev/null: no such file or directory". + sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true + if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi + # --one-file-system stops rm descending into a still-mounted fs (e.g. a + # leaked devtmpfs) and deleting host device nodes. + sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/* - uses: actions/create-github-app-token@v3 if: steps.gate.outputs.selected == 'true' @@ -78,6 +88,13 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga + # spinifex tree on the runner for the e2e-analyze / junit-report composite + # actions and .github/scripts used by the collect/analyze steps below. + - uses: actions/checkout@v6 + if: steps.gate.outputs.selected == 'true' + with: + path: spinifex + - name: Build Install Artifacts if: steps.gate.outputs.selected == 'true' working-directory: mulga/scripts/tofu-cluster @@ -132,8 +149,21 @@ jobs: WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ - "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh" + # Go harness (single/cert/lb) run in-VM; the suites use the node's + # spinifex AWS profile. Per-suite logs land in ARTIFACT_DIR on the VM + # and are pulled back by the Collect step. + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \ + for SUITE in single cert lb; do \ + echo \"::group::suite \$SUITE\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./\$SUITE.test -test.v -test.timeout=30m \ + 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \ + rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \ + if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \ + done; exit \$FAIL" - name: Run Multi Node E2E if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi' @@ -144,32 +174,59 @@ jobs: SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" - NODE_IPS="" + NODES_CSV="" NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') for i in $(seq 0 $((NODE_COUNT - 1))); do - NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")" + ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") + NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip" done - ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ - "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}" + # Go harness (multinode/cert/lb). The multinode suite drives peer nodes + # from node1 over $HOME/.ssh/tf-user-ap-southeast-2 (matches e2e.yml). + ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ + "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \ + for SUITE in multinode cert lb; do \ + echo \"::group::suite \$SUITE\"; \ + date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \ + SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \ + SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \ + ARTIFACT_DIR='${ARTIFACT_DIR}' \ + ./\$SUITE.test -test.v -test.timeout=30m \ + 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \ + rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \ + if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \ + done; exit \$FAIL" - name: Run Tofu Workbooks if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples' working-directory: mulga/scripts/tofu-cluster + # The driver emits go test -v markers per workbook; teeing to + # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload + # chain the Go suites use, so cell 17 produces an RCA bundle too. run: | + set +e + mkdir -p "${ARTIFACT_DIR}" INVENTORY=$(tofu output -json inventory) WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') SSH_KEY="${SSH_KEY/#\~/$HOME}" + date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start" ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \ - "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" + "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \ + 2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log" + rc=${PIPESTATUS[0]} + if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi + exit $rc - name: Run Reboot Resilience E2E if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single' working-directory: mulga/scripts/tofu-cluster - # Driver runs from the runner (NOT via ssh-and-run) so it survives the + # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the # systemctl-reboot-induced ssh disconnect mid-run. run: | + set +e + mkdir -p "${ARTIFACT_DIR}" INVENTORY=$(tofu output -json inventory) WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') @@ -178,31 +235,96 @@ jobs: REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e" mkdir -p "$REBOOT_DIR" tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR" - chmod +x "$REBOOT_DIR/run-reboot-e2e.sh" + chmod +x "$REBOOT_DIR/_bin/reboot.test" - WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY") + CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials') + AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}') + AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}') + echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}" - - name: Dump Cluster Journals on Failure - if: failure() && steps.gate.outputs.selected == 'true' + date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start" + GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ + SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \ + SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \ + SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \ + ARTIFACT_DIR="${ARTIFACT_DIR}" \ + "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \ + 2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" + rc=${PIPESTATUS[0]} + if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi + exit $rc + + - name: Collect e2e artifacts + if: always() && steps.gate.outputs.selected == 'true' working-directory: mulga/scripts/tofu-cluster run: | + set +e + mkdir -p "${ARTIFACT_DIR}" INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; } - SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') - SSH_KEY="${SSH_KEY/#\~/$HOME}" + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}" NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY") + + # In-VM suite artifacts (single/multi write ARTIFACT_DIR on node1; no-op for reboot). + ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + + # All VM serial consoles (LB/system microVMs + guest instances) — the + # lb-agent logs nginx/HAProxy activation + health probes here, and guest + # cloud-init/sshd bring-up lands here too (the cell-18 LB-datapath + + # cell-9/10 guest-ssh blind spots). Pulled per node since any VM may be + # hosted on any node. for i in $(seq 0 $((NODE_COUNT - 1))); do - IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") - echo "===== node$((i+1)) ($IP) =====" - for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd dhcpcd; do - echo "--- $svc ---" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \ - "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)" - done - echo "--- DHCP client state ---" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \ - "systemctl is-enabled dhcpcd.service 2>/dev/null; systemctl is-active dhcpcd.service 2>/dev/null; sudo ss -lupn 'sport = 68' 2>/dev/null; pgrep -laf dhcpcd 2>/dev/null" 2>&1 || echo "(node unreachable)" + IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1)) + mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}" + ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \ + | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null + ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null done + # Canonical journal the e2e-analyze bundler slices into each failure + # bundle as journal.log — it reads "${ARTIFACT_DIR}/spinifex-journal.log". + # Concat every node: SliceJournal filters by each line's short-iso + # timestamp, so cross-node interleaving is fine and the failing node's + # elbv2/vpcd/daemon lines land in the failure window regardless of host. + cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \ + > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true + echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries" + + - name: Install go-junit-report + if: always() && steps.gate.outputs.selected == 'true' + run: | + go install github.com/jstemmer/go-junit-report/v2@latest + echo "$HOME/go/bin" >> "$GITHUB_PATH" + + - name: Convert test logs to JUnit XML + if: always() && steps.gate.outputs.selected == 'true' + run: | + shopt -s nullglob + for LOG in "${ARTIFACT_DIR}"/test-*.log; do + BASE=$(basename "$LOG" .log); SUITE=${BASE#test-} + go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true + done + + - name: Analyze e2e failures + if: always() && steps.gate.outputs.selected == 'true' + uses: ./spinifex/.github/actions/e2e-analyze + with: + junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml + log-dir: ${{ env.ARTIFACT_DIR }} + title: "E2E failure analysis (cell ${{ matrix.cell }})" + + - uses: actions/upload-artifact@v7 + if: always() && steps.gate.outputs.selected == 'true' + with: + name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: warn + retention-days: 14 + - name: Teardown if: always() working-directory: mulga/scripts/tofu-cluster diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh index af5966fe8..340cbeeb5 100755 --- a/tests/e2e/run-tofu-examples-e2e.sh +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -338,12 +338,33 @@ if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then fi log "Using instance_type=${INSTANCE_TYPE}" +# Each workbook is emitted as a top-level `go test -v` testcase +# (=== RUN / --- PASS|FAIL / package trailer) so go-junit-report converts the +# tee'd log into junit-tofu.xml and the e2e-analyze action produces the same +# RCA bundle the Go suites do (nightly cell 17). The per-workbook log() output +# between RUN and the result line is captured as the failure diagnostics. +SUITE_RC=0 for workbook in nginx-alb bastion-private-subnet nginx-webserver s3-webapp; do - if ! run_workbook "$workbook"; then + tname="TestTofuWorkbook_${workbook//-/_}" + wb_start=$SECONDS + echo "=== RUN ${tname}" + if run_workbook "$workbook"; then + printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))" + else + printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))" log "FAIL ${workbook} — aborting remaining workbooks" - exit 1 + SUITE_RC=1 + break fi done CURRENT_WORKBOOK="" -log "All workbooks passed" +if [ "$SUITE_RC" -eq 0 ]; then + log "All workbooks passed" + echo "PASS" + printf 'ok \ttofu-examples\t%d.000s\n' "$SECONDS" +else + echo "FAIL" + printf 'FAIL\ttofu-examples\t%d.000s\n' "$SECONDS" +fi +exit "$SUITE_RC"