Feat/nightly e2e fix (#330)

* ci(e2e-nightly): run Go harness suites instead of bash run-*.sh Swap nightly cells 1-3 (single), 8-10 (real-multi) and 18 (reboot) to the compiled <suite>.test binaries, mirroring e2e.yml: SPINIFEX_E2E=1 with SPINIFEX_MODE single|multinode, per-suite logs to ARTIFACT_DIR, then go-junit-report + e2e-analyze + upload-artifact. Adds a spinifex checkout for the composite actions and pulls /run/spinifex/console-*.log so LB microVM (lb-agent) state is captured on failure. Cells 17 (tofu) and 20 (baremetal) stay on bash — no Go suite exists. * ci(e2e-nightly): cell-17 RCA analytics + cell-20 cosmetic alignment run-tofu-examples-e2e.sh now emits go test -v markers per workbook so the cell-17 step's tee'd test-tofu.log flows through the same go-junit-report -> e2e-analyze -> upload-artifact chain as the Go suites, producing junit-tofu.xml and an analysis bundle for RCA. Align e2e-baremetal.yml (cell 20) ARTIFACT_DIR, analyze title, and artifact name with the matrix cells so CI-RCA discovers its bundle uniformly. * ci(e2e-nightly): wire host journal + VM consoles into RCA bundles Collect now emits the canonical spinifex-journal.log (concat of all nodes) that the e2e-analyze bundler slices into each failure's journal.log, so elbv2-service/vpcd/daemon lines in the failure window travel with the bundle (previously the journal uploaded as spinifex-journal-node*.log, a name the bundler never reads). Rename the console capture to vm-console-node* and note it also carries guest cloud-init/sshd consoles, covering the cell-3 internal-LB, cell-18 reboot-datapath, and cell-9/10 guest-ssh blind spots. * ci(e2e): self-heal runner /dev + --one-file-system on workspace clean Clean Workspace now detaches any leaked iso-builder rootfs mount, recreates /dev/null if a prior build clobbered it, and uses rm --one-file-system so the clean can't descend into a still-mounted devtmpfs and delete host device nodes. Guards both the matrix cells and the bare-metal job against the host-wide /dev/null wipe that fails go build with 'open /dev/null: no such file or directory'.
2026-10-15 17:11:30 +00:00 · 2026-06-10 07:52:09 +10:00
parent 6483c6f732
commit 90369371ee
3 changed files with 184 additions and 32 deletions
@@ -27,10 +27,19 @@ jobs:
      cancel-in-progress: false
    env:
      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      ARTIFACT_DIR: /tmp/spinifex-e2e-baremetal-artifacts-${{ github.run_id }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
    steps:
      - name: Clean Workspace
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          # Self-heal a host whose /dev was clobbered by a leaked iso-builder
+          # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover
+          # rootfs mounts and recreate /dev/null before anything runs `go build`,
+          # which otherwise dies "open /dev/null: no such file or directory".
+          sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          # --one-file-system stops rm descending into a still-mounted fs (e.g. a
+          # leaked devtmpfs) and deleting host device nodes.
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/checkout@v6
        with:
@@ -142,12 +151,12 @@ jobs:
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "Bare-Metal E2E failure analysis"
+          title: "E2E failure analysis (cell 20)"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
-          name: e2e-baremetal-artifacts-${{ github.run_id }}
+          name: e2e-artifacts-cell-20-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14
@@ -40,6 +40,7 @@ jobs:
    timeout-minutes: 35
    env:
      TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}

    steps:
      # blank or scheduled run = full matrix.
@@ -60,7 +61,16 @@ jobs:

      - name: Clean Workspace
        if: steps.gate.outputs.selected == 'true'
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          # Self-heal a host whose /dev was clobbered by a leaked iso-builder
+          # devtmpfs mount (see mulga build-rootfs.sh): detach any leftover
+          # rootfs mounts and recreate /dev/null before anything runs `go build`,
+          # which otherwise dies "open /dev/null: no such file or directory".
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          # --one-file-system stops rm descending into a still-mounted fs (e.g. a
+          # leaked devtmpfs) and deleting host device nodes.
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        if: steps.gate.outputs.selected == 'true'
@@ -78,6 +88,13 @@ jobs:
          sparse-checkout: scripts/tofu-cluster
          path: mulga

+      # spinifex tree on the runner for the e2e-analyze / junit-report composite
+      # actions and .github/scripts used by the collect/analyze steps below.
+      - uses: actions/checkout@v6
+        if: steps.gate.outputs.selected == 'true'
+        with:
+          path: spinifex
+
      - name: Build Install Artifacts
        if: steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
@@ -132,8 +149,21 @@ jobs:
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
+          # Go harness (single/cert/lb) run in-VM; the suites use the node's
+          # spinifex AWS profile. Per-suite logs land in ARTIFACT_DIR on the VM
+          # and are pulled back by the Collect step.
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in single cert lb; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"

      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
@@ -144,32 +174,59 @@ jobs:
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

-          NODE_IPS=""
+          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
-            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
+          # Go harness (multinode/cert/lb). The multinode suite drives peer nodes
+          # from node1 over $HOME/.ssh/tf-user-ap-southeast-2 (matches e2e.yml).
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in multinode cert lb; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+                 SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+                 SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"

      - name: Run Tofu Workbooks
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
        working-directory: mulga/scripts/tofu-cluster
+        # The driver emits go test -v markers per workbook; teeing to
+        # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
+        # chain the Go suites use, so cell 17 produces an RCA bundle too.
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh"
+            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
+            2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc

      - name: Run Reboot Resilience E2E
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
        working-directory: mulga/scripts/tofu-cluster
-        # Driver runs from the runner (NOT via ssh-and-run) so it survives the
+        # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
        # systemctl-reboot-induced ssh disconnect mid-run.
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -178,31 +235,96 @@ jobs:
          REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
          mkdir -p "$REBOOT_DIR"
          tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
-          chmod +x "$REBOOT_DIR/run-reboot-e2e.sh"
+          chmod +x "$REBOOT_DIR/_bin/reboot.test"

-          WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
+          CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
+          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"

-      - name: Dump Cluster Journals on Failure
-        if: failure() && steps.gate.outputs.selected == 'true'
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
+          GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+            SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
+            SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
+            SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
+            ARTIFACT_DIR="${ARTIFACT_DIR}" \
+            "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
+            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc
+
+      - name: Collect e2e artifacts
+        if: always() && steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          # In-VM suite artifacts (single/multi write ARTIFACT_DIR on node1; no-op for reboot).
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+
+          # All VM serial consoles (LB/system microVMs + guest instances) — the
+          # lb-agent logs nginx/HAProxy activation + health probes here, and guest
+          # cloud-init/sshd bring-up lands here too (the cell-18 LB-datapath +
+          # cell-9/10 guest-ssh blind spots). Pulled per node since any VM may be
+          # hosted on any node.
          for i in $(seq 0 $((NODE_COUNT - 1))); do
-            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
-            echo "===== node$((i+1)) ($IP) ====="
-            for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd dhcpcd; do
-              echo "--- $svc ---"
-              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-                "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)"
-            done
-            echo "--- DHCP client state ---"
-            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-              "systemctl is-enabled dhcpcd.service 2>/dev/null; systemctl is-active dhcpcd.service 2>/dev/null; sudo ss -lupn 'sport = 68' 2>/dev/null; pgrep -laf dhcpcd 2>/dev/null" 2>&1 || echo "(node unreachable)"
+            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
+            mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+              | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
          done

+          # Canonical journal the e2e-analyze bundler slices into each failure
+          # bundle as journal.log — it reads "${ARTIFACT_DIR}/spinifex-journal.log".
+          # Concat every node: SliceJournal filters by each line's short-iso
+          # timestamp, so cross-node interleaving is fine and the failing node's
+          # elbv2/vpcd/daemon lines land in the failure window regardless of host.
+          cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Install go-junit-report
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Convert test logs to JUnit XML
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E failure analysis (cell ${{ matrix.cell }})"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.selected == 'true'
+        with:
+          name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
@@ -338,12 +338,33 @@ if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then
 fi
 log "Using instance_type=${INSTANCE_TYPE}"

+# Each workbook is emitted as a top-level `go test -v` testcase
+# (=== RUN / --- PASS|FAIL / package trailer) so go-junit-report converts the
+# tee'd log into junit-tofu.xml and the e2e-analyze action produces the same
+# RCA bundle the Go suites do (nightly cell 17). The per-workbook log() output
+# between RUN and the result line is captured as the failure diagnostics.
+SUITE_RC=0
 for workbook in nginx-alb bastion-private-subnet nginx-webserver s3-webapp; do
-    if ! run_workbook "$workbook"; then
+    tname="TestTofuWorkbook_${workbook//-/_}"
+    wb_start=$SECONDS
+    echo "=== RUN   ${tname}"
+    if run_workbook "$workbook"; then
+        printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
+    else
+        printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - wb_start))"
        log "FAIL ${workbook} — aborting remaining workbooks"
-        exit 1
+        SUITE_RC=1
+        break
    fi
 done

 CURRENT_WORKBOOK=""
-log "All workbooks passed"
+if [ "$SUITE_RC" -eq 0 ]; then
+    log "All workbooks passed"
+    echo "PASS"
+    printf 'ok  \ttofu-examples\t%d.000s\n' "$SECONDS"
+else
+    echo "FAIL"
+    printf 'FAIL\ttofu-examples\t%d.000s\n' "$SECONDS"
+fi
+exit "$SUITE_RC"