From 0cf3627430e7172fd0c8e05c2b983a29a71bee50 Mon Sep 17 00:00:00 2001
From: joshsiv-mulga <mrk@mulgadc.com>
Date: Mon, 11 May 2026 13:23:20 +1000
Subject: [PATCH] ci(e2e-ddil): collect cluster logs on failure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Witness `findHost` is currently failing with "could not locate QEMU host
for i-XXX across 3 nodes" — daemon reports running but no QEMU fork is
visible on any node. Cluster gets destroyed during teardown, so logs
have to be pulled before that runs.

Adds a failure-only step that ssh's each node and bundles
journalctl spinifex-*, process listings, and state dirs into the
ddil-results artifact for offline triage.
---
 .github/workflows/e2e-ddil.yml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/.github/workflows/e2e-ddil.yml b/.github/workflows/e2e-ddil.yml
index 244e7ea36..2c20e32b9 100644
--- a/.github/workflows/e2e-ddil.yml
+++ b/.github/workflows/e2e-ddil.yml
@@ -148,6 +148,31 @@ jobs:
                       --format testname \
                       -- -tags=e2e -timeout 45m ./tests/e2e/ddil/scenarios/...
 
+      - name: Collect Cluster Logs
+        if: failure()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          # Pull spinifex-* journals + QEMU process listings from every
+          # node into the artifact bundle so failures can be triaged
+          # without re-provisioning the cluster. Best-effort — a missing
+          # node should not mask the original test failure.
+          set +e
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
+          mkdir -p "${LOG_DIR}"
+
+          mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
+          for ip in "${NODE_IPS[@]}"; do
+            echo "=== collecting ${ip} ==="
+            SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_USER}@${ip}"
+            ${SSH} 'sudo journalctl -u "spinifex-*" --no-pager -n 5000' > "${LOG_DIR}/${ip}-spinifex.log" 2>&1
+            ${SSH} 'ps auxw | grep -E "qemu-system|spinifex" | grep -v grep' > "${LOG_DIR}/${ip}-procs.txt" 2>&1
+            ${SSH} 'ls -la /var/lib/spinifex 2>/dev/null; ls -la /var/log/spinifex 2>/dev/null' > "${LOG_DIR}/${ip}-state.txt" 2>&1
+          done
+
       - name: Upload DDIL Results
         if: always()
         uses: actions/upload-artifact@v7