From 72eb43f71c17af9de91508aaf4cb33fc1bcc6601 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Wed, 29 Apr 2026 13:32:29 +1000 Subject: [PATCH] ci(e2e-nightly): dump cluster journals on failure (mulga-siv-32) run-multinode-e2e.sh dumps node journals via its own EXIT trap, but run-cert-e2e.sh / run-lb-e2e.sh / reboot-e2e do not. When those scripts fail the cluster is destroyed in Teardown without surfacing any daemon logs, blocking root-cause analysis. Add a workflow step that runs after the test scripts but before Teardown when a previous step failed, ssh'ing to each node and pulling the last 200 lines of every spinifex-* unit. Best-effort: skips gracefully when inventory or nodes are unreachable. Diagnostic-only; no behavioural change to the test path. --- .github/workflows/e2e-nightly.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index b349567c8..da210f76e 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -196,6 +196,24 @@ jobs: WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh" + - name: Dump Cluster Journals on Failure + if: failure() && steps.gate.outputs.selected == 'true' + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; } + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + for i in $(seq 0 $((NODE_COUNT - 1))); do + IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") + echo "===== node$((i+1)) ($IP) =====" + for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd; do + echo "--- $svc ---" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \ + "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)" + done + done + - name: Teardown if: always() working-directory: mulga/scripts/tofu-cluster