diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index b349567c8..da210f76e 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -196,6 +196,24 @@ jobs: WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh" + - name: Dump Cluster Journals on Failure + if: failure() && steps.gate.outputs.selected == 'true' + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; } + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length') + for i in $(seq 0 $((NODE_COUNT - 1))); do + IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip") + echo "===== node$((i+1)) ($IP) =====" + for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd; do + echo "--- $svc ---" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \ + "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)" + done + done + - name: Teardown if: always() working-directory: mulga/scripts/tofu-cluster