mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 03:11:46 +00:00
ci(e2e-nightly): dump cluster journals on failure (mulga-siv-32)
run-multinode-e2e.sh dumps node journals via its own EXIT trap, but run-cert-e2e.sh / run-lb-e2e.sh / reboot-e2e do not. When those scripts fail the cluster is destroyed in Teardown without surfacing any daemon logs, blocking root-cause analysis. Add a workflow step that runs after the test scripts but before Teardown when a previous step failed, ssh'ing to each node and pulling the last 200 lines of every spinifex-* unit. Best-effort: skips gracefully when inventory or nodes are unreachable. Diagnostic-only; no behavioural change to the test path.
This commit is contained in:
@@ -196,6 +196,24 @@ jobs:
|
||||
|
||||
WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh"
|
||||
|
||||
- name: Dump Cluster Journals on Failure
|
||||
if: failure() && steps.gate.outputs.selected == 'true'
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
|
||||
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
||||
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
||||
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
||||
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
||||
IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
|
||||
echo "===== node$((i+1)) ($IP) ====="
|
||||
for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd; do
|
||||
echo "--- $svc ---"
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
|
||||
"sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)"
|
||||
done
|
||||
done
|
||||
|
||||
- name: Teardown
|
||||
if: always()
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
|
||||
Reference in New Issue
Block a user