gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(e2e): poll guest SSH after reboot in reboot-e2e cell

Browse Source 
The post-reboot guest uptime check was a single-shot ssh with a 10s connect
timeout. After the host comes back, guest QEMU + cloud-init + sshd inside
each guest still need a few seconds to settle, so the check raced them.

Poll up to GUEST_SSH_SECS (default 180s) per guest, mirroring the host
TCP/22 poll. Run 25329248329 cell-18 reproduced this — ALB targets went
healthy ~2s after the second SSH attempt failed.
This commit is contained in:
Julian Sommer
2026-05-05 09:54:47 +10:00
parent 89aec5c709
commit ce0bbde8dc
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/run-reboot-e2e.sh
+9 -3
View File
@@ -30,6 +30,7 @@ REBOOT_WAIT_SECS="${REBOOT_WAIT_SECS:-300}"
DAEMON_READY_SECS="${DAEMON_READY_SECS:-180}"
INSTANCE_RUNNING_SECS="${INSTANCE_RUNNING_SECS:-120}"
LB_RECOVER_SECS="${LB_RECOVER_SECS:-180}"
GUEST_SSH_SECS="${GUEST_SSH_SECS:-180}"
NODE_KEY_PATH="\$HOME/reboot-e2e-key"
NODE_USERDATA_PATH="\$HOME/reboot-e2e-userdata.txt"
@@ -484,13 +485,18 @@ if [ "$IPS_PRESERVED" = true ]; then
fi
# 8.3: guest VMs actually rebooted (uptime check via two-hop ssh)
log "Verifying guest VMs rebooted (ssh into each guest from spinifex node)..."
log "Verifying guest VMs rebooted (poll up to ${GUEST_SSH_SECS}s per guest)..."
GUESTS_REBOOTED=true
for inst_id in "${APP_INSTANCE_IDS[@]}"; do
GUEST_IP="${PRE_INSTANCE_IPS[$inst_id]}"
GUEST_UPTIME=$(node_ssh "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o LogLevel=ERROR -i $NODE_KEY_PATH ubuntu@${GUEST_IP} 'cat /proc/uptime'" 2>/dev/null | awk '{print int($1)}')
GUEST_UPTIME=""
for attempt in $(seq 1 $((GUEST_SSH_SECS / 5))); do
GUEST_UPTIME=$(node_ssh "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR -i $NODE_KEY_PATH ubuntu@${GUEST_IP} 'cat /proc/uptime'" 2>/dev/null | awk '{print int($1)}')
if [ -n "$GUEST_UPTIME" ]; then break; fi
sleep 5
done
if [ -z "$GUEST_UPTIME" ]; then
fail "$inst_id ($GUEST_IP): guest ssh failed"
fail "$inst_id ($GUEST_IP): guest ssh failed after ${GUEST_SSH_SECS}s"
GUESTS_REBOOTED=false
elif [ "$GUEST_UPTIME" -gt 600 ]; then
fail "$inst_id ($GUEST_IP): uptime=${GUEST_UPTIME}s — guest did not reboot"