chore: merge main into dev

.github/workflows/e2e-nightly.yml

@@ -12,8 +12,11 @@ on:
 permissions:
   contents: read
 
+# Scheduled runs target dev (active development trunk). GitHub loads scheduled
+# workflow files from the repo's default branch, so this file must be kept in
+# sync on main for the schedule to use it.
 env:
-  GIT_BRANCH: ${{ github.ref_name }}
+  GIT_BRANCH: dev
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
 
 jobs:

tests/e2e/run-reboot-e2e.sh

@@ -30,6 +30,7 @@ REBOOT_WAIT_SECS="${REBOOT_WAIT_SECS:-300}"
 DAEMON_READY_SECS="${DAEMON_READY_SECS:-180}"
 INSTANCE_RUNNING_SECS="${INSTANCE_RUNNING_SECS:-120}"
 LB_RECOVER_SECS="${LB_RECOVER_SECS:-180}"
+GUEST_SSH_SECS="${GUEST_SSH_SECS:-180}"
 
 NODE_KEY_PATH="\$HOME/reboot-e2e-key"
 NODE_USERDATA_PATH="\$HOME/reboot-e2e-userdata.txt"
@@ -484,13 +485,18 @@ if [ "$IPS_PRESERVED" = true ]; then
 fi
 
 # 8.3: guest VMs actually rebooted (uptime check via two-hop ssh)
-log "Verifying guest VMs rebooted (ssh into each guest from spinifex node)..."
+log "Verifying guest VMs rebooted (poll up to ${GUEST_SSH_SECS}s per guest)..."
 GUESTS_REBOOTED=true
 for inst_id in "${APP_INSTANCE_IDS[@]}"; do
     GUEST_IP="${PRE_INSTANCE_IPS[$inst_id]}"
-    GUEST_UPTIME=$(node_ssh "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o LogLevel=ERROR -i $NODE_KEY_PATH ubuntu@${GUEST_IP} 'cat /proc/uptime'" 2>/dev/null | awk '{print int($1)}')
+    GUEST_UPTIME=""
+    for attempt in $(seq 1 $((GUEST_SSH_SECS / 5))); do
+        GUEST_UPTIME=$(node_ssh "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR -i $NODE_KEY_PATH ubuntu@${GUEST_IP} 'cat /proc/uptime'" 2>/dev/null | awk '{print int($1)}')
+        if [ -n "$GUEST_UPTIME" ]; then break; fi
+        sleep 5
+    done
     if [ -z "$GUEST_UPTIME" ]; then
-        fail "$inst_id ($GUEST_IP): guest ssh failed"
+        fail "$inst_id ($GUEST_IP): guest ssh failed after ${GUEST_SSH_SECS}s"
         GUESTS_REBOOTED=false
     elif [ "$GUEST_UPTIME" -gt 600 ]; then
         fail "$inst_id ($GUEST_IP): uptime=${GUEST_UPTIME}s — guest did not reboot"