diff --git a/tests/e2e/run-reboot-e2e.sh b/tests/e2e/run-reboot-e2e.sh index e4fd191d7..898028886 100755 --- a/tests/e2e/run-reboot-e2e.sh +++ b/tests/e2e/run-reboot-e2e.sh @@ -10,15 +10,12 @@ # SSH_KEY — path to ssh key the runner can use to reach tf-user@WAN_IP # # Sequence: -# 1. Pre-reboot: VPC + 2 app EC2s + internet-facing ALB; verify round-robin. +# 1. Pre-reboot: VPC + 2 app EC2s + internal ALB; verify round-robin. # 2. Snapshot pre-reboot state (instance IPs, ALB ENI MAC, ovn-nbctl show). # 3. systemctl reboot the host; poll TCP/22 from the runner until SSH returns. # 4. Wait for spinifex services + AWS gateway to respond. # 5. Post-reboot assertions (each a separate pass/fail line). # 6. Cleanup via EXIT trap. -# -# This cell is expected red until the OVN persistence fix lands — the -# `ovn-nbctl show` diff assertion is the canonical reproducer for that bug. set -u @@ -30,7 +27,7 @@ REBOOT_WAIT_SECS="${REBOOT_WAIT_SECS:-300}" DAEMON_READY_SECS="${DAEMON_READY_SECS:-180}" INSTANCE_RUNNING_SECS="${INSTANCE_RUNNING_SECS:-120}" LB_RECOVER_SECS="${LB_RECOVER_SECS:-180}" -GUEST_SSH_SECS="${GUEST_SSH_SECS:-180}" +LAUNCH_TIME_SECS="${LAUNCH_TIME_SECS:-30}" NODE_KEY_PATH="\$HOME/reboot-e2e-key" NODE_USERDATA_PATH="\$HOME/reboot-e2e-userdata.txt" @@ -314,7 +311,7 @@ done pass "all instances have private IPs" # ========================================================================== -# Phase 3: ALB (internet-facing) +# Phase 3: ALB (internal) # ========================================================================== log "Creating HTTP target group..." ALB_TG_ARN=$(node_aws_elbv2 "create-target-group \ @@ -334,10 +331,10 @@ node_aws_elbv2 "register-targets --target-group-arn $ALB_TG_ARN \ || { fail "register-targets"; exit 1; } pass "registered 2 targets" -log "Creating internet-facing ALB..." +log "Creating internal ALB..." ALB_LB_ARN=$(node_aws_elbv2 "create-load-balancer \ --name reboot-e2e-alb \ - --scheme internet-facing \ + --scheme internal \ --subnets $SUBNET_ID \ --output json" | jq -r '.LoadBalancers[0].LoadBalancerArn') if [ -z "$ALB_LB_ARN" ] || [ "$ALB_LB_ARN" = "null" ]; then fail "create-load-balancer"; exit 1; fi @@ -358,11 +355,14 @@ wait_for_lb_active "$ALB_LB_ARN" "ALB" || { dump_diagnostics; exit 1; } ENI_OUT=$(node_aws_ec2 "describe-network-interfaces \ --filters Name=description,Values='ELB app/reboot-e2e-alb/${ALB_LB_ID}' \ --output json" 2>/dev/null) -ALB_PUBLIC_IP=$(echo "$ENI_OUT" | jq -r '.NetworkInterfaces[0].Association.PublicIp // empty') ALB_ENI_ID=$(echo "$ENI_OUT" | jq -r '.NetworkInterfaces[0].NetworkInterfaceId // empty') ALB_ENI_MAC=$(echo "$ENI_OUT" | jq -r '.NetworkInterfaces[0].MacAddress // empty') -if [ -z "$ALB_PUBLIC_IP" ] || [ "$ALB_PUBLIC_IP" = "null" ]; then fail "ALB has no public IP"; exit 1; fi -pass "ALB ENI $ALB_ENI_ID public IP: $ALB_PUBLIC_IP" +ALB_PRIVATE_IP=$(echo "$ENI_OUT" | jq -r '.NetworkInterfaces[0].PrivateIpAddress // empty') +if [ -z "$ALB_ENI_ID" ] || [ "$ALB_ENI_ID" = "null" ]; then fail "ALB ENI not found"; exit 1; fi +# Internal ALB: no public IP expected; traffic uses mgmt IP reachable via br-mgmt +ALB_MGMT_IP=$(node_ssh "ip -4 neigh show dev br-mgmt 2>/dev/null | awk '/'"$ALB_ENI_MAC"'/{print \$1}'" || echo "") +if [ -z "$ALB_MGMT_IP" ]; then fail "ALB mgmt IP not found in br-mgmt neighbours (MAC $ALB_ENI_MAC)"; exit 1; fi +pass "ALB ENI $ALB_ENI_ID private IP: $ALB_PRIVATE_IP mgmt IP: $ALB_MGMT_IP" wait_for_targets_healthy "$ALB_TG_ARN" 2 "ALB" || { dump_diagnostics; exit 1; } @@ -371,7 +371,7 @@ wait_for_targets_healthy "$ALB_TG_ARN" 2 "ALB" || { dump_diagnostics; exit 1; } # ========================================================================== echo "" echo "=== Phase 4: Pre-reboot traffic ===" -run_http_burst "http://${ALB_PUBLIC_IP}:80" "ALB pre-reboot" +run_http_burst "http://${ALB_MGMT_IP}:80" "ALB pre-reboot" # ========================================================================== # Phase 5: Snapshot pre-reboot state @@ -484,29 +484,31 @@ if [ "$IPS_PRESERVED" = true ]; then pass "private IPs preserved across reboot" fi -# 8.3: guest VMs actually rebooted (uptime check via two-hop ssh) -log "Verifying guest VMs rebooted (poll up to ${GUEST_SSH_SECS}s per guest)..." -GUESTS_REBOOTED=true +# 8.3: guest VMs relaunched post-reboot (LaunchTime > reboot time via describe-instances) +# Two-hop SSH to VPC private IPs is not possible in single-node macvlan mode — +# the host has no route to VPC subnets. LaunchTime from the daemon's recovery +# path (reset to time.Now() on relaunch) is a reliable reboot signal. +log "Verifying guest VMs relaunched post-reboot (LaunchTime check)..." +GUESTS_RELAUNCHED=true for inst_id in "${APP_INSTANCE_IDS[@]}"; do - GUEST_IP="${PRE_INSTANCE_IPS[$inst_id]}" - GUEST_UPTIME="" - for attempt in $(seq 1 $((GUEST_SSH_SECS / 5))); do - GUEST_UPTIME=$(node_ssh "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR -i $NODE_KEY_PATH ubuntu@${GUEST_IP} 'cat /proc/uptime'" 2>/dev/null | awk '{print int($1)}') - if [ -n "$GUEST_UPTIME" ]; then break; fi - sleep 5 - done - if [ -z "$GUEST_UPTIME" ]; then - fail "$inst_id ($GUEST_IP): guest ssh failed after ${GUEST_SSH_SECS}s" - GUESTS_REBOOTED=false - elif [ "$GUEST_UPTIME" -gt 600 ]; then - fail "$inst_id ($GUEST_IP): uptime=${GUEST_UPTIME}s — guest did not reboot" - GUESTS_REBOOTED=false + LAUNCH_TS=$(node_aws_ec2 "describe-instances --instance-ids $inst_id \ + --query 'Reservations[0].Instances[0].LaunchTime' \ + --output text" 2>/dev/null || echo "") + if [ -z "$LAUNCH_TS" ] || [ "$LAUNCH_TS" = "None" ]; then + fail "$inst_id: could not retrieve LaunchTime" + GUESTS_RELAUNCHED=false + continue + fi + LAUNCH_EPOCH=$(date -d "$LAUNCH_TS" +%s 2>/dev/null || echo "0") + if [ "$LAUNCH_EPOCH" -ge "$REBOOT_START" ]; then + log " $inst_id: LaunchTime=${LAUNCH_TS} ✓" else - log " $inst_id ($GUEST_IP): uptime=${GUEST_UPTIME}s ✓" + fail "$inst_id: LaunchTime=${LAUNCH_TS} predates reboot — VM not relaunched" + GUESTS_RELAUNCHED=false fi done -if [ "$GUESTS_REBOOTED" = true ]; then - pass "all guests rebooted (uptime < 600s)" +if [ "$GUESTS_RELAUNCHED" = true ]; then + pass "all instances relaunched post-reboot" fi # 8.4: ALB back to active @@ -518,7 +520,7 @@ wait_for_targets_healthy "$ALB_TG_ARN" 2 "ALB post-reboot" "$LB_RECOVER_SECS" || # 8.6: re-run the traffic burst echo "" log "Re-running traffic burst against same ALB..." -run_http_burst "http://${ALB_PUBLIC_IP}:80" "ALB post-reboot" +run_http_burst "http://${ALB_MGMT_IP}:80" "ALB post-reboot" # 8.7: ovn-nbctl diff (this is where the known persistence bug lands) log "Diffing ovn-nbctl show against pre-reboot snapshot..."