From 2a3ada5e930ef25f9c315108627208bb5dea13db Mon Sep 17 00:00:00 2001
From: Ben Duncan <5178243+benduncan@users.noreply.github.com>
Date: Thu, 26 Mar 2026 15:29:48 +1000
Subject: [PATCH] =?UTF-8?q?fix:=20NAT=20Gateway=20E2E=20=E2=80=94=20poll?=
 =?UTF-8?q?=20for=20private=20instance=20SSH=20readiness=20(150s)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The bastion SSH hop failed because the private instance's cloud-init
hadn't completed (SSH daemon not ready). The instance was "running"
from the hypervisor perspective but cloud-init takes ~30-40s.

- Replace 2-attempt retry with 30-attempt poll (5s intervals, 150s max)
- Hard failure with OVN debug output instead of silent skip
- Matches how wait_for_ssh works for public instances
---
 tests/e2e/run-e2e.sh | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh
index 62e3f3ffe..cce30786d 100755
--- a/tests/e2e/run-e2e.sh
+++ b/tests/e2e/run-e2e.sh
@@ -2948,22 +2948,30 @@ echo "Setting up bastion SSH key..."
 scp -o StrictHostKeyChecking=no -o LogLevel=ERROR -i test-key-1.pem test-key-1.pem "ec2-user@$PUB_IP:/tmp/key.pem"
 $BASTION_SSH "chmod 600 /tmp/key.pem"
 
-# Step 1: Verify intra-VPC SSH works (bastion → private VM)
+# Step 1: Wait for private instance SSH to be ready via bastion hop
+# The private VM needs cloud-init time (~30-40s after "running" state).
 echo ""
-echo "Step 1: Verify bastion can reach private instance via VPC"
-PRIV_HOSTNAME=$($BASTION_SSH "$PRIV_SSH_CMD 'hostname'" 2>/dev/null) || {
-    echo "WARN: Cannot SSH from bastion to private instance (intra-VPC may need time)"
-    echo "Waiting 15s for OVN routing to settle..."
-    sleep 15
-    PRIV_HOSTNAME=$($BASTION_SSH "$PRIV_SSH_CMD 'hostname'" 2>/dev/null) || {
-        echo "FAIL: Cannot reach private instance from bastion (intra-VPC broken)"
-        echo "Skipping NAT Gateway tests — bastion hop not working"
-        # Fall through to cleanup
-        SKIP_NATGW=true
-    }
-}
-if [ "${SKIP_NATGW:-}" != "true" ]; then
-    echo "  PASS: Bastion → private instance SSH works (hostname: $PRIV_HOSTNAME)"
+echo "Step 1: Waiting for private instance SSH via bastion (cloud-init boot)..."
+BASTION_SSH_OK=false
+for attempt in $(seq 1 30); do
+    if PRIV_HOSTNAME=$($BASTION_SSH "$PRIV_SSH_CMD 'hostname'" 2>/dev/null); then
+        BASTION_SSH_OK=true
+        break
+    fi
+    echo "  Attempt $attempt/30: private instance SSH not ready yet..."
+    sleep 5
+done
+
+if [ "$BASTION_SSH_OK" != "true" ]; then
+    echo "FAIL: Cannot reach private instance from bastion after 150s"
+    echo "  Public VM (bastion): $PUB_IP → $PUB_PRIVATE_IP"
+    echo "  Private VM target: $PRIV_PRIVATE_IP"
+    echo "  Debugging: checking OVN port bindings..."
+    sudo ovn-sbctl --no-leader-only show 2>/dev/null | head -20 || true
+    exit 1
+fi
+echo "  PASS: Bastion → private instance SSH works (hostname: $PRIV_HOSTNAME)"
+{
 
     # Step 2: Baseline — private instance has NO internet
     echo ""
@@ -3042,7 +3050,7 @@ if [ "${SKIP_NATGW:-}" != "true" ]; then
 
     echo ""
     echo "Phase 8d: NAT Gateway E2E PASSED"
-fi
+}
 
 echo ""
 echo "Phase 8b/8d Step: Cleanup"