fix: add detailed logging to e2e

tests/e2e/harness/ec2helpers.go

@@ -43,6 +43,27 @@ func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) {
 	return pub, 22
 }
 
+// InstancePrivateIP returns inst's primary private IPv4, or "" when unset or
+// unresolvable. Used to populate VPCDiagnosticsOpts.LogicalIP so the datapath
+// captures grep both halves of the NAT translation. Non-fatal — it feeds a
+// best-effort diagnostic, so a describe miss logs and returns "".
+func InstancePrivateIP(t *testing.T, c *AWSClient, instanceID string) string {
+	t.Helper()
+	out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	})
+	if err != nil {
+		t.Logf("InstancePrivateIP: describe %s: %v", instanceID, err)
+		return ""
+	}
+	for _, r := range out.Reservations {
+		for _, in := range r.Instances {
+			return aws.StringValue(in.PrivateIpAddress)
+		}
+	}
+	return ""
+}
+
 // LsblkRootGiB SSHes into the VM and returns the root disk size in GiB
 // (bytes / 1<<30, rounded down) by running findmnt + lsblk in the guest.
 func LsblkRootGiB(t *testing.T, tgt SSHTarget) int {

tests/e2e/harness/multinode_guest_ssh.go

@@ -59,6 +59,39 @@ func GuestSSHReady(t *testing.T, host string, port int, user, pemfile string, op
 	}, cfg.timeout, cfg.interval)
 }
 
+// TryGuestSSHReady is the non-fatal twin of GuestSSHReady: it polls SSH up to
+// timeout and returns whether the probe ever succeeded instead of t.Fatal-ing.
+// Callers use a false return to dump datapath diagnostics before failing — a
+// fresh cross-node public-IP guest unreachable for the full window is the
+// SNAT/DNAT flow-install flake signature, only diagnosable from CI artifacts.
+func TryGuestSSHReady(host string, port int, user, pemfile string, timeout time.Duration) bool {
+	target := fmt.Sprintf("%s@%s", user, host)
+	args := []string{
+		"-i", pemfile,
+		"-p", strconv.Itoa(port),
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "ConnectTimeout=2",
+		"-o", "BatchMode=yes",
+		"-o", "LogLevel=ERROR",
+		target,
+		"echo ready",
+	}
+	deadline := time.Now().Add(timeout)
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
+		err := runSSH(ctx, args)
+		cancel()
+		if err == nil {
+			return true
+		}
+		if time.Now().After(deadline) {
+			return false
+		}
+		time.Sleep(2 * time.Second)
+	}
+}
+
 func runSSH(ctx context.Context, args []string) error {
 	out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
 	if err != nil {

tests/e2e/multinode/volume_durability_test.go

@@ -3,6 +3,7 @@
 package multinode
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -93,10 +94,25 @@ func pickCrossNodePair(t *testing.T, fix *Fixture, trio []string) (srcID, dstID,
 }
 
 // guestTarget resolves an SSH target for a guest, waiting until SSH answers.
+// On a 2min timeout it dumps VPC/IGW datapath diagnostics (guest console + OVN
+// state + flow/conntrack/NAT captures) before failing — a fresh cross-node
+// public-IP VM unreachable for the full window is the SNAT/DNAT flow-install
+// flake signature, only diagnosable from CI artifacts.
 func guestTarget(t *testing.T, fix *Fixture, instanceID, pemPath string) harness.SSHTarget {
 	t.Helper()
 	host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, instanceID)
-	harness.GuestSSHReady(t, host, port, "ec2-user", pemPath,
-		harness.WithTimeout(2*time.Minute), harness.WithPoll(2*time.Second))
+	if !harness.TryGuestSSHReady(host, port, "ec2-user", pemPath, 2*time.Minute) {
+		privIP := harness.InstancePrivateIP(t, fix.AWS, instanceID)
+		harness.DumpVPCFlowDiagnostics(t, fix.AWS, instanceID,
+			fmt.Sprintf("volume-durability guest SSH timeout — instance=%s pub=%s priv=%s",
+				instanceID, host, privIP),
+			harness.VPCDiagnosticsOpts{
+				ExternalIP:  host,
+				LogicalIP:   privIP,
+				ArtifactDir: fix.Artifacts,
+			})
+		t.Fatalf("guest %s SSH %s:%d never became reachable within 2min (see diagnostics above)",
+			instanceID, host, port)
+	}
 	return harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: pemPath}
 }

tests/e2e/run-tofu-examples-e2e.sh

@@ -231,6 +231,27 @@ assert_nginx_webserver() {
     wait_for_http_200 "http://${ip}/"
 }
 
+# dump_s3_webapp_guest SSHes into the webapp instance and tails cloud-init's
+# output log plus the s3-webapp unit status, so a first-boot apt/pip/egress
+# failure is distinguishable from a control-plane regression in the bundle.
+# Best-effort: a missing key or unreachable SSH just logs and returns.
+dump_s3_webapp_guest() {
+    local host="$1" key="$(pwd)/s3-webapp-demo.pem"
+    if [ ! -f "$key" ]; then
+        log "  s3-webapp: ${key} missing, skipping guest cloud-init dump"
+        return 0
+    fi
+    chmod 600 "$key" 2>/dev/null || true
+    log "--- s3-webapp guest: cloud-init status + output log + unit status ---"
+    ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" '
+        echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true
+        echo "== /var/log/cloud-init-output.log (tail 80) =="
+        sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true
+        echo "== s3-webapp.service =="
+        systemctl status s3-webapp --no-pager 2>/dev/null | head -20 || true
+    ' 2>&1 | sed "s|^|    |" || log "  s3-webapp: guest SSH unreachable for cloud-init dump"
+}
+
 assert_s3_webapp() {
     # Prove the INSTANCE wrote to S3 with IMDS-sourced STS creds: upload a file
     # through the app (PutObject), then read it back from the listing (ListBucket
@@ -241,8 +262,12 @@ assert_s3_webapp() {
     ip=$(tofu output -raw public_ip)
     sentinel="spinifex-nightly-$(date +%s).txt"
 
-    wait_for_http_200 "http://${ip}/" || {
-        log "  s3-webapp: webapp not reachable on http://${ip}/"
+    # 420s budget: this is the heaviest first-boot of any workbook — cloud-init
+    # runs apt-get install python3-pip/venv + pip install flask boto3 before the
+    # listener binds :80. The default 150s isn't enough on a slow runner / PyPI.
+    wait_for_http_200 "http://${ip}/" 420 || {
+        log "  s3-webapp: webapp not reachable on http://${ip}/ after 420s"
+        dump_s3_webapp_guest "$ip"
         return 1
     }