diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go index d834eb054..67ec1827e 100644 --- a/tests/e2e/harness/ec2helpers.go +++ b/tests/e2e/harness/ec2helpers.go @@ -43,6 +43,27 @@ func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) { return pub, 22 } +// InstancePrivateIP returns inst's primary private IPv4, or "" when unset or +// unresolvable. Used to populate VPCDiagnosticsOpts.LogicalIP so the datapath +// captures grep both halves of the NAT translation. Non-fatal — it feeds a +// best-effort diagnostic, so a describe miss logs and returns "". +func InstancePrivateIP(t *testing.T, c *AWSClient, instanceID string) string { + t.Helper() + out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + if err != nil { + t.Logf("InstancePrivateIP: describe %s: %v", instanceID, err) + return "" + } + for _, r := range out.Reservations { + for _, in := range r.Instances { + return aws.StringValue(in.PrivateIpAddress) + } + } + return "" +} + // LsblkRootGiB SSHes into the VM and returns the root disk size in GiB // (bytes / 1<<30, rounded down) by running findmnt + lsblk in the guest. func LsblkRootGiB(t *testing.T, tgt SSHTarget) int { diff --git a/tests/e2e/harness/multinode_guest_ssh.go b/tests/e2e/harness/multinode_guest_ssh.go index bd915c740..a52d7c670 100644 --- a/tests/e2e/harness/multinode_guest_ssh.go +++ b/tests/e2e/harness/multinode_guest_ssh.go @@ -59,6 +59,39 @@ func GuestSSHReady(t *testing.T, host string, port int, user, pemfile string, op }, cfg.timeout, cfg.interval) } +// TryGuestSSHReady is the non-fatal twin of GuestSSHReady: it polls SSH up to +// timeout and returns whether the probe ever succeeded instead of t.Fatal-ing. +// Callers use a false return to dump datapath diagnostics before failing — a +// fresh cross-node public-IP guest unreachable for the full window is the +// SNAT/DNAT flow-install flake signature, only diagnosable from CI artifacts. +func TryGuestSSHReady(host string, port int, user, pemfile string, timeout time.Duration) bool { + target := fmt.Sprintf("%s@%s", user, host) + args := []string{ + "-i", pemfile, + "-p", strconv.Itoa(port), + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "ConnectTimeout=2", + "-o", "BatchMode=yes", + "-o", "LogLevel=ERROR", + target, + "echo ready", + } + deadline := time.Now().Add(timeout) + for { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + err := runSSH(ctx, args) + cancel() + if err == nil { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(2 * time.Second) + } +} + func runSSH(ctx context.Context, args []string) error { out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput() if err != nil { diff --git a/tests/e2e/multinode/volume_durability_test.go b/tests/e2e/multinode/volume_durability_test.go index e386b7b11..7aad5d8b3 100644 --- a/tests/e2e/multinode/volume_durability_test.go +++ b/tests/e2e/multinode/volume_durability_test.go @@ -3,6 +3,7 @@ package multinode import ( + "fmt" "testing" "time" @@ -93,10 +94,25 @@ func pickCrossNodePair(t *testing.T, fix *Fixture, trio []string) (srcID, dstID, } // guestTarget resolves an SSH target for a guest, waiting until SSH answers. +// On a 2min timeout it dumps VPC/IGW datapath diagnostics (guest console + OVN +// state + flow/conntrack/NAT captures) before failing — a fresh cross-node +// public-IP VM unreachable for the full window is the SNAT/DNAT flow-install +// flake signature, only diagnosable from CI artifacts. func guestTarget(t *testing.T, fix *Fixture, instanceID, pemPath string) harness.SSHTarget { t.Helper() host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, instanceID) - harness.GuestSSHReady(t, host, port, "ec2-user", pemPath, - harness.WithTimeout(2*time.Minute), harness.WithPoll(2*time.Second)) + if !harness.TryGuestSSHReady(host, port, "ec2-user", pemPath, 2*time.Minute) { + privIP := harness.InstancePrivateIP(t, fix.AWS, instanceID) + harness.DumpVPCFlowDiagnostics(t, fix.AWS, instanceID, + fmt.Sprintf("volume-durability guest SSH timeout — instance=%s pub=%s priv=%s", + instanceID, host, privIP), + harness.VPCDiagnosticsOpts{ + ExternalIP: host, + LogicalIP: privIP, + ArtifactDir: fix.Artifacts, + }) + t.Fatalf("guest %s SSH %s:%d never became reachable within 2min (see diagnostics above)", + instanceID, host, port) + } return harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: pemPath} } diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh index d3b1adcf3..51a885893 100755 --- a/tests/e2e/run-tofu-examples-e2e.sh +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -231,6 +231,27 @@ assert_nginx_webserver() { wait_for_http_200 "http://${ip}/" } +# dump_s3_webapp_guest SSHes into the webapp instance and tails cloud-init's +# output log plus the s3-webapp unit status, so a first-boot apt/pip/egress +# failure is distinguishable from a control-plane regression in the bundle. +# Best-effort: a missing key or unreachable SSH just logs and returns. +dump_s3_webapp_guest() { + local host="$1" key="$(pwd)/s3-webapp-demo.pem" + if [ ! -f "$key" ]; then + log " s3-webapp: ${key} missing, skipping guest cloud-init dump" + return 0 + fi + chmod 600 "$key" 2>/dev/null || true + log "--- s3-webapp guest: cloud-init status + output log + unit status ---" + ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" ' + echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true + echo "== /var/log/cloud-init-output.log (tail 80) ==" + sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true + echo "== s3-webapp.service ==" + systemctl status s3-webapp --no-pager 2>/dev/null | head -20 || true + ' 2>&1 | sed "s|^| |" || log " s3-webapp: guest SSH unreachable for cloud-init dump" +} + assert_s3_webapp() { # Prove the INSTANCE wrote to S3 with IMDS-sourced STS creds: upload a file # through the app (PutObject), then read it back from the listing (ListBucket @@ -241,8 +262,12 @@ assert_s3_webapp() { ip=$(tofu output -raw public_ip) sentinel="spinifex-nightly-$(date +%s).txt" - wait_for_http_200 "http://${ip}/" || { - log " s3-webapp: webapp not reachable on http://${ip}/" + # 420s budget: this is the heaviest first-boot of any workbook — cloud-init + # runs apt-get install python3-pip/venv + pip install flask boto3 before the + # listener binds :80. The default 150s isn't enough on a slow runner / PyPI. + wait_for_http_200 "http://${ip}/" 420 || { + log " s3-webapp: webapp not reachable on http://${ip}/ after 420s" + dump_s3_webapp_guest "$ip" return 1 }