diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go index 9490c5162..d881098f4 100644 --- a/tests/e2e/harness/ec2helpers.go +++ b/tests/e2e/harness/ec2helpers.go @@ -4,9 +4,7 @@ package harness import ( "bytes" - "fmt" "os/exec" - "regexp" "strconv" "strings" "testing" @@ -28,59 +26,27 @@ type SSHTarget struct { KeyPath string } -// InstancePublicSSHHost returns (host, port) for an SSH connection to inst. +// InstancePublicSSHHost returns (host, 22) for an SSH connection to inst's +// public IP. // -// Preference order matches run-e2e.sh: -// 1. Instance's public IP on port 22, when assigned (external networking). -// 2. QEMU hostfwd discovered by scanning `ps -ef` for the qemu process whose -// command line contains the instance ID and a `hostfwd=tcp::-:22` -// argument (dev_networking mode). -// -// t.Fatal on neither resolving — both phases need a usable SSH endpoint. +// There is deliberately NO qemu-hostfwd fallback. hostfwd (127.0.0.1 -> +// guest:22) is a dev_networking shortcut that bypasses the OVN datapath +// entirely — no SG ACL, no IGW, no SNAT/DNAT — so a test that reached a guest +// through it would validate nothing real and mask exactly the networking +// regressions these tests exist to catch. An instance with no public IP is a +// hard failure: the routing it depends on is broken or unconfigured. func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) { t.Helper() if inst == nil { t.Fatalf("InstancePublicSSHHost: nil instance") } - if pub := aws.StringValue(inst.PublicIpAddress); pub != "" && pub != "None" { - return pub, 22 + pub := aws.StringValue(inst.PublicIpAddress) + if pub == "" || pub == "None" { + t.Fatalf("InstancePublicSSHHost: instance %s has no public IP; "+ + "qemu-hostfwd fallback is disabled (it bypasses the OVN datapath)", + aws.StringValue(inst.InstanceId)) } - id := aws.StringValue(inst.InstanceId) - host, port, err := discoverHostfwd(id) - if err != nil { - t.Fatalf("InstancePublicSSHHost: %v", err) - } - return host, port -} - -// hostfwd=tcp:HOST:PORT-:22 — HOST may be empty (qemu defaults to 127.0.0.1). -// Anchor on -:22 so we ignore other hostfwd rules (e.g. metadata, serial). -var hostfwdRE = regexp.MustCompile(`hostfwd=tcp:([^:]*):([0-9]+)-:22`) - -func discoverHostfwd(instanceID string) (string, int, error) { - out, err := exec.Command("ps", "-eo", "args=").CombinedOutput() - if err != nil { - return "", 0, fmt.Errorf("ps: %w", err) - } - for _, line := range strings.Split(string(out), "\n") { - if !strings.Contains(line, instanceID) || !strings.Contains(line, "qemu") { - continue - } - m := hostfwdRE.FindStringSubmatch(line) - if m == nil { - continue - } - host := m[1] - if host == "" { - host = "127.0.0.1" - } - port, err := strconv.Atoi(m[2]) - if err != nil { - return "", 0, fmt.Errorf("parse hostfwd port %q: %w", m[2], err) - } - return host, port, nil - } - return "", 0, fmt.Errorf("no qemu hostfwd found for %s", instanceID) + return pub, 22 } // LsblkRootGiB SSHes into the VM and returns the root disk size in GiB, diff --git a/tests/e2e/harness/multinode_guest_ssh.go b/tests/e2e/harness/multinode_guest_ssh.go index f63f0be79..5a19af634 100644 --- a/tests/e2e/harness/multinode_guest_ssh.go +++ b/tests/e2e/harness/multinode_guest_ssh.go @@ -14,63 +14,23 @@ import ( "github.com/aws/aws-sdk-go/aws" ) -// GuestSSHPort extracts the QEMU hostfwd port for an instance from the -// hosting node's qemu process command line. Reuses ec2helpers.hostfwdRE -// (same `hostfwd=tcp:HOST:PORT-:22` pattern). Polls until the qemu -// process appears (the daemon may still be spawning at the time of the -// first ssh in). Returns 0 if the port is never observable within the -// poll window. -func GuestSSHPort(t *testing.T, hostNode Node, instanceID string, opts ...PollOpt) int { - t.Helper() - cfg := applyOpts(pollCfg{timeout: 30 * time.Second, interval: 1 * time.Second}, opts...) - ssh := NewPeerSSH() - var port int - EventuallyErr(t, func() error { - ctx, cancel := context.WithTimeout(context.Background(), cfg.interval+5*time.Second) - defer cancel() - out, err := runPSGrep(ctx, ssh, hostNode, instanceID) - if err != nil { - return fmt.Errorf("%s ps grep: %w", hostNode.Name, err) - } - m := hostfwdRE.FindStringSubmatch(out) - if len(m) < 3 { - return fmt.Errorf("%s no hostfwd port for %s yet", hostNode.Name, instanceID) - } - p, err := strconv.Atoi(m[2]) - if err != nil { - return fmt.Errorf("%s parse hostfwd port %q: %w", hostNode.Name, m[2], err) - } - port = p - return nil - }, cfg.timeout, cfg.interval) - return port -} - -// GuestSSHEndpoint resolves the (host, port) pair for SSH-ing into a guest -// VM. Prefers the instance's PublicIpAddress when set; falls back to the -// hosting node's WAN IP + QEMU hostfwd port. Mirrors the conditional in -// run-multinode-e2e.sh phase 4. +// GuestSSHEndpoint resolves (host, 22) for SSH-ing into a guest VM via its +// public IP. // -// Returns (host, port, nil) on success; t.Fatal on full failure. Public-IP -// path is the steady-state once OVN's public subnet is wired; the -// hostfwd path is the legacy fallback for VPCs without public routing. +// There is deliberately NO qemu-hostfwd fallback (see InstancePublicSSHHost): +// hostfwd bypasses the OVN datapath, so reaching a guest through it would mask +// the networking regressions these tests exist to catch. A running instance +// with no public IP is a hard failure. func GuestSSHEndpoint(t *testing.T, c *AWSClient, cluster *Cluster, instanceID string, opts ...PollOpt) (host string, port int) { t.Helper() - + _ = cluster // retained for signature stability; node hostfwd lookup removed inst := WaitForInstanceState(t, c, instanceID, "running", opts...) - if pub := aws.StringValue(inst.PublicIpAddress); pub != "" { - return pub, 22 + pub := aws.StringValue(inst.PublicIpAddress) + if pub == "" { + t.Fatalf("GuestSSHEndpoint: instance %s has no public IP; "+ + "qemu-hostfwd fallback is disabled (it bypasses the OVN datapath)", instanceID) } - - hostNode := InstanceHostingNode(t, cluster, instanceID) - if hostNode == nil { - t.Fatalf("GuestSSHEndpoint: no node hosts %s", instanceID) - } - port = GuestSSHPort(t, *hostNode, instanceID, opts...) - if port == 0 { - t.Fatalf("GuestSSHEndpoint: hostfwd port not resolved for %s on %s", instanceID, hostNode.Name) - } - return hostNode.Addr, port + return pub, 22 } // GuestSSHReady polls SSH into the named instance until a probe command diff --git a/tests/e2e/multinode/baseline_test.go b/tests/e2e/multinode/baseline_test.go new file mode 100644 index 000000000..3acaecefc --- /dev/null +++ b/tests/e2e/multinode/baseline_test.go @@ -0,0 +1,229 @@ +//go:build e2e + +package multinode + +import ( + "context" + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fresh-install reachability baselines for the multi-node cluster. These run +// right after preflight and before any test triggers needInstanceTrio (which +// authorizes SSH on the default SG), so the default SG / subnet / route table +// are observed in their pristine, out-of-box state. +// +// Two facts they pin: +// +// - runMultinodeDefaultSGReachabilityBaseline: an instance in the default +// subnet behind a dedicated default-deny SG is NOT reachable from the +// runner until tcp/22 is authorized — the SG, not routing, is the gate. +// +// - runMultinodeSameSGCrossHostComms: two instances on DIFFERENT nodes that +// share the default SG can reach each other over the VPC datapath with no +// ingress rule added — the default SG's self-reference rule is enforced as +// an OVN address set spanning chassis. The probe is ICMP, which only the +// default SG's self-ingress permits, so the signal can't be confounded by +// the tcp/22 rule other tests add to the default SG later. +// +// Neither mutates a default resource: dedicated SGs are authorized, the +// default SG is only read / used for membership. + +// baselineLaunch launches one instance with the given SGs into subnetID and +// registers a terminate-and-wait cleanup. Returns the instance ID once +// "running". Self-cleaning so the cluster VM inventory is unperturbed for +// sibling tests. +func baselineLaunch(t *testing.T, fix *Fixture, amiID, instType, keyName, subnetID string, sgIDs []string) string { + t.Helper() + sgs := make([]*string, 0, len(sgIDs)) + for _, id := range sgIDs { + sgs = append(sgs, aws.String(id)) + } + input := &ec2.RunInstancesInput{ + ImageId: aws.String(amiID), + InstanceType: aws.String(instType), + KeyName: aws.String(keyName), + SubnetId: aws.String(subnetID), + SecurityGroupIds: sgs, + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + } + var id string + for attempt := 1; attempt <= 6; attempt++ { + out, err := fix.AWS.EC2.RunInstances(input) + if err == nil { + require.NotEmpty(t, out.Instances, "RunInstances returned no instances") + id = aws.StringValue(out.Instances[0].InstanceId) + break + } + if !strings.Contains(err.Error(), "InsufficientInstanceCapacity") { + t.Fatalf("RunInstances: %v", err) + } + t.Logf("baselineLaunch attempt %d: InsufficientInstanceCapacity, retrying", attempt) + time.Sleep(10 * time.Second) + } + require.NotEmpty(t, id, "RunInstances never succeeded") + t.Cleanup(func() { + _, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + harness.WaitForInstanceState(t, fix.AWS, id, "terminated") + }) + harness.WaitForInstanceState(t, fix.AWS, id, "running") + return id +} + +// instancePrivateIP returns the instance's primary VPC private IP. +func instancePrivateIP(t *testing.T, fix *Fixture, id string) string { + t.Helper() + out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + require.NoError(t, err, "describe-instances %s", id) + require.NotEmpty(t, out.Reservations, "no reservations for %s", id) + require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id) + ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress) + require.NotEmptyf(t, ip, "instance %s has no private IP", id) + return ip +} + +// sshCapture runs cmd over SSH and returns combined output + error without +// fataling, so callers can assert on the exit status (ping success/failure). +func sshCapture(pem, user, host string, port int, cmd string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + args := []string{ + "-i", pem, + "-p", strconv.Itoa(port), + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-o", "LogLevel=ERROR", + fmt.Sprintf("%s@%s", user, host), + cmd, + } + out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput() + return string(out), err +} + +// runMultinodeDefaultSGReachabilityBaseline asserts the default-deny SG gate +// on the public-IP datapath: blocked before authorize, reachable after. +func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { + harness.Phase(t, "Multinode — Baseline: default-deny SG blocks external reach until authorized") + + vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS) + instType, arch := needInstanceTypeArch(t, fix) + amiID := needAMI(t, fix, arch) + keyName, keyPath := needKeyPair(t, fix) + + sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg") + id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{sgID}) + + inst := harness.WaitForInstanceState(t, fix.AWS, id, "running") + pubIP := aws.StringValue(inst.PublicIpAddress) + if pubIP == "" || pubIP == "None" { + t.Fatalf("instance %s has no public IP; the datapath it depends on is "+ + "broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", id) + } + harness.Detail(t, "instance", id, "public_ip", pubIP, "sg", sgID) + + harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG") + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + conn, derr := net.DialTimeout("tcp", net.JoinHostPort(pubIP, "22"), 3*time.Second) + if derr == nil { + _ = conn.Close() + t.Fatalf("tcp/22 to %s connected with NO ingress rule — default-deny SG must block external traffic", pubIP) + } + time.Sleep(3 * time.Second) + } + + harness.Step(t, "authorizing tcp/22, expecting reachability") + harness.AuthorizeSSHIngress(t, fix.AWS, sgID) + harness.GuestSSHReady(t, pubIP, 22, "ec2-user", keyPath, + harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second)) + + out, err := sshCapture(keyPath, "ec2-user", pubIP, 22, "id") + require.NoErrorf(t, err, "ssh id after authorize: %s", out) + assert.Containsf(t, out, "ec2-user", "ssh id after authorize\n%s", out) +} + +// runMultinodeSameSGCrossHostComms launches two instances on different nodes, +// both in the default SG plus a dedicated runner-SSH SG, and asserts that one +// can ICMP-ping the other over the VPC. ICMP is permitted only by the default +// SG's self-reference rule, so success proves that rule is enforced across +// chassis with no default-SG mutation. +func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) { + harness.Phase(t, "Multinode — Baseline: same default-SG instances communicate across hosts") + + vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS) + instType, arch := needInstanceTypeArch(t, fix) + amiID := needAMI(t, fix, arch) + keyName, keyPath := needKeyPair(t, fix) + + // Dedicated SG so the runner can SSH into a probe instance without touching + // the default SG. Opens tcp/22 only — ICMP between guests still depends on + // the default SG's self-ingress, keeping the cross-host signal clean. + runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg") + harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG) + + // Launch instances until two land on distinct nodes (best-effort scheduler + // spread). Bounded so a degenerate single-node placement can't loop. + type placed struct { + id string + node string + } + var instances []placed + var srcIdx, dstIdx = -1, -1 + for attempt := 0; attempt < 4 && (srcIdx < 0 || dstIdx < 0); attempt++ { + id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{defSGID, runnerSG}) + node := harness.InstanceHostingNode(t, fix.Cluster, id) + nodeName := "" + if node != nil { + nodeName = node.Name + } + instances = append(instances, placed{id: id, node: nodeName}) + // Re-scan for a distinct-node pair. + srcIdx, dstIdx = -1, -1 + for i := range instances { + for j := range instances { + if i != j && instances[i].node != "" && instances[i].node != instances[j].node { + srcIdx, dstIdx = i, j + } + } + } + } + if srcIdx < 0 || dstIdx < 0 { + t.Skipf("could not place two instances on distinct nodes (got %v); scheduler colocated", instances) + } + src, dst := instances[srcIdx], instances[dstIdx] + harness.Detail(t, "src", src.id, "src_node", src.node, "dst", dst.id, "dst_node", dst.node) + + dstPriv := instancePrivateIP(t, fix, dst.id) + + // Shell into the source instance via its dedicated runner-SSH SG. + host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, src.id) + harness.GuestSSHReady(t, host, port, "ec2-user", keyPath, + harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second)) + + harness.Step(t, "ping %s (%s) from %s across hosts via default-SG self-ingress", dst.id, dstPriv, src.id) + out, err := sshCapture(keyPath, "ec2-user", host, port, + fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv)) + require.NoErrorf(t, err, + "cross-host ping %s -> %s failed; default SG self-ingress not enforced across chassis\n%s", + src.id, dst.id, out) + assert.Containsf(t, out, "0% packet loss", + "cross-host ping had loss; default SG self-ingress datapath degraded\n%s", out) +} diff --git a/tests/e2e/multinode/tests_test.go b/tests/e2e/multinode/tests_test.go index aa24f4d7c..4a979b64d 100644 --- a/tests/e2e/multinode/tests_test.go +++ b/tests/e2e/multinode/tests_test.go @@ -42,6 +42,19 @@ func TestMultinodePreflight(t *testing.T) { runPreflight(t, requireMultiNodeFixture(t)) } +// Fresh-install reachability baselines run right after preflight and before +// any test calls needInstanceTrio (which authorizes SSH on the default SG), +// so the default SG / subnet / route table are exercised pristine. Both own +// their mutable resources (dedicated SGs, self-cleaning instances) and never +// mutate a default resource. +func TestMultinodeDefaultSGReachabilityBaseline(t *testing.T) { + runMultinodeDefaultSGReachabilityBaseline(t, requireMultiNodeFixture(t)) +} + +func TestMultinodeSameSGCrossHostComms(t *testing.T) { + runMultinodeSameSGCrossHostComms(t, requireMultiNodeFixture(t)) +} + func TestMultinodeClusterHealth(t *testing.T) { t.Parallel() runClusterHealth(t, requireMultiNodeFixture(t)) diff --git a/tests/e2e/single/baseline_test.go b/tests/e2e/single/baseline_test.go new file mode 100644 index 000000000..80386cf37 --- /dev/null +++ b/tests/e2e/single/baseline_test.go @@ -0,0 +1,274 @@ +//go:build e2e + +package single + +import ( + "context" + "fmt" + "net" + "os/exec" + "strconv" + "time" + + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fresh-install reachability baselines. +// +// The rest of the suite calls harness.AuthorizeSSHIngress before every SSH +// probe, so it never exercises the out-of-box default configuration. These +// two tests pin the two independent barriers a real user hits when they +// launch an instance and expect to reach it: +// +// 1. SG barrier (runDefaultSGReachabilityBaseline): an instance in the +// fully-routed default subnet is NOT reachable from outside until an +// ingress rule is added — AWS default-SG semantics admit only same-SG +// members. Routes are fine; the SG is the gate. +// +// 2. Route/egress barrier (runNewSubnetPublicEgressBaseline): an instance +// in a freshly-created subnet with an open SG and a public IP must still +// be reachable, because the new subnet implicitly inherits the VPC main +// route table's 0.0.0.0/0 -> IGW route. If the per-subnet egress policy +// is only installed on route/association mutations (not on subnet +// creation), this fails — the subnet has no datapath path to the IGW. +// +// Both own all their resources (dedicated SG + instance, dedicated subnet) +// so they don't perturb the singleton VM or the shared default SG. + +// tcpReachable reports whether a TCP connect to host:port succeeds within +// timeout. An OVN ACL drop yields a dial timeout (no RST); a reject yields +// connection-refused — either way the connect fails and we return false. +func tcpReachable(host string, port int, timeout time.Duration) bool { + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +// launchBaselineInstance launches one instance into subnetID with the given +// SGs and registers a terminate-and-wait cleanup so the VM is gone before the +// next test runs. Unlike harness.EnsureInstance (which defers teardown to +// fixture cleanup at suite end), these baselines own their VMs for the test's +// duration only — they must not perturb cluster-VM-count assertions in +// sibling tests. +func launchBaselineInstance(t *testing.T, fix *Fixture, ami, instType, keyName, subnetID string, sgIDs []string) string { + t.Helper() + sgs := make([]*string, 0, len(sgIDs)) + for _, id := range sgIDs { + sgs = append(sgs, aws.String(id)) + } + out, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{ + ImageId: aws.String(ami), + InstanceType: aws.String(instType), + KeyName: aws.String(keyName), + SubnetId: aws.String(subnetID), + SecurityGroupIds: sgs, + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + }) + require.NoError(t, err, "RunInstances") + require.NotEmpty(t, out.Instances, "RunInstances returned no instances") + id := aws.StringValue(out.Instances[0].InstanceId) + t.Cleanup(func() { + _, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + harness.WaitForInstanceState(t, fix.AWS, id, "terminated") + }) + harness.WaitForInstanceState(t, fix.AWS, id, "running") + return id +} + +// instancePrivateIP returns the instance's primary VPC private IP. +func instancePrivateIP(t *testing.T, fix *Fixture, id string) string { + t.Helper() + out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(id)}, + }) + require.NoError(t, err, "describe-instances %s", id) + require.NotEmpty(t, out.Reservations, "no reservations for %s", id) + require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id) + ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress) + require.NotEmptyf(t, ip, "instance %s has no private IP", id) + return ip +} + +// sshCapture runs cmd over SSH and returns combined output + error without +// fataling, so callers can assert on the exit status (e.g. ping result). +func sshCapture(tgt harness.SSHTarget, cmd string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + args := []string{ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-p", strconv.Itoa(tgt.Port), + "-i", tgt.KeyPath, + tgt.User + "@" + tgt.Host, + cmd, + } + out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput() + return string(out), err +} + +// instancePublicIP returns the instance's routable public IP. A missing public +// IP is a hard failure: these baselines must exercise the real OVN datapath, +// and the qemu-hostfwd shortcut (which would otherwise stand in) is disabled +// suite-wide because it bypasses SG ACLs / IGW / SNAT and masks regressions. +func instancePublicIP(t *testing.T, fix *Fixture, instanceID string) string { + t.Helper() + out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances %s", instanceID) + require.NotEmpty(t, out.Reservations, "no reservations for %s", instanceID) + require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", instanceID) + ip := aws.StringValue(out.Reservations[0].Instances[0].PublicIpAddress) + if ip == "" || ip == "None" { + t.Fatalf("instance %s has no public IP; the datapath it depends on is "+ + "broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", instanceID) + } + return ip +} + +// runDefaultSGReachabilityBaseline launches an instance into the default +// subnet behind a dedicated default-deny SG, asserts it is unreachable from +// the runner, then opens tcp/22 and asserts it becomes reachable. Confirms +// the SG — not routing — gates a fresh default-config instance. +func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { + harness.Phase(t, "Single — Baseline: default-SG blocks external reach until authorized") + + vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS) + instType, _ := needInstanceTypeArch(t, fix) + keyName, keyPath := needKeyPair(t, fix) + ami := needAMI(t, fix) + + // Dedicated SG with NO ingress rules. CreateSecurityGroup seeds allow-all + // egress, so the SSH return path is unaffected — ingress is the only gate. + sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg") + harness.Detail(t, "vpc", vpcID, "subnet", subnetID, "sg", sgID) + + instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID}) + + pubIP := instancePublicIP(t, fix, instanceID) + harness.Detail(t, "instance", instanceID, "public_ip", pubIP) + + // Phase A — default-deny SG: tcp/22 must stay blocked. Poll for a window + // so a slow datapath converging to "open" would still be caught. + harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG") + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + require.Falsef(t, tcpReachable(pubIP, 22, 3*time.Second), + "tcp/22 to %s connected with NO ingress rule — default SG must deny external traffic", pubIP) + time.Sleep(3 * time.Second) + } + + // Phase B — authorize tcp/22, then a full handshake must succeed. Proves + // the subnet routes to the IGW and only the SG was gating reachability. + harness.Step(t, "authorizing tcp/22 ingress, expecting reachability") + harness.AuthorizeSSHIngress(t, fix.AWS, sgID) + require.Truef(t, trySSHReady(pubIP, 22, keyPath, 3*time.Minute), + "tcp/22 to %s never became reachable after authorizing ingress — "+ + "default subnet egress/IGW datapath is broken", pubIP) + + tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath} + idOut := runSSH(t, tgt, "id") + assert.Containsf(t, idOut, "ec2-user", "ssh id after authorize\n%s", idOut) +} + +// runNewSubnetPublicEgressBaseline launches an instance into a freshly-created +// subnet (MapPublicIpOnLaunch=true, open SG) and asserts external SSH works. +// The new subnet has no explicit RT association, so it inherits the VPC main +// route table's 0.0.0.0/0 -> IGW route. If the per-subnet egress policy is not +// installed at subnet-create time, the instance has no datapath to the IGW and +// this fails — the SG is open, so the only remaining barrier is routing. +func runNewSubnetPublicEgressBaseline(t *testing.T, fix *Fixture) { + harness.Phase(t, "Single — Baseline: new public subnet inherits IGW egress") + + vpcID, _, _ := harness.DiscoverDefaultVPC(t, fix.AWS) + az := needAZ(t, fix) + instType, _ := needInstanceTypeArch(t, fix) + keyName, keyPath := needKeyPair(t, fix) + ami := needAMI(t, fix) + + // High /24 inside the default VPC CIDR (172.31.0.0/16); clear of the + // default subnet which sits at the low end. + const newSubnetCIDR = "172.31.250.0/24" + subnetID := harness.EnsureSubnet(t, fix.Harness, vpcID, newSubnetCIDR, az) + + _, err := fix.AWS.EC2.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + SubnetId: aws.String(subnetID), + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{Value: aws.Bool(true)}, + }) + require.NoError(t, err, "ModifySubnetAttribute MapPublicIpOnLaunch on %s", subnetID) + + // Open SG so the SG is provably not the barrier — only routing remains. + sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-opensg") + harness.AuthorizeSSHIngress(t, fix.AWS, sgID) + harness.Detail(t, "vpc", vpcID, "new_subnet", subnetID, "cidr", newSubnetCIDR, "sg", sgID) + + instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID}) + + pubIP := instancePublicIP(t, fix, instanceID) + harness.Detail(t, "instance", instanceID, "public_ip", pubIP) + + harness.Step(t, "expecting external SSH to a new public subnet's instance") + require.Truef(t, trySSHReady(pubIP, 22, keyPath, 3*time.Minute), + "tcp/22 to %s in new subnet %s never became reachable — the implicit "+ + "main-RT IGW egress policy was not installed for a subnet created "+ + "after the route existed", pubIP, subnetID) + + tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath} + idOut := runSSH(t, tgt, "id") + assert.Containsf(t, idOut, "ec2-user", "ssh id in new subnet\n%s", idOut) +} + +// runSameSGComms launches two instances in the default SG (plus a dedicated +// runner-SSH SG so the runner can shell into one) and asserts that one can +// ICMP-ping the other over the VPC. ICMP between them is permitted only by the +// default SG's self-reference rule, so success proves that rule is enforced on +// the OVN datapath — the east-west counterpart to the external-reach baseline. +// No default resource is mutated (only the dedicated runner SG is authorized). +func runSameSGComms(t *testing.T, fix *Fixture) { + harness.Phase(t, "Single — Baseline: same default-SG instances communicate") + + vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS) + instType, _ := needInstanceTypeArch(t, fix) + keyName, keyPath := needKeyPair(t, fix) + ami := needAMI(t, fix) + + // Dedicated SG opens tcp/22 only, so ICMP between guests still depends on + // the default SG's self-ingress — keeps the signal unambiguous. + runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg") + harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG) + harness.Detail(t, "vpc", vpcID, "default_sg", defSGID, "runner_sg", runnerSG) + + srcID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG}) + dstID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG}) + + dstPriv := instancePrivateIP(t, fix, dstID) + + srcInst := harness.WaitForInstanceState(t, fix.AWS, srcID, "running") + host, port := harness.InstancePublicSSHHost(t, srcInst) + harness.Detail(t, "src", srcID, "dst", dstID, "dst_private_ip", dstPriv, "ssh_host", host) + waitForSSHHandshake(t, host, port, keyPath) + + tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + harness.Step(t, "ping %s (%s) from %s via default-SG self-ingress", dstID, dstPriv, srcID) + out, err := sshCapture(tgt, fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv)) + require.NoErrorf(t, err, + "intra-default-SG ping %s -> %s failed; self-ingress not enforced on the datapath\n%s", + srcID, dstID, out) + assert.Containsf(t, out, "0% packet loss", + "intra-default-SG ping had loss; self-ingress datapath degraded\n%s", out) +} diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index 9f8569cde..d33dfd5db 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -94,6 +94,22 @@ func TestClusterStatsCLI(t *testing.T) { runClusterStatsCLI(t, requireSingleNodeFixture(t)) } +// Fresh-install reachability baselines run right after the 0-VM cluster-stats +// gate and before the singleton launch. They own all mutable resources +// (dedicated SG / subnet / self-cleaning instance) and never mutate the +// default SG, subnet, or route table. +func TestDefaultSGReachabilityBaseline(t *testing.T) { + runDefaultSGReachabilityBaseline(t, requireSingleNodeFixture(t)) +} + +func TestNewSubnetPublicEgressBaseline(t *testing.T) { + runNewSubnetPublicEgressBaseline(t, requireSingleNodeFixture(t)) +} + +func TestSameSGComms(t *testing.T) { + runSameSGComms(t, requireSingleNodeFixture(t)) +} + func TestInstanceLaunch(t *testing.T) { runInstanceLaunch(t, requireSingleNodeFixture(t)) }