mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
test(e2e): fresh-install reachability baselines + drop qemu hostfwd fallback
Add baseline tests that confirm two fresh-install regressions the existing
suite masks (it authorizes tcp/22 on the default SG before every SSH probe):
- default SG default-deny gates external reach of instances in the
default subnet (SG, not routing)
- a newly-created public subnet never gets its OVN IGW egress LRP, so it
has no route to the VPC IGW despite MapPublicIpOnLaunch
Plus a positive test: instances sharing the default SG can communicate via
ICMP (same-host single-node, cross-host multinode).
Single-node baselines run after TestClusterStatsCLI (0-VM gate); multinode
after TestMultinodePreflight (fixture init). Instances are self-cleaning so
they don't perturb the singleton-VM ordering.
Remove the qemu hostfwd SSH fallback from the harness (InstancePublicSSHHost,
GuestSSHEndpoint) - it bypasses the OVN datapath and masks these regressions.
Endpoints now t.Fatal when no public IP is assigned.
This commit is contained in:
joshsiv-mulga
@@ -4,9 +4,7 @@ package harness
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -28,59 +26,27 @@ type SSHTarget struct {
|
||||
KeyPath string
|
||||
}
|
||||
|
||||
// InstancePublicSSHHost returns (host, port) for an SSH connection to inst.
|
||||
// InstancePublicSSHHost returns (host, 22) for an SSH connection to inst's
|
||||
// public IP.
|
||||
//
|
||||
// Preference order matches run-e2e.sh:
|
||||
// 1. Instance's public IP on port 22, when assigned (external networking).
|
||||
// 2. QEMU hostfwd discovered by scanning `ps -ef` for the qemu process whose
|
||||
// command line contains the instance ID and a `hostfwd=tcp:<ip>:<port>-:22`
|
||||
// argument (dev_networking mode).
|
||||
//
|
||||
// t.Fatal on neither resolving — both phases need a usable SSH endpoint.
|
||||
// There is deliberately NO qemu-hostfwd fallback. hostfwd (127.0.0.1 ->
|
||||
// guest:22) is a dev_networking shortcut that bypasses the OVN datapath
|
||||
// entirely — no SG ACL, no IGW, no SNAT/DNAT — so a test that reached a guest
|
||||
// through it would validate nothing real and mask exactly the networking
|
||||
// regressions these tests exist to catch. An instance with no public IP is a
|
||||
// hard failure: the routing it depends on is broken or unconfigured.
|
||||
func InstancePublicSSHHost(t *testing.T, inst *ec2.Instance) (string, int) {
|
||||
t.Helper()
|
||||
if inst == nil {
|
||||
t.Fatalf("InstancePublicSSHHost: nil instance")
|
||||
}
|
||||
if pub := aws.StringValue(inst.PublicIpAddress); pub != "" && pub != "None" {
|
||||
return pub, 22
|
||||
pub := aws.StringValue(inst.PublicIpAddress)
|
||||
if pub == "" || pub == "None" {
|
||||
t.Fatalf("InstancePublicSSHHost: instance %s has no public IP; "+
|
||||
"qemu-hostfwd fallback is disabled (it bypasses the OVN datapath)",
|
||||
aws.StringValue(inst.InstanceId))
|
||||
}
|
||||
id := aws.StringValue(inst.InstanceId)
|
||||
host, port, err := discoverHostfwd(id)
|
||||
if err != nil {
|
||||
t.Fatalf("InstancePublicSSHHost: %v", err)
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
|
||||
// hostfwd=tcp:HOST:PORT-:22 — HOST may be empty (qemu defaults to 127.0.0.1).
|
||||
// Anchor on -:22 so we ignore other hostfwd rules (e.g. metadata, serial).
|
||||
var hostfwdRE = regexp.MustCompile(`hostfwd=tcp:([^:]*):([0-9]+)-:22`)
|
||||
|
||||
func discoverHostfwd(instanceID string) (string, int, error) {
|
||||
out, err := exec.Command("ps", "-eo", "args=").CombinedOutput()
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("ps: %w", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if !strings.Contains(line, instanceID) || !strings.Contains(line, "qemu") {
|
||||
continue
|
||||
}
|
||||
m := hostfwdRE.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
host := m[1]
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
port, err := strconv.Atoi(m[2])
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("parse hostfwd port %q: %w", m[2], err)
|
||||
}
|
||||
return host, port, nil
|
||||
}
|
||||
return "", 0, fmt.Errorf("no qemu hostfwd found for %s", instanceID)
|
||||
return pub, 22
|
||||
}
|
||||
|
||||
// LsblkRootGiB SSHes into the VM and returns the root disk size in GiB,
|
||||
|
||||
@@ -14,63 +14,23 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
)
|
||||
|
||||
// GuestSSHPort extracts the QEMU hostfwd port for an instance from the
|
||||
// hosting node's qemu process command line. Reuses ec2helpers.hostfwdRE
|
||||
// (same `hostfwd=tcp:HOST:PORT-:22` pattern). Polls until the qemu
|
||||
// process appears (the daemon may still be spawning at the time of the
|
||||
// first ssh in). Returns 0 if the port is never observable within the
|
||||
// poll window.
|
||||
func GuestSSHPort(t *testing.T, hostNode Node, instanceID string, opts ...PollOpt) int {
|
||||
t.Helper()
|
||||
cfg := applyOpts(pollCfg{timeout: 30 * time.Second, interval: 1 * time.Second}, opts...)
|
||||
ssh := NewPeerSSH()
|
||||
var port int
|
||||
EventuallyErr(t, func() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.interval+5*time.Second)
|
||||
defer cancel()
|
||||
out, err := runPSGrep(ctx, ssh, hostNode, instanceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s ps grep: %w", hostNode.Name, err)
|
||||
}
|
||||
m := hostfwdRE.FindStringSubmatch(out)
|
||||
if len(m) < 3 {
|
||||
return fmt.Errorf("%s no hostfwd port for %s yet", hostNode.Name, instanceID)
|
||||
}
|
||||
p, err := strconv.Atoi(m[2])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s parse hostfwd port %q: %w", hostNode.Name, m[2], err)
|
||||
}
|
||||
port = p
|
||||
return nil
|
||||
}, cfg.timeout, cfg.interval)
|
||||
return port
|
||||
}
|
||||
|
||||
// GuestSSHEndpoint resolves the (host, port) pair for SSH-ing into a guest
|
||||
// VM. Prefers the instance's PublicIpAddress when set; falls back to the
|
||||
// hosting node's WAN IP + QEMU hostfwd port. Mirrors the conditional in
|
||||
// run-multinode-e2e.sh phase 4.
|
||||
// GuestSSHEndpoint resolves (host, 22) for SSH-ing into a guest VM via its
|
||||
// public IP.
|
||||
//
|
||||
// Returns (host, port, nil) on success; t.Fatal on full failure. Public-IP
|
||||
// path is the steady-state once OVN's public subnet is wired; the
|
||||
// hostfwd path is the legacy fallback for VPCs without public routing.
|
||||
// There is deliberately NO qemu-hostfwd fallback (see InstancePublicSSHHost):
|
||||
// hostfwd bypasses the OVN datapath, so reaching a guest through it would mask
|
||||
// the networking regressions these tests exist to catch. A running instance
|
||||
// with no public IP is a hard failure.
|
||||
func GuestSSHEndpoint(t *testing.T, c *AWSClient, cluster *Cluster, instanceID string, opts ...PollOpt) (host string, port int) {
|
||||
t.Helper()
|
||||
|
||||
_ = cluster // retained for signature stability; node hostfwd lookup removed
|
||||
inst := WaitForInstanceState(t, c, instanceID, "running", opts...)
|
||||
if pub := aws.StringValue(inst.PublicIpAddress); pub != "" {
|
||||
return pub, 22
|
||||
pub := aws.StringValue(inst.PublicIpAddress)
|
||||
if pub == "" {
|
||||
t.Fatalf("GuestSSHEndpoint: instance %s has no public IP; "+
|
||||
"qemu-hostfwd fallback is disabled (it bypasses the OVN datapath)", instanceID)
|
||||
}
|
||||
|
||||
hostNode := InstanceHostingNode(t, cluster, instanceID)
|
||||
if hostNode == nil {
|
||||
t.Fatalf("GuestSSHEndpoint: no node hosts %s", instanceID)
|
||||
}
|
||||
port = GuestSSHPort(t, *hostNode, instanceID, opts...)
|
||||
if port == 0 {
|
||||
t.Fatalf("GuestSSHEndpoint: hostfwd port not resolved for %s on %s", instanceID, hostNode.Name)
|
||||
}
|
||||
return hostNode.Addr, port
|
||||
return pub, 22
|
||||
}
|
||||
|
||||
// GuestSSHReady polls SSH into the named instance until a probe command
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
//go:build e2e
|
||||
|
||||
package multinode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Fresh-install reachability baselines for the multi-node cluster. These run
|
||||
// right after preflight and before any test triggers needInstanceTrio (which
|
||||
// authorizes SSH on the default SG), so the default SG / subnet / route table
|
||||
// are observed in their pristine, out-of-box state.
|
||||
//
|
||||
// Two facts they pin:
|
||||
//
|
||||
// - runMultinodeDefaultSGReachabilityBaseline: an instance in the default
|
||||
// subnet behind a dedicated default-deny SG is NOT reachable from the
|
||||
// runner until tcp/22 is authorized — the SG, not routing, is the gate.
|
||||
//
|
||||
// - runMultinodeSameSGCrossHostComms: two instances on DIFFERENT nodes that
|
||||
// share the default SG can reach each other over the VPC datapath with no
|
||||
// ingress rule added — the default SG's self-reference rule is enforced as
|
||||
// an OVN address set spanning chassis. The probe is ICMP, which only the
|
||||
// default SG's self-ingress permits, so the signal can't be confounded by
|
||||
// the tcp/22 rule other tests add to the default SG later.
|
||||
//
|
||||
// Neither mutates a default resource: dedicated SGs are authorized, the
|
||||
// default SG is only read / used for membership.
|
||||
|
||||
// baselineLaunch launches one instance with the given SGs into subnetID and
|
||||
// registers a terminate-and-wait cleanup. Returns the instance ID once
|
||||
// "running". Self-cleaning so the cluster VM inventory is unperturbed for
|
||||
// sibling tests.
|
||||
func baselineLaunch(t *testing.T, fix *Fixture, amiID, instType, keyName, subnetID string, sgIDs []string) string {
|
||||
t.Helper()
|
||||
sgs := make([]*string, 0, len(sgIDs))
|
||||
for _, id := range sgIDs {
|
||||
sgs = append(sgs, aws.String(id))
|
||||
}
|
||||
input := &ec2.RunInstancesInput{
|
||||
ImageId: aws.String(amiID),
|
||||
InstanceType: aws.String(instType),
|
||||
KeyName: aws.String(keyName),
|
||||
SubnetId: aws.String(subnetID),
|
||||
SecurityGroupIds: sgs,
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
}
|
||||
var id string
|
||||
for attempt := 1; attempt <= 6; attempt++ {
|
||||
out, err := fix.AWS.EC2.RunInstances(input)
|
||||
if err == nil {
|
||||
require.NotEmpty(t, out.Instances, "RunInstances returned no instances")
|
||||
id = aws.StringValue(out.Instances[0].InstanceId)
|
||||
break
|
||||
}
|
||||
if !strings.Contains(err.Error(), "InsufficientInstanceCapacity") {
|
||||
t.Fatalf("RunInstances: %v", err)
|
||||
}
|
||||
t.Logf("baselineLaunch attempt %d: InsufficientInstanceCapacity, retrying", attempt)
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
require.NotEmpty(t, id, "RunInstances never succeeded")
|
||||
t.Cleanup(func() {
|
||||
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id)},
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id, "terminated")
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id, "running")
|
||||
return id
|
||||
}
|
||||
|
||||
// instancePrivateIP returns the instance's primary VPC private IP.
|
||||
func instancePrivateIP(t *testing.T, fix *Fixture, id string) string {
|
||||
t.Helper()
|
||||
out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id)},
|
||||
})
|
||||
require.NoError(t, err, "describe-instances %s", id)
|
||||
require.NotEmpty(t, out.Reservations, "no reservations for %s", id)
|
||||
require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id)
|
||||
ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress)
|
||||
require.NotEmptyf(t, ip, "instance %s has no private IP", id)
|
||||
return ip
|
||||
}
|
||||
|
||||
// sshCapture runs cmd over SSH and returns combined output + error without
|
||||
// fataling, so callers can assert on the exit status (ping success/failure).
|
||||
func sshCapture(pem, user, host string, port int, cmd string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
args := []string{
|
||||
"-i", pem,
|
||||
"-p", strconv.Itoa(port),
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "LogLevel=ERROR",
|
||||
fmt.Sprintf("%s@%s", user, host),
|
||||
cmd,
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// runMultinodeDefaultSGReachabilityBaseline asserts the default-deny SG gate
|
||||
// on the public-IP datapath: blocked before authorize, reachable after.
|
||||
func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Multinode — Baseline: default-deny SG blocks external reach until authorized")
|
||||
|
||||
vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
||||
instType, arch := needInstanceTypeArch(t, fix)
|
||||
amiID := needAMI(t, fix, arch)
|
||||
keyName, keyPath := needKeyPair(t, fix)
|
||||
|
||||
sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg")
|
||||
id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{sgID})
|
||||
|
||||
inst := harness.WaitForInstanceState(t, fix.AWS, id, "running")
|
||||
pubIP := aws.StringValue(inst.PublicIpAddress)
|
||||
if pubIP == "" || pubIP == "None" {
|
||||
t.Fatalf("instance %s has no public IP; the datapath it depends on is "+
|
||||
"broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", id)
|
||||
}
|
||||
harness.Detail(t, "instance", id, "public_ip", pubIP, "sg", sgID)
|
||||
|
||||
harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, derr := net.DialTimeout("tcp", net.JoinHostPort(pubIP, "22"), 3*time.Second)
|
||||
if derr == nil {
|
||||
_ = conn.Close()
|
||||
t.Fatalf("tcp/22 to %s connected with NO ingress rule — default-deny SG must block external traffic", pubIP)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
harness.Step(t, "authorizing tcp/22, expecting reachability")
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
|
||||
harness.GuestSSHReady(t, pubIP, 22, "ec2-user", keyPath,
|
||||
harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second))
|
||||
|
||||
out, err := sshCapture(keyPath, "ec2-user", pubIP, 22, "id")
|
||||
require.NoErrorf(t, err, "ssh id after authorize: %s", out)
|
||||
assert.Containsf(t, out, "ec2-user", "ssh id after authorize\n%s", out)
|
||||
}
|
||||
|
||||
// runMultinodeSameSGCrossHostComms launches two instances on different nodes,
|
||||
// both in the default SG plus a dedicated runner-SSH SG, and asserts that one
|
||||
// can ICMP-ping the other over the VPC. ICMP is permitted only by the default
|
||||
// SG's self-reference rule, so success proves that rule is enforced across
|
||||
// chassis with no default-SG mutation.
|
||||
func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Multinode — Baseline: same default-SG instances communicate across hosts")
|
||||
|
||||
vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
||||
instType, arch := needInstanceTypeArch(t, fix)
|
||||
amiID := needAMI(t, fix, arch)
|
||||
keyName, keyPath := needKeyPair(t, fix)
|
||||
|
||||
// Dedicated SG so the runner can SSH into a probe instance without touching
|
||||
// the default SG. Opens tcp/22 only — ICMP between guests still depends on
|
||||
// the default SG's self-ingress, keeping the cross-host signal clean.
|
||||
runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg")
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG)
|
||||
|
||||
// Launch instances until two land on distinct nodes (best-effort scheduler
|
||||
// spread). Bounded so a degenerate single-node placement can't loop.
|
||||
type placed struct {
|
||||
id string
|
||||
node string
|
||||
}
|
||||
var instances []placed
|
||||
var srcIdx, dstIdx = -1, -1
|
||||
for attempt := 0; attempt < 4 && (srcIdx < 0 || dstIdx < 0); attempt++ {
|
||||
id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{defSGID, runnerSG})
|
||||
node := harness.InstanceHostingNode(t, fix.Cluster, id)
|
||||
nodeName := ""
|
||||
if node != nil {
|
||||
nodeName = node.Name
|
||||
}
|
||||
instances = append(instances, placed{id: id, node: nodeName})
|
||||
// Re-scan for a distinct-node pair.
|
||||
srcIdx, dstIdx = -1, -1
|
||||
for i := range instances {
|
||||
for j := range instances {
|
||||
if i != j && instances[i].node != "" && instances[i].node != instances[j].node {
|
||||
srcIdx, dstIdx = i, j
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if srcIdx < 0 || dstIdx < 0 {
|
||||
t.Skipf("could not place two instances on distinct nodes (got %v); scheduler colocated", instances)
|
||||
}
|
||||
src, dst := instances[srcIdx], instances[dstIdx]
|
||||
harness.Detail(t, "src", src.id, "src_node", src.node, "dst", dst.id, "dst_node", dst.node)
|
||||
|
||||
dstPriv := instancePrivateIP(t, fix, dst.id)
|
||||
|
||||
// Shell into the source instance via its dedicated runner-SSH SG.
|
||||
host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, src.id)
|
||||
harness.GuestSSHReady(t, host, port, "ec2-user", keyPath,
|
||||
harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second))
|
||||
|
||||
harness.Step(t, "ping %s (%s) from %s across hosts via default-SG self-ingress", dst.id, dstPriv, src.id)
|
||||
out, err := sshCapture(keyPath, "ec2-user", host, port,
|
||||
fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv))
|
||||
require.NoErrorf(t, err,
|
||||
"cross-host ping %s -> %s failed; default SG self-ingress not enforced across chassis\n%s",
|
||||
src.id, dst.id, out)
|
||||
assert.Containsf(t, out, "0% packet loss",
|
||||
"cross-host ping had loss; default SG self-ingress datapath degraded\n%s", out)
|
||||
}
|
||||
@@ -42,6 +42,19 @@ func TestMultinodePreflight(t *testing.T) {
|
||||
runPreflight(t, requireMultiNodeFixture(t))
|
||||
}
|
||||
|
||||
// Fresh-install reachability baselines run right after preflight and before
|
||||
// any test calls needInstanceTrio (which authorizes SSH on the default SG),
|
||||
// so the default SG / subnet / route table are exercised pristine. Both own
|
||||
// their mutable resources (dedicated SGs, self-cleaning instances) and never
|
||||
// mutate a default resource.
|
||||
func TestMultinodeDefaultSGReachabilityBaseline(t *testing.T) {
|
||||
runMultinodeDefaultSGReachabilityBaseline(t, requireMultiNodeFixture(t))
|
||||
}
|
||||
|
||||
func TestMultinodeSameSGCrossHostComms(t *testing.T) {
|
||||
runMultinodeSameSGCrossHostComms(t, requireMultiNodeFixture(t))
|
||||
}
|
||||
|
||||
func TestMultinodeClusterHealth(t *testing.T) {
|
||||
t.Parallel()
|
||||
runClusterHealth(t, requireMultiNodeFixture(t))
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
//go:build e2e
|
||||
|
||||
package single
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Fresh-install reachability baselines.
|
||||
//
|
||||
// The rest of the suite calls harness.AuthorizeSSHIngress before every SSH
|
||||
// probe, so it never exercises the out-of-box default configuration. These
|
||||
// two tests pin the two independent barriers a real user hits when they
|
||||
// launch an instance and expect to reach it:
|
||||
//
|
||||
// 1. SG barrier (runDefaultSGReachabilityBaseline): an instance in the
|
||||
// fully-routed default subnet is NOT reachable from outside until an
|
||||
// ingress rule is added — AWS default-SG semantics admit only same-SG
|
||||
// members. Routes are fine; the SG is the gate.
|
||||
//
|
||||
// 2. Route/egress barrier (runNewSubnetPublicEgressBaseline): an instance
|
||||
// in a freshly-created subnet with an open SG and a public IP must still
|
||||
// be reachable, because the new subnet implicitly inherits the VPC main
|
||||
// route table's 0.0.0.0/0 -> IGW route. If the per-subnet egress policy
|
||||
// is only installed on route/association mutations (not on subnet
|
||||
// creation), this fails — the subnet has no datapath path to the IGW.
|
||||
//
|
||||
// Both own all their resources (dedicated SG + instance, dedicated subnet)
|
||||
// so they don't perturb the singleton VM or the shared default SG.
|
||||
|
||||
// tcpReachable reports whether a TCP connect to host:port succeeds within
|
||||
// timeout. An OVN ACL drop yields a dial timeout (no RST); a reject yields
|
||||
// connection-refused — either way the connect fails and we return false.
|
||||
func tcpReachable(host string, port int, timeout time.Duration) bool {
|
||||
conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// launchBaselineInstance launches one instance into subnetID with the given
|
||||
// SGs and registers a terminate-and-wait cleanup so the VM is gone before the
|
||||
// next test runs. Unlike harness.EnsureInstance (which defers teardown to
|
||||
// fixture cleanup at suite end), these baselines own their VMs for the test's
|
||||
// duration only — they must not perturb cluster-VM-count assertions in
|
||||
// sibling tests.
|
||||
func launchBaselineInstance(t *testing.T, fix *Fixture, ami, instType, keyName, subnetID string, sgIDs []string) string {
|
||||
t.Helper()
|
||||
sgs := make([]*string, 0, len(sgIDs))
|
||||
for _, id := range sgIDs {
|
||||
sgs = append(sgs, aws.String(id))
|
||||
}
|
||||
out, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{
|
||||
ImageId: aws.String(ami),
|
||||
InstanceType: aws.String(instType),
|
||||
KeyName: aws.String(keyName),
|
||||
SubnetId: aws.String(subnetID),
|
||||
SecurityGroupIds: sgs,
|
||||
MinCount: aws.Int64(1),
|
||||
MaxCount: aws.Int64(1),
|
||||
})
|
||||
require.NoError(t, err, "RunInstances")
|
||||
require.NotEmpty(t, out.Instances, "RunInstances returned no instances")
|
||||
id := aws.StringValue(out.Instances[0].InstanceId)
|
||||
t.Cleanup(func() {
|
||||
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id)},
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id, "terminated")
|
||||
})
|
||||
harness.WaitForInstanceState(t, fix.AWS, id, "running")
|
||||
return id
|
||||
}
|
||||
|
||||
// instancePrivateIP returns the instance's primary VPC private IP.
|
||||
func instancePrivateIP(t *testing.T, fix *Fixture, id string) string {
|
||||
t.Helper()
|
||||
out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{aws.String(id)},
|
||||
})
|
||||
require.NoError(t, err, "describe-instances %s", id)
|
||||
require.NotEmpty(t, out.Reservations, "no reservations for %s", id)
|
||||
require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id)
|
||||
ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress)
|
||||
require.NotEmptyf(t, ip, "instance %s has no private IP", id)
|
||||
return ip
|
||||
}
|
||||
|
||||
// sshCapture runs cmd over SSH and returns combined output + error without
|
||||
// fataling, so callers can assert on the exit status (e.g. ping result).
|
||||
func sshCapture(tgt harness.SSHTarget, cmd string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
args := []string{
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes",
|
||||
"-p", strconv.Itoa(tgt.Port),
|
||||
"-i", tgt.KeyPath,
|
||||
tgt.User + "@" + tgt.Host,
|
||||
cmd,
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
// instancePublicIP returns the instance's routable public IP. A missing public
|
||||
// IP is a hard failure: these baselines must exercise the real OVN datapath,
|
||||
// and the qemu-hostfwd shortcut (which would otherwise stand in) is disabled
|
||||
// suite-wide because it bypasses SG ACLs / IGW / SNAT and masks regressions.
|
||||
func instancePublicIP(t *testing.T, fix *Fixture, instanceID string) string {
|
||||
t.Helper()
|
||||
out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{aws.String(instanceID)},
|
||||
})
|
||||
require.NoError(t, err, "describe-instances %s", instanceID)
|
||||
require.NotEmpty(t, out.Reservations, "no reservations for %s", instanceID)
|
||||
require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", instanceID)
|
||||
ip := aws.StringValue(out.Reservations[0].Instances[0].PublicIpAddress)
|
||||
if ip == "" || ip == "None" {
|
||||
t.Fatalf("instance %s has no public IP; the datapath it depends on is "+
|
||||
"broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", instanceID)
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// runDefaultSGReachabilityBaseline launches an instance into the default
|
||||
// subnet behind a dedicated default-deny SG, asserts it is unreachable from
|
||||
// the runner, then opens tcp/22 and asserts it becomes reachable. Confirms
|
||||
// the SG — not routing — gates a fresh default-config instance.
|
||||
func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Single — Baseline: default-SG blocks external reach until authorized")
|
||||
|
||||
vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
||||
instType, _ := needInstanceTypeArch(t, fix)
|
||||
keyName, keyPath := needKeyPair(t, fix)
|
||||
ami := needAMI(t, fix)
|
||||
|
||||
// Dedicated SG with NO ingress rules. CreateSecurityGroup seeds allow-all
|
||||
// egress, so the SSH return path is unaffected — ingress is the only gate.
|
||||
sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg")
|
||||
harness.Detail(t, "vpc", vpcID, "subnet", subnetID, "sg", sgID)
|
||||
|
||||
instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID})
|
||||
|
||||
pubIP := instancePublicIP(t, fix, instanceID)
|
||||
harness.Detail(t, "instance", instanceID, "public_ip", pubIP)
|
||||
|
||||
// Phase A — default-deny SG: tcp/22 must stay blocked. Poll for a window
|
||||
// so a slow datapath converging to "open" would still be caught.
|
||||
harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
require.Falsef(t, tcpReachable(pubIP, 22, 3*time.Second),
|
||||
"tcp/22 to %s connected with NO ingress rule — default SG must deny external traffic", pubIP)
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
// Phase B — authorize tcp/22, then a full handshake must succeed. Proves
|
||||
// the subnet routes to the IGW and only the SG was gating reachability.
|
||||
harness.Step(t, "authorizing tcp/22 ingress, expecting reachability")
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
|
||||
require.Truef(t, trySSHReady(pubIP, 22, keyPath, 3*time.Minute),
|
||||
"tcp/22 to %s never became reachable after authorizing ingress — "+
|
||||
"default subnet egress/IGW datapath is broken", pubIP)
|
||||
|
||||
tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath}
|
||||
idOut := runSSH(t, tgt, "id")
|
||||
assert.Containsf(t, idOut, "ec2-user", "ssh id after authorize\n%s", idOut)
|
||||
}
|
||||
|
||||
// runNewSubnetPublicEgressBaseline launches an instance into a freshly-created
|
||||
// subnet (MapPublicIpOnLaunch=true, open SG) and asserts external SSH works.
|
||||
// The new subnet has no explicit RT association, so it inherits the VPC main
|
||||
// route table's 0.0.0.0/0 -> IGW route. If the per-subnet egress policy is not
|
||||
// installed at subnet-create time, the instance has no datapath to the IGW and
|
||||
// this fails — the SG is open, so the only remaining barrier is routing.
|
||||
func runNewSubnetPublicEgressBaseline(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Single — Baseline: new public subnet inherits IGW egress")
|
||||
|
||||
vpcID, _, _ := harness.DiscoverDefaultVPC(t, fix.AWS)
|
||||
az := needAZ(t, fix)
|
||||
instType, _ := needInstanceTypeArch(t, fix)
|
||||
keyName, keyPath := needKeyPair(t, fix)
|
||||
ami := needAMI(t, fix)
|
||||
|
||||
// High /24 inside the default VPC CIDR (172.31.0.0/16); clear of the
|
||||
// default subnet which sits at the low end.
|
||||
const newSubnetCIDR = "172.31.250.0/24"
|
||||
subnetID := harness.EnsureSubnet(t, fix.Harness, vpcID, newSubnetCIDR, az)
|
||||
|
||||
_, err := fix.AWS.EC2.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{
|
||||
SubnetId: aws.String(subnetID),
|
||||
MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
|
||||
})
|
||||
require.NoError(t, err, "ModifySubnetAttribute MapPublicIpOnLaunch on %s", subnetID)
|
||||
|
||||
// Open SG so the SG is provably not the barrier — only routing remains.
|
||||
sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-opensg")
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
|
||||
harness.Detail(t, "vpc", vpcID, "new_subnet", subnetID, "cidr", newSubnetCIDR, "sg", sgID)
|
||||
|
||||
instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID})
|
||||
|
||||
pubIP := instancePublicIP(t, fix, instanceID)
|
||||
harness.Detail(t, "instance", instanceID, "public_ip", pubIP)
|
||||
|
||||
harness.Step(t, "expecting external SSH to a new public subnet's instance")
|
||||
require.Truef(t, trySSHReady(pubIP, 22, keyPath, 3*time.Minute),
|
||||
"tcp/22 to %s in new subnet %s never became reachable — the implicit "+
|
||||
"main-RT IGW egress policy was not installed for a subnet created "+
|
||||
"after the route existed", pubIP, subnetID)
|
||||
|
||||
tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath}
|
||||
idOut := runSSH(t, tgt, "id")
|
||||
assert.Containsf(t, idOut, "ec2-user", "ssh id in new subnet\n%s", idOut)
|
||||
}
|
||||
|
||||
// runSameSGComms launches two instances in the default SG (plus a dedicated
|
||||
// runner-SSH SG so the runner can shell into one) and asserts that one can
|
||||
// ICMP-ping the other over the VPC. ICMP between them is permitted only by the
|
||||
// default SG's self-reference rule, so success proves that rule is enforced on
|
||||
// the OVN datapath — the east-west counterpart to the external-reach baseline.
|
||||
// No default resource is mutated (only the dedicated runner SG is authorized).
|
||||
func runSameSGComms(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Single — Baseline: same default-SG instances communicate")
|
||||
|
||||
vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
||||
instType, _ := needInstanceTypeArch(t, fix)
|
||||
keyName, keyPath := needKeyPair(t, fix)
|
||||
ami := needAMI(t, fix)
|
||||
|
||||
// Dedicated SG opens tcp/22 only, so ICMP between guests still depends on
|
||||
// the default SG's self-ingress — keeps the signal unambiguous.
|
||||
runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg")
|
||||
harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG)
|
||||
harness.Detail(t, "vpc", vpcID, "default_sg", defSGID, "runner_sg", runnerSG)
|
||||
|
||||
srcID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG})
|
||||
dstID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG})
|
||||
|
||||
dstPriv := instancePrivateIP(t, fix, dstID)
|
||||
|
||||
srcInst := harness.WaitForInstanceState(t, fix.AWS, srcID, "running")
|
||||
host, port := harness.InstancePublicSSHHost(t, srcInst)
|
||||
harness.Detail(t, "src", srcID, "dst", dstID, "dst_private_ip", dstPriv, "ssh_host", host)
|
||||
waitForSSHHandshake(t, host, port, keyPath)
|
||||
|
||||
tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath}
|
||||
harness.Step(t, "ping %s (%s) from %s via default-SG self-ingress", dstID, dstPriv, srcID)
|
||||
out, err := sshCapture(tgt, fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv))
|
||||
require.NoErrorf(t, err,
|
||||
"intra-default-SG ping %s -> %s failed; self-ingress not enforced on the datapath\n%s",
|
||||
srcID, dstID, out)
|
||||
assert.Containsf(t, out, "0% packet loss",
|
||||
"intra-default-SG ping had loss; self-ingress datapath degraded\n%s", out)
|
||||
}
|
||||
@@ -94,6 +94,22 @@ func TestClusterStatsCLI(t *testing.T) {
|
||||
runClusterStatsCLI(t, requireSingleNodeFixture(t))
|
||||
}
|
||||
|
||||
// Fresh-install reachability baselines run right after the 0-VM cluster-stats
|
||||
// gate and before the singleton launch. They own all mutable resources
|
||||
// (dedicated SG / subnet / self-cleaning instance) and never mutate the
|
||||
// default SG, subnet, or route table.
|
||||
func TestDefaultSGReachabilityBaseline(t *testing.T) {
|
||||
runDefaultSGReachabilityBaseline(t, requireSingleNodeFixture(t))
|
||||
}
|
||||
|
||||
func TestNewSubnetPublicEgressBaseline(t *testing.T) {
|
||||
runNewSubnetPublicEgressBaseline(t, requireSingleNodeFixture(t))
|
||||
}
|
||||
|
||||
func TestSameSGComms(t *testing.T) {
|
||||
runSameSGComms(t, requireSingleNodeFixture(t))
|
||||
}
|
||||
|
||||
func TestInstanceLaunch(t *testing.T) {
|
||||
runInstanceLaunch(t, requireSingleNodeFixture(t))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user