spinifex/tests/e2e/single/baseline_test.go

//go:build e2e

package single

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/mulgadc/spinifex/tests/e2e/harness"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Fresh-install reachability baselines. Tests #1 and #2 pin two independent
// barriers a user hits on a fresh cluster:
//
//  1. SG barrier: default-SG only admits same-SG members; routes are fine.
//  2. Route/egress gap: 0.0.0.0/0 added to the main RT before the subnet
//     exists; CreateSubnet doesn't recompute the per-subnet OVN egress LRP.

// tcpReachable reports whether a TCP connect to host:port succeeds within
// timeout. An OVN ACL drop yields a dial timeout (no RST); a reject yields
// connection-refused — either way the connect fails and we return false.
func tcpReachable(host string, port int, timeout time.Duration) bool {
	conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), timeout)
	if err != nil {
		return false
	}
	_ = conn.Close()
	return true
}

// launchBaselineInstance launches one instance and registers a terminate-and-wait
// cleanup so the VM is gone before the next test. Not memoized — each baseline
// owns its VM for its own duration only.
func launchBaselineInstance(t *testing.T, fix *Fixture, ami, instType, keyName, subnetID string, sgIDs []string) string {
	t.Helper()
	sgs := make([]*string, 0, len(sgIDs))
	for _, id := range sgIDs {
		sgs = append(sgs, aws.String(id))
	}
	out, err := fix.AWS.EC2.RunInstances(&ec2.RunInstancesInput{
		ImageId:          aws.String(ami),
		InstanceType:     aws.String(instType),
		KeyName:          aws.String(keyName),
		SubnetId:         aws.String(subnetID),
		SecurityGroupIds: sgs,
		MinCount:         aws.Int64(1),
		MaxCount:         aws.Int64(1),
	})
	require.NoError(t, err, "RunInstances")
	require.NotEmpty(t, out.Instances, "RunInstances returned no instances")
	id := aws.StringValue(out.Instances[0].InstanceId)
	t.Cleanup(func() {
		_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
			InstanceIds: []*string{aws.String(id)},
		})
		harness.WaitForInstanceState(t, fix.AWS, id, "terminated")
	})
	harness.WaitForInstanceState(t, fix.AWS, id, "running")
	return id
}

// instancePrivateIP returns the instance's primary VPC private IP.
func instancePrivateIP(t *testing.T, fix *Fixture, id string) string {
	t.Helper()
	out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(id)},
	})
	require.NoError(t, err, "describe-instances %s", id)
	require.NotEmpty(t, out.Reservations, "no reservations for %s", id)
	require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id)
	ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress)
	require.NotEmptyf(t, ip, "instance %s has no private IP", id)
	return ip
}

// pingConverged probes dst over ICMP from tgt, retrying until the east-west
// datapath converges (0% loss) or timeout elapses. L2 between two freshly
// launched instances has a warm-up: ARP cannot resolve until the destination
// guest NIC is up and OVN has programmed its port flows, which lags the EC2
// "running" transition the launch waits on. A single un-retried ping races that
// window and reports a false "Destination Host Unreachable". Returns the last
// ping output and whether it ever converged.
func pingConverged(tgt harness.SSHTarget, dst string, timeout time.Duration) (string, bool) {
	deadline := time.Now().Add(timeout)
	var out string
	for {
		var err error
		out, err = sshCapture(tgt, fmt.Sprintf("ping -c 3 -W 2 %s", dst))
		if err == nil && strings.Contains(out, "0% packet loss") {
			return out, true
		}
		if time.Now().After(deadline) {
			return out, false
		}
		time.Sleep(2 * time.Second)
	}
}

// sshCapture runs cmd over SSH and returns combined output + error without
// fataling, so callers can assert on the exit status (e.g. ping result).
func sshCapture(tgt harness.SSHTarget, cmd string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	out, err := harness.RunGuestSSH(ctx, tgt, cmd)
	return string(out), err
}

// instancePublicIP returns the instance's routable public IP. Missing is fatal:
// baselines must exercise the real OVN datapath (hostfwd is disabled suite-wide).
func instancePublicIP(t *testing.T, fix *Fixture, instanceID string) string {
	t.Helper()
	out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(instanceID)},
	})
	require.NoError(t, err, "describe-instances %s", instanceID)
	require.NotEmpty(t, out.Reservations, "no reservations for %s", instanceID)
	require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", instanceID)
	ip := aws.StringValue(out.Reservations[0].Instances[0].PublicIpAddress)
	if ip == "" || ip == "None" {
		t.Fatalf("instance %s has no public IP; the datapath it depends on is "+
			"broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", instanceID)
	}
	return ip
}

// runDefaultSGReachabilityBaseline launches an instance into the default
// subnet behind a dedicated default-deny SG, asserts it is unreachable from
// the runner, then opens tcp/22 and asserts it becomes reachable. Confirms
// the SG — not routing — gates a fresh default-config instance.
func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) {
	harness.Phase(t, "Single — Baseline: default-SG blocks external reach until authorized")

	vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
	instType, _ := needInstanceTypeArch(t, fix)
	keyName, keyPath := needKeyPair(t, fix)
	ami := needAMI(t, fix)

	// No ingress rules; egress is allow-all by default so only inbound is gated.
	sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg")
	harness.Detail(t, "vpc", vpcID, "subnet", subnetID, "sg", sgID)

	instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID})

	pubIP := instancePublicIP(t, fix, instanceID)
	harness.Detail(t, "instance", instanceID, "public_ip", pubIP)

	// Probe a short window to confirm the default-deny ACL is applied and stable.
	// This overlaps guest boot, and the positive phase below still pays the full
	// boot wait via trySSHReady, so a longer window buys little extra coverage.
	harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
	deadline := time.Now().Add(10 * time.Second)
	for time.Now().Before(deadline) {
		require.Falsef(t, tcpReachable(pubIP, 22, 3*time.Second),
			"tcp/22 to %s connected with NO ingress rule — default SG must deny external traffic", pubIP)
		time.Sleep(3 * time.Second)
	}

	harness.Step(t, "authorizing tcp/22 ingress, expecting reachability")
	harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
	require.Truef(t, trySSHReady(pubIP, 22, keyPath, sshReadyBudget),
		"tcp/22 to %s never became reachable after authorizing ingress — "+
			"default subnet egress/IGW datapath is broken", pubIP)

	tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath}
	idOut := runSSH(t, tgt, "id")
	assert.Containsf(t, idOut, "ubuntu", "ssh id after authorize\n%s", idOut)
}

// mainRouteTableID returns the main (implicitly-associated) route table for
// vpcID — the one a subnet joins when it has no explicit RT association.
func mainRouteTableID(t *testing.T, c *harness.AWSClient, vpcID string) string {
	t.Helper()
	out, err := c.EC2.DescribeRouteTables(&ec2.DescribeRouteTablesInput{
		Filters: []*ec2.Filter{
			{Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}},
			{Name: aws.String("association.main"), Values: []*string{aws.String("true")}},
		},
	})
	require.NoError(t, err, "describe-route-tables main vpc=%s", vpcID)
	require.NotEmptyf(t, out.RouteTables, "no main route table for vpc %s", vpcID)
	id := aws.StringValue(out.RouteTables[0].RouteTableId)
	require.NotEmpty(t, id, "main RouteTableId empty")
	return id
}

// runNewVPCEgressBaseline reproduces the IGW egress gap when a 0.0.0.0/0 route
// is added to the main RT before the subnet is created. The subnet implicitly
// joins the RT but CreateSubnet doesn't install the per-subnet OVN egress LRP,
// leaving the instance with no datapath to the IGW. Pool-mode gated.
func runNewVPCEgressBaseline(t *testing.T, fix *Fixture) {
	if !fix.PoolMode {
		t.Skip("fresh-VPC egress baseline requires pool-mode networking")
	}
	harness.Phase(t, "Single — Baseline: fresh VPC main-RT IGW egress (route before subnet)")

	c := fix.AWS
	az := needAZ(t, fix)
	instType, _ := needInstanceTypeArch(t, fix)
	keyName, keyPath := needKeyPair(t, fix)
	ami := needAMI(t, fix)

	const vpcCIDR = "10.231.0.0/16"
	const subnetCIDR = "10.231.1.0/24"

	// --- VPC ---
	vpcOut, err := c.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(vpcCIDR)})
	require.NoError(t, err, "create-vpc")
	require.NotNil(t, vpcOut.Vpc, "create-vpc returned nil Vpc")
	vpcID := aws.StringValue(vpcOut.Vpc.VpcId)
	require.NotEmpty(t, vpcID, "VpcId empty")
	t.Cleanup(func() { _, _ = c.EC2.DeleteVpc(&ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}) })

	// --- IGW + attach ---
	igwOut, err := c.EC2.CreateInternetGateway(&ec2.CreateInternetGatewayInput{})
	require.NoError(t, err, "create-internet-gateway")
	require.NotNil(t, igwOut.InternetGateway, "create-internet-gateway returned nil InternetGateway")
	igwID := aws.StringValue(igwOut.InternetGateway.InternetGatewayId)
	require.NotEmpty(t, igwID, "InternetGatewayId empty")
	t.Cleanup(func() {
		_, _ = c.EC2.DetachInternetGateway(&ec2.DetachInternetGatewayInput{
			InternetGatewayId: aws.String(igwID), VpcId: aws.String(vpcID),
		})
		_, _ = c.EC2.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{
			InternetGatewayId: aws.String(igwID),
		})
	})
	_, err = c.EC2.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
		InternetGatewayId: aws.String(igwID), VpcId: aws.String(vpcID),
	})
	require.NoError(t, err, "attach-internet-gateway")

	mainRT := mainRouteTableID(t, c, vpcID)
	harness.Detail(t, "vpc", vpcID, "igw", igwID, "main_rtb", mainRT)

	// --- Route FIRST, on the main RT, before the subnet exists ---
	harness.Step(t, "create-route 0.0.0.0/0 -> %s on main RT %s (before subnet)", igwID, mainRT)
	_, err = c.EC2.CreateRoute(&ec2.CreateRouteInput{
		RouteTableId:         aws.String(mainRT),
		DestinationCidrBlock: aws.String("0.0.0.0/0"),
		GatewayId:            aws.String(igwID),
	})
	require.NoError(t, err, "create-route 0.0.0.0/0 -> IGW on main RT")
	t.Cleanup(func() {
		_, _ = c.EC2.DeleteRoute(&ec2.DeleteRouteInput{
			RouteTableId:         aws.String(mainRT),
			DestinationCidrBlock: aws.String("0.0.0.0/0"),
		})
	})

	// --- Subnet AFTER the route (implicit main-RT membership) ---
	harness.Step(t, "create-subnet %s (implicit main RT, after IGW route)", subnetCIDR)
	subOut, err := c.EC2.CreateSubnet(&ec2.CreateSubnetInput{
		VpcId:            aws.String(vpcID),
		CidrBlock:        aws.String(subnetCIDR),
		AvailabilityZone: aws.String(az),
	})
	require.NoError(t, err, "create-subnet")
	require.NotNil(t, subOut.Subnet, "create-subnet returned nil Subnet")
	subnetID := aws.StringValue(subOut.Subnet.SubnetId)
	require.NotEmpty(t, subnetID, "SubnetId empty")
	t.Cleanup(func() { _, _ = c.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(subnetID)}) })

	_, err = c.EC2.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{
		SubnetId:            aws.String(subnetID),
		MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
	})
	require.NoError(t, err, "modify-subnet-attribute MapPublicIpOnLaunch on %s", subnetID)

	// --- Open SG so routing is provably the only barrier ---
	sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-newvpc-opensg")
	harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
	harness.Detail(t, "subnet", subnetID, "cidr", subnetCIDR, "sg", sgID)

	instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID})
	pubIP := instancePublicIP(t, fix, instanceID)
	harness.Detail(t, "instance", instanceID, "public_ip", pubIP)

	harness.Step(t, "expecting external SSH to instance in fresh-VPC public subnet")
	if !trySSHReady(pubIP, 22, keyPath, sshReadyBudget) {
		harness.DumpVPCFlowDiagnostics(t, c, instanceID,
			fmt.Sprintf("fresh-VPC egress baseline SSH timeout — vpc=%s igw=%s pub=%s", vpcID, igwID, pubIP),
			harness.VPCDiagnosticsOpts{
				ExternalIP:  pubIP,
				LogicalIP:   instancePrivateIP(t, fix, instanceID),
				ArtifactDir: fix.ArtifactDir(t),
			})
		t.Fatalf("tcp/22 to %s in fresh-VPC subnet %s never became reachable — the "+
			"per-subnet IGW egress policy was not installed for a subnet created "+
			"after the main-RT IGW route existed", pubIP, subnetID)
	}

	tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath}
	idOut := runSSH(t, tgt, "id")
	assert.Containsf(t, idOut, "ubuntu", "ssh id in fresh-VPC subnet\n%s", idOut)
}

// runSameSGComms verifies that two instances in the default SG can ICMP-ping
// each other (permitted by the default SG's self-reference rule) while a
// separate runner SG handles tcp/22 from outside. No default resource mutated.
func runSameSGComms(t *testing.T, fix *Fixture) {
	harness.Phase(t, "Single — Baseline: same default-SG instances communicate")

	vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
	instType, _ := needInstanceTypeArch(t, fix)
	keyName, keyPath := needKeyPair(t, fix)
	ami := needAMI(t, fix)

	runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg")
	harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG)
	harness.Detail(t, "vpc", vpcID, "default_sg", defSGID, "runner_sg", runnerSG)

	srcID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG})
	dstID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{defSGID, runnerSG})

	dstPriv := instancePrivateIP(t, fix, dstID)

	srcInst := harness.WaitForInstanceState(t, fix.AWS, srcID, "running")
	host, port := harness.InstancePublicSSHHost(t, srcInst)
	harness.Detail(t, "src", srcID, "dst", dstID, "dst_private_ip", dstPriv, "ssh_host", host)
	waitForSSHHandshake(t, host, port, keyPath)

	tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath}
	harness.Step(t, "ping %s (%s) from %s via default-SG self-ingress", dstID, dstPriv, srcID)
	out, converged := pingConverged(tgt, dstPriv, 45*time.Second)
	require.Truef(t, converged,
		"intra-default-SG east-west %s -> %s never reached 0%% loss within 45s; "+
			"ARP/L2 datapath unreachable across the default-SG self-ingress\n%s",
		srcID, dstID, out)
}