mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
Julian Sommer
212 lines
8.1 KiB
Go
212 lines
8.1 KiB
Go
//go:build e2e
|
|
|
|
package multinode
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Fresh-install reachability baselines. Run before needInstanceTrio so the
|
|
// default SG/subnet/route-table are in their pristine state.
|
|
//
|
|
// - runMultinodeDefaultSGReachabilityBaseline: proves the SG (not routing)
|
|
// gates external tcp/22 — blocked before authorize, reachable after.
|
|
// - runMultinodeSameSGCrossHostComms: proves the default SG self-reference
|
|
// rule spans chassis — ICMP between two instances on different nodes
|
|
// succeeds with no added ingress rule.
|
|
|
|
// baselineLaunch launches one instance with the given SGs into subnetID,
|
|
// registers a terminate-and-wait cleanup, and returns the instance ID once "running".
|
|
func baselineLaunch(t *testing.T, fix *Fixture, amiID, instType, keyName, subnetID string, sgIDs []string) string {
|
|
t.Helper()
|
|
sgs := make([]*string, 0, len(sgIDs))
|
|
for _, id := range sgIDs {
|
|
sgs = append(sgs, aws.String(id))
|
|
}
|
|
input := &ec2.RunInstancesInput{
|
|
ImageId: aws.String(amiID),
|
|
InstanceType: aws.String(instType),
|
|
KeyName: aws.String(keyName),
|
|
SubnetId: aws.String(subnetID),
|
|
SecurityGroupIds: sgs,
|
|
MinCount: aws.Int64(1),
|
|
MaxCount: aws.Int64(1),
|
|
}
|
|
var id string
|
|
for attempt := 1; attempt <= 6; attempt++ {
|
|
out, err := fix.AWS.EC2.RunInstances(input)
|
|
if err == nil {
|
|
require.NotEmpty(t, out.Instances, "RunInstances returned no instances")
|
|
id = aws.StringValue(out.Instances[0].InstanceId)
|
|
break
|
|
}
|
|
if !strings.Contains(err.Error(), "InsufficientInstanceCapacity") {
|
|
t.Fatalf("RunInstances: %v", err)
|
|
}
|
|
t.Logf("baselineLaunch attempt %d: InsufficientInstanceCapacity, retrying", attempt)
|
|
time.Sleep(10 * time.Second)
|
|
}
|
|
require.NotEmpty(t, id, "RunInstances never succeeded")
|
|
t.Cleanup(func() {
|
|
_, _ = fix.AWS.EC2.TerminateInstances(&ec2.TerminateInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
harness.WaitForInstanceState(t, fix.AWS, id, "terminated")
|
|
})
|
|
harness.WaitForInstanceState(t, fix.AWS, id, "running")
|
|
return id
|
|
}
|
|
|
|
// instancePrivateIP returns the instance's primary VPC private IP.
|
|
func instancePrivateIP(t *testing.T, fix *Fixture, id string) string {
|
|
t.Helper()
|
|
out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
require.NoError(t, err, "describe-instances %s", id)
|
|
require.NotEmpty(t, out.Reservations, "no reservations for %s", id)
|
|
require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", id)
|
|
ip := aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress)
|
|
require.NotEmptyf(t, ip, "instance %s has no private IP", id)
|
|
return ip
|
|
}
|
|
|
|
// sshCapture runs cmd over SSH and returns combined output + error without fataling.
|
|
func sshCapture(pem, user, host string, port int, cmd string) (string, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
args := []string{
|
|
"-i", pem,
|
|
"-p", strconv.Itoa(port),
|
|
"-o", "StrictHostKeyChecking=no",
|
|
"-o", "UserKnownHostsFile=/dev/null",
|
|
"-o", "ConnectTimeout=5",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "LogLevel=ERROR",
|
|
fmt.Sprintf("%s@%s", user, host),
|
|
cmd,
|
|
}
|
|
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
|
|
return string(out), err
|
|
}
|
|
|
|
// runMultinodeDefaultSGReachabilityBaseline asserts the default-deny SG gate
|
|
// on the public-IP datapath: blocked before authorize, reachable after.
|
|
func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) {
|
|
harness.Phase(t, "Multinode — Baseline: default-deny SG blocks external reach until authorized")
|
|
|
|
vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
|
instType, arch := needInstanceTypeArch(t, fix)
|
|
amiID := needAMI(t, fix, arch)
|
|
keyName, keyPath := needKeyPair(t, fix)
|
|
|
|
sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg")
|
|
id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{sgID})
|
|
|
|
inst := harness.WaitForInstanceState(t, fix.AWS, id, "running")
|
|
pubIP := aws.StringValue(inst.PublicIpAddress)
|
|
if pubIP == "" || pubIP == "None" {
|
|
t.Fatalf("instance %s has no public IP; the datapath it depends on is "+
|
|
"broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", id)
|
|
}
|
|
harness.Detail(t, "instance", id, "public_ip", pubIP, "sg", sgID)
|
|
|
|
harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
|
|
deadline := time.Now().Add(30 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
conn, derr := net.DialTimeout("tcp", net.JoinHostPort(pubIP, "22"), 3*time.Second)
|
|
if derr == nil {
|
|
_ = conn.Close()
|
|
t.Fatalf("tcp/22 to %s connected with NO ingress rule — default-deny SG must block external traffic", pubIP)
|
|
}
|
|
time.Sleep(3 * time.Second)
|
|
}
|
|
|
|
harness.Step(t, "authorizing tcp/22, expecting reachability")
|
|
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
|
|
harness.GuestSSHReady(t, pubIP, 22, "ec2-user", keyPath,
|
|
harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second))
|
|
|
|
out, err := sshCapture(keyPath, "ec2-user", pubIP, 22, "id")
|
|
require.NoErrorf(t, err, "ssh id after authorize: %s", out)
|
|
assert.Containsf(t, out, "ec2-user", "ssh id after authorize\n%s", out)
|
|
}
|
|
|
|
// runMultinodeSameSGCrossHostComms launches two instances on different nodes sharing
|
|
// the default SG and asserts ICMP-ping succeeds across hosts. ICMP is permitted only
|
|
// by the default SG self-reference rule, proving it is enforced across chassis.
|
|
func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) {
|
|
harness.Phase(t, "Multinode — Baseline: same default-SG instances communicate across hosts")
|
|
|
|
vpcID, defSGID, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
|
|
instType, arch := needInstanceTypeArch(t, fix)
|
|
amiID := needAMI(t, fix, arch)
|
|
keyName, keyPath := needKeyPair(t, fix)
|
|
|
|
// Dedicated SG for runner SSH — opens tcp/22 only; ICMP between guests
|
|
// still depends on the default SG self-ingress, keeping the signal clean.
|
|
runnerSG := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-runnersg")
|
|
harness.AuthorizeSSHIngress(t, fix.AWS, runnerSG)
|
|
|
|
// Launch instances until two land on distinct nodes. Bounded so a degenerate
|
|
// single-node placement can't loop forever.
|
|
type placed struct {
|
|
id string
|
|
node string
|
|
}
|
|
var instances []placed
|
|
var srcIdx, dstIdx = -1, -1
|
|
for attempt := 0; attempt < 4 && (srcIdx < 0 || dstIdx < 0); attempt++ {
|
|
id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{defSGID, runnerSG})
|
|
node := harness.InstanceHostingNode(t, fix.Cluster, id)
|
|
nodeName := ""
|
|
if node != nil {
|
|
nodeName = node.Name
|
|
}
|
|
instances = append(instances, placed{id: id, node: nodeName})
|
|
// Re-scan for a distinct-node pair.
|
|
srcIdx, dstIdx = -1, -1
|
|
for i := range instances {
|
|
for j := range instances {
|
|
if i != j && instances[i].node != "" && instances[i].node != instances[j].node {
|
|
srcIdx, dstIdx = i, j
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if srcIdx < 0 || dstIdx < 0 {
|
|
t.Skipf("could not place two instances on distinct nodes (got %v); scheduler colocated", instances)
|
|
}
|
|
src, dst := instances[srcIdx], instances[dstIdx]
|
|
harness.Detail(t, "src", src.id, "src_node", src.node, "dst", dst.id, "dst_node", dst.node)
|
|
|
|
dstPriv := instancePrivateIP(t, fix, dst.id)
|
|
|
|
// Shell into the source instance via its dedicated runner-SSH SG.
|
|
host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, src.id)
|
|
harness.GuestSSHReady(t, host, port, "ec2-user", keyPath,
|
|
harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second))
|
|
|
|
harness.Step(t, "ping %s (%s) from %s across hosts via default-SG self-ingress", dst.id, dstPriv, src.id)
|
|
out, err := sshCapture(keyPath, "ec2-user", host, port,
|
|
fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv))
|
|
require.NoErrorf(t, err,
|
|
"cross-host ping %s -> %s failed; default SG self-ingress not enforced across chassis\n%s",
|
|
src.id, dst.id, out)
|
|
assert.Containsf(t, out, "0% packet loss",
|
|
"cross-host ping had loss; default SG self-ingress datapath degraded\n%s", out)
|
|
}
|