Files
Josh Siviour d076a434d4 Feat/e2e targeted suites (#352)
* test(e2e): add targeted suite selector + shadow-mode CI step

select-suites maps a diff to the e2e suites it must run: changed files →
services (manifest path/additional_paths) → consumer closure over
depends_on vs publishes/subscribes → suites whose covers intersect.
Special cases force-all (infra/harness/workflow, ≥3 services, submodule
bumps) or a lone suite dir; empty diffs fall back to default suites. An
AST scan of each suite's harness.Ensure* call graph derives per-test
service coverage and emits an optional -test.run subset (RUN_PATTERN).

Wired into e2e.yml as a non-gating 'Select Suites (shadow)' job that
logs the computed selection to the run summary so predicted vs actual
can be compared before the selector drives SUITES.

* ci(e2e): add dispatch-only per-suite job blocks grouped by env

Restructure the looped e2e flow into env-grouped provision -> matrix
suites -> teardown blocks so each suite surfaces as its own node in the
Actions graph instead of a single green tick:

  single_provision -> single_suite [single,cert,lb,eks] -> single_teardown
  multi_provision  -> multi_suite  [multinode]          -> multi_teardown

Provision resolves the VM/cluster and ships test binaries, emitting WAN
IP + ssh key path as job outputs; each matrix leg runs one suite over
SSH and uploads its own junit/analysis artifacts; teardown (if:always)
reuses the persisted tofu workspace on the same runner to destroy.

workflow_dispatch only -- runs alongside the gating e2e.yml for
validation and posts no commit status.

* test(e2e): add manifest drift lint (fixture-create + NATS subject)

Bead 5 of the targeted-suite-selection plan. Two ratcheted drift guards in
tests/e2e/manifest-lint, wired into make preflight and the spinifex.yml PR job:

- Fixture lint: AST-flags direct resource-create AWS calls
  (c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...) in suite test files,
  steering them through harness.Ensure* fixtures. ~104 pre-existing call sites
  are captured in a checked-in baseline; the lint fails only on NEW creates.
  Intentional create-path tests opt out with an inline // e2e:allow-create.

- Subject lint: extracts NATS subject literals from the natsSub{} registry and
  literal Subscribe/Publish/Request args, cross-referenced wildcard-aware
  against service-interfaces.yaml. Code subjects missing from the manifest fail
  (ratcheted); manifest subjects with no static reference warn only (many are
  produced dynamically). Fixed real drift surfaced on first run: declared the
  five spinifex.cluster.shutdown.* phase subjects under spinifex_daemon.

manifest-check is now CI-enforced alongside manifest-lint (previously
preflight/local-only).

* feat: run-scoped e2e teardown sweep

Tag every Ensure* fixture resource with e2e:run=<run-id> when
SPINIFEX_E2E_RUN_ID is set, and add a teardown.test suite that sweeps
everything carrying that tag at end of run — reclaiming resources leaked
by a crashed suite, an interrupted run, or a persistent cluster.

- harness/sweep.go: tagRunResources (EC2 CreateTags) + tagRunELB (ELBv2
  AddTags) + SweepRunResources in dependency order (LB, instances, NAT,
  snapshots, volumes, images, SGs, subnets, key pairs). Blank run id
  no-ops both tagging and sweeping, so the sweep can never touch an
  untagged or foreign resource.
- harness/fixtures.go: stamp the run tag on each created resource.
- tests/e2e/teardown: thin suite that runs the sweep; skips without a
  cluster or run id, logs the report, never fails the run.
- e2e-suites.yml: build + ship teardown.test, pass the run id into suite
  legs, sweep on the VM in both teardown jobs before tofu destroy.
2026-06-11 21:09:50 +10:00

354 lines
12 KiB
Go

//go:build e2e
package harness
import (
"sort"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
)
// sweepFakeEC2 returns a seeded set of resources from each Describe* and
// records every delete/terminate call. The embedded ec2iface.EC2API panics on
// any un-mocked method so the sweep can't silently grow a new dependency.
type sweepFakeEC2 struct {
ec2iface.EC2API
instances []*ec2.Instance
natGateways []*ec2.NatGateway
snapshots []*ec2.Snapshot
volumes []*ec2.Volume
images []*ec2.Image
securityGroups []*ec2.SecurityGroup
subnets []*ec2.Subnet
keyPairs []*ec2.KeyPairInfo
deleted map[string][]string
tagged map[string][]string // runID → resource IDs passed to CreateTags
describeN map[string]int
}
func newSweepFakeEC2() *sweepFakeEC2 {
return &sweepFakeEC2{deleted: map[string][]string{}, tagged: map[string][]string{}, describeN: map[string]int{}}
}
func (f *sweepFakeEC2) rec(kind string, ids ...string) {
f.deleted[kind] = append(f.deleted[kind], ids...)
}
func (f *sweepFakeEC2) CreateTags(in *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
var runID string
for _, t := range in.Tags {
if aws.StringValue(t.Key) == runTagKey {
runID = aws.StringValue(t.Value)
}
}
f.tagged[runID] = append(f.tagged[runID], aws.StringValueSlice(in.Resources)...)
return &ec2.CreateTagsOutput{}, nil
}
func (f *sweepFakeEC2) DescribeInstances(in *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
// Wait-for-terminated path describes by explicit IDs — report them gone so
// waitInstancesTerminated returns on the first poll.
if len(in.InstanceIds) > 0 {
var insts []*ec2.Instance
for _, id := range in.InstanceIds {
insts = append(insts, &ec2.Instance{
InstanceId: id,
State: &ec2.InstanceState{Name: aws.String("terminated")},
})
}
return &ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{{Instances: insts}},
}, nil
}
f.describeN["instances"]++
return &ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{{Instances: f.instances}},
}, nil
}
func (f *sweepFakeEC2) TerminateInstances(in *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) {
f.rec("instance", aws.StringValueSlice(in.InstanceIds)...)
return &ec2.TerminateInstancesOutput{}, nil
}
func (f *sweepFakeEC2) DescribeNatGateways(*ec2.DescribeNatGatewaysInput) (*ec2.DescribeNatGatewaysOutput, error) {
f.describeN["natgateways"]++
return &ec2.DescribeNatGatewaysOutput{NatGateways: f.natGateways}, nil
}
func (f *sweepFakeEC2) DeleteNatGateway(in *ec2.DeleteNatGatewayInput) (*ec2.DeleteNatGatewayOutput, error) {
f.rec("natgateway", aws.StringValue(in.NatGatewayId))
return &ec2.DeleteNatGatewayOutput{}, nil
}
func (f *sweepFakeEC2) DescribeSnapshots(*ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error) {
f.describeN["snapshots"]++
return &ec2.DescribeSnapshotsOutput{Snapshots: f.snapshots}, nil
}
func (f *sweepFakeEC2) DeleteSnapshot(in *ec2.DeleteSnapshotInput) (*ec2.DeleteSnapshotOutput, error) {
f.rec("snapshot", aws.StringValue(in.SnapshotId))
return &ec2.DeleteSnapshotOutput{}, nil
}
func (f *sweepFakeEC2) DescribeVolumes(*ec2.DescribeVolumesInput) (*ec2.DescribeVolumesOutput, error) {
f.describeN["volumes"]++
return &ec2.DescribeVolumesOutput{Volumes: f.volumes}, nil
}
func (f *sweepFakeEC2) DeleteVolume(in *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) {
f.rec("volume", aws.StringValue(in.VolumeId))
return &ec2.DeleteVolumeOutput{}, nil
}
func (f *sweepFakeEC2) DescribeImages(*ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) {
f.describeN["images"]++
return &ec2.DescribeImagesOutput{Images: f.images}, nil
}
func (f *sweepFakeEC2) DeregisterImage(in *ec2.DeregisterImageInput) (*ec2.DeregisterImageOutput, error) {
f.rec("image", aws.StringValue(in.ImageId))
return &ec2.DeregisterImageOutput{}, nil
}
func (f *sweepFakeEC2) DescribeSecurityGroups(*ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) {
f.describeN["securitygroups"]++
return &ec2.DescribeSecurityGroupsOutput{SecurityGroups: f.securityGroups}, nil
}
func (f *sweepFakeEC2) DeleteSecurityGroup(in *ec2.DeleteSecurityGroupInput) (*ec2.DeleteSecurityGroupOutput, error) {
f.rec("securitygroup", aws.StringValue(in.GroupId))
return &ec2.DeleteSecurityGroupOutput{}, nil
}
func (f *sweepFakeEC2) DescribeSubnets(*ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) {
f.describeN["subnets"]++
return &ec2.DescribeSubnetsOutput{Subnets: f.subnets}, nil
}
func (f *sweepFakeEC2) DeleteSubnet(in *ec2.DeleteSubnetInput) (*ec2.DeleteSubnetOutput, error) {
f.rec("subnet", aws.StringValue(in.SubnetId))
return &ec2.DeleteSubnetOutput{}, nil
}
func (f *sweepFakeEC2) DescribeKeyPairs(*ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
f.describeN["keypairs"]++
return &ec2.DescribeKeyPairsOutput{KeyPairs: f.keyPairs}, nil
}
func (f *sweepFakeEC2) DeleteKeyPair(in *ec2.DeleteKeyPairInput) (*ec2.DeleteKeyPairOutput, error) {
f.rec("keypair", aws.StringValue(in.KeyName))
return &ec2.DeleteKeyPairOutput{}, nil
}
// sweepFakeELB seeds DescribeLoadBalancers + per-ARN DescribeTags and records
// deletes / AddTags.
type sweepFakeELB struct {
elbv2iface.ELBV2API
lbs []*elbv2.LoadBalancer
tags map[string][]*elbv2.Tag // ARN → tags
deleted []string
added map[string][]string // runID → ARNs passed to AddTags
}
func newSweepFakeELB() *sweepFakeELB {
return &sweepFakeELB{tags: map[string][]*elbv2.Tag{}, added: map[string][]string{}}
}
func (f *sweepFakeELB) DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) {
return &elbv2.DescribeLoadBalancersOutput{LoadBalancers: f.lbs}, nil
}
func (f *sweepFakeELB) DescribeTags(in *elbv2.DescribeTagsInput) (*elbv2.DescribeTagsOutput, error) {
var descs []*elbv2.TagDescription
for _, arn := range in.ResourceArns {
descs = append(descs, &elbv2.TagDescription{
ResourceArn: arn,
Tags: f.tags[aws.StringValue(arn)],
})
}
return &elbv2.DescribeTagsOutput{TagDescriptions: descs}, nil
}
func (f *sweepFakeELB) DeleteLoadBalancer(in *elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) {
f.deleted = append(f.deleted, aws.StringValue(in.LoadBalancerArn))
return &elbv2.DeleteLoadBalancerOutput{}, nil
}
func (f *sweepFakeELB) AddTags(in *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) {
var runID string
for _, t := range in.Tags {
if aws.StringValue(t.Key) == runTagKey {
runID = aws.StringValue(t.Value)
}
}
f.added[runID] = append(f.added[runID], aws.StringValueSlice(in.ResourceArns)...)
return &elbv2.AddTagsOutput{}, nil
}
var (
_ ec2iface.EC2API = (*sweepFakeEC2)(nil)
_ elbv2iface.ELBV2API = (*sweepFakeELB)(nil)
)
// TestSweepRunResources_BlankRunIDNoop verifies a blank run id deletes nothing
// and never dials AWS.
func TestSweepRunResources_BlankRunIDNoop(t *testing.T) {
ec2c := newSweepFakeEC2()
elbc := newSweepFakeELB()
rep := SweepRunResources(ec2c, elbc, "")
if len(rep.Deleted) != 0 || len(rep.Errors) != 0 {
t.Fatalf("blank run id: report = %+v, want empty", rep)
}
if len(ec2c.describeN) != 0 {
t.Fatalf("blank run id issued Describe calls: %v", ec2c.describeN)
}
}
// TestSweepRunResources_DeletesEachKind seeds one live resource of every kind
// and verifies the sweep deletes them all and records them in the report.
func TestSweepRunResources_DeletesEachKind(t *testing.T) {
ec2c := newSweepFakeEC2()
ec2c.instances = []*ec2.Instance{{
InstanceId: aws.String("i-1"),
State: &ec2.InstanceState{Name: aws.String("running")},
}}
ec2c.natGateways = []*ec2.NatGateway{{
NatGatewayId: aws.String("nat-1"), State: aws.String("available"),
}}
ec2c.snapshots = []*ec2.Snapshot{{SnapshotId: aws.String("snap-1")}}
ec2c.volumes = []*ec2.Volume{{VolumeId: aws.String("vol-1"), State: aws.String("available")}}
ec2c.images = []*ec2.Image{{ImageId: aws.String("ami-1")}}
ec2c.securityGroups = []*ec2.SecurityGroup{{GroupId: aws.String("sg-1"), GroupName: aws.String("e2e-sg")}}
ec2c.subnets = []*ec2.Subnet{{SubnetId: aws.String("subnet-1")}}
ec2c.keyPairs = []*ec2.KeyPairInfo{{KeyName: aws.String("e2e-key")}}
arn := "arn:lb/1"
elbc := newSweepFakeELB()
elbc.lbs = []*elbv2.LoadBalancer{{LoadBalancerArn: aws.String(arn)}}
elbc.tags[arn] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-1")}}
rep := SweepRunResources(ec2c, elbc, "run-1")
want := map[string]string{
"instance": "i-1", "natgateway": "nat-1", "snapshot": "snap-1",
"volume": "vol-1", "image": "ami-1", "securitygroup": "sg-1",
"subnet": "subnet-1", "keypair": "e2e-key", "loadbalancer": arn,
}
for kind, id := range want {
got := rep.Deleted[kind]
if len(got) != 1 || got[0] != id {
t.Errorf("Deleted[%s] = %v, want [%s]", kind, got, id)
}
}
if len(rep.Errors) != 0 {
t.Fatalf("unexpected errors: %v", rep.Errors)
}
}
// TestSweepInstances_SkipsTerminated verifies already-terminated instances are
// not re-terminated.
func TestSweepInstances_SkipsTerminated(t *testing.T) {
ec2c := newSweepFakeEC2()
ec2c.instances = []*ec2.Instance{{
InstanceId: aws.String("i-dead"),
State: &ec2.InstanceState{Name: aws.String("terminated")},
}}
rep := SweepRunResources(ec2c, newSweepFakeELB(), "run-1")
if len(rep.Deleted["instance"]) != 0 {
t.Fatalf("terminated instance swept: %v", rep.Deleted["instance"])
}
}
// TestSweepSecurityGroups_SkipsDefault verifies the default SG is never
// deleted even if it carries the run tag.
func TestSweepSecurityGroups_SkipsDefault(t *testing.T) {
ec2c := newSweepFakeEC2()
ec2c.securityGroups = []*ec2.SecurityGroup{
{GroupId: aws.String("sg-def"), GroupName: aws.String("default")},
{GroupId: aws.String("sg-e2e"), GroupName: aws.String("e2e")},
}
rep := SweepRunResources(ec2c, newSweepFakeELB(), "run-1")
got := rep.Deleted["securitygroup"]
if len(got) != 1 || got[0] != "sg-e2e" {
t.Fatalf("securitygroup deleted = %v, want [sg-e2e]", got)
}
}
// TestSweepLoadBalancers_MatchesByTag verifies only LBs carrying the matching
// run tag are deleted (ELBv2 Describe has no tag filter).
func TestSweepLoadBalancers_MatchesByTag(t *testing.T) {
elbc := newSweepFakeELB()
elbc.lbs = []*elbv2.LoadBalancer{
{LoadBalancerArn: aws.String("arn:match")},
{LoadBalancerArn: aws.String("arn:other")},
{LoadBalancerArn: aws.String("arn:untagged")},
}
elbc.tags["arn:match"] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-1")}}
elbc.tags["arn:other"] = []*elbv2.Tag{{Key: aws.String(runTagKey), Value: aws.String("run-2")}}
SweepRunResources(newSweepFakeEC2(), elbc, "run-1")
sort.Strings(elbc.deleted)
if len(elbc.deleted) != 1 || elbc.deleted[0] != "arn:match" {
t.Fatalf("deleted LBs = %v, want [arn:match]", elbc.deleted)
}
}
// TestSweepLoadBalancers_NilClientNoop verifies a nil ELBv2 client is tolerated
// (suites with no LB surface).
func TestSweepLoadBalancers_NilClientNoop(t *testing.T) {
rep := SweepRunResources(newSweepFakeEC2(), nil, "run-1")
if len(rep.Deleted["loadbalancer"]) != 0 {
t.Fatalf("nil elbc swept LBs: %v", rep.Deleted["loadbalancer"])
}
}
// TestTagRunResources_NoRunIDSkips verifies tagging no-ops without a run id.
func TestTagRunResources_NoRunIDSkips(t *testing.T) {
ec2c := newSweepFakeEC2()
fx := newFixture(t, ec2c, newSweepFakeELB())
fx.runID = ""
fx.tagRunResources("i-1")
if len(ec2c.tagged) != 0 {
t.Fatalf("tagRunResources tagged with empty run id: %v", ec2c.tagged)
}
}
// TestTagRunResources_StampsRunTag verifies the configured run id is stamped.
func TestTagRunResources_StampsRunTag(t *testing.T) {
ec2c := newSweepFakeEC2()
fx := newFixture(t, ec2c, newSweepFakeELB())
fx.runID = "run-9"
fx.tagRunResources("vol-1", "vol-2")
got := ec2c.tagged["run-9"]
if len(got) != 2 || got[0] != "vol-1" || got[1] != "vol-2" {
t.Fatalf("tagged[run-9] = %v, want [vol-1 vol-2]", got)
}
}
// TestTagRunELB_StampsRunTag verifies the ELBv2 AddTags path stamps the run id.
func TestTagRunELB_StampsRunTag(t *testing.T) {
elbc := newSweepFakeELB()
fx := newFixture(t, newSweepFakeEC2(), elbc)
fx.runID = "run-9"
fx.tagRunELB("arn:lb/1")
got := elbc.added["run-9"]
if len(got) != 1 || got[0] != "arn:lb/1" {
t.Fatalf("added[run-9] = %v, want [arn:lb/1]", got)
}
}