test(e2e-single): port run-e2e.sh phases 1-4 to Go

Stage B of mulga-siv-89. Lands the TestSingleNode orchestrator + Fixture
that threads state between sequential phase subtests, plus phases:

- Phase 1: KVM check, AWS gateway TLS reachability, daemon NATS
  readiness polling, region/AZ smoke.
- Phase 1b: spx get nodes / top nodes / get vms (single-node only).
- Phase 2: nano instance-type + arch discovery.
- Phase 2b: serial-console-access default/enable/disable round-trips.
- Phase 3: create-key-pair + import-key-pair lifecycle; test-key-1 PEM
  kept for Phase 5 instance launch (cleanup deferred to teardown phase
  in stage G).
- Phase 4: architecture-aware AMI discovery via name filter
  ami-ubuntu-24.04-{x86_64,arm64}, falls back from spx import output.

Phase ordering is contractual (Phase 5+ depend on AMI, key pair, AZ).
detectPoolMode reads [network] external_mode from spinifex.toml so
stages F/G can skip pool-only phases on dev_networking fixtures.

No production code changed. cert + lb scenarios continue to compile.
This commit is contained in:
joshsiv-mulga
2026-05-18 20:01:31 +10:00
parent 651377fe9c
commit 242d47074b
4 changed files with 523 additions and 0 deletions
+190
View File
@@ -0,0 +1,190 @@
//go:build e2e
package single
import (
"errors"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// phase1_Environment verifies KVM, AWS gateway TLS reachability, daemon NATS
// readiness, and the basic region/AZ discovery calls. Maps to run-e2e.sh
// lines ~51–93.
func phase1_Environment(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 1 — Environment")
// /dev/kvm — skip rather than fatal in local-dev to match the bash
// "exit 1" only when KVM is genuinely required (CI). Local devs without
// nested virt would otherwise be unable to even compile-run this suite.
harness.Step(t, "checking /dev/kvm")
st, err := os.Stat("/dev/kvm")
if err != nil {
t.Skipf("/dev/kvm not present (%v); single-node suite needs KVM", err)
}
if st.Mode()&0o200 == 0 {
// Bash treated non-writable as fatal — replicate.
t.Fatalf("/dev/kvm exists but is not writable (mode=%v)", st.Mode())
}
harness.Detail(t, "kvm", "writable")
// AWS gateway TLS handshake. Bash does `curl -sk` so it accepts any
// non-zero status as success; we use HTTPSGet with a nil pool so the
// system trust store is used (the cert suite already validates CA
// installation), and treat a non-zero HTTP status as a successful
// handshake.
harness.Step(t, "waiting for AWS gateway")
host := "127.0.0.1"
if len(fix.Env.ServiceIPs) > 0 {
host = fix.Env.ServiceIPs[0]
}
gwURL := fmt.Sprintf("https://%s:%d/", host, fix.Env.AWSGWPort)
harness.Eventually(t, func() bool {
code, _, err := harness.HTTPSGet(gwURL, nil, fix.Env.DefaultTimeout)
return err == nil && code != 0
}, 30*time.Second, 1*time.Second, "AWS gateway not reachable at "+gwURL)
harness.Detail(t, "gateway", gwURL)
// Daemon NATS readiness: bash polls describe-instance-types via the
// AWS CLI until it returns a non-empty list. Replicate via the SDK so
// we exercise the same NATS subscription path.
harness.Step(t, "waiting for daemon NATS subscriptions")
harness.EventuallyErr(t, func() error {
out, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
if err != nil {
return err
}
if len(out.InstanceTypes) == 0 {
return errors.New("describe-instance-types: empty result")
}
return nil
}, 30*time.Second, 1*time.Second)
// Region + AZ smoke calls.
harness.Step(t, "describe-regions / describe-availability-zones")
regions, err := fix.AWS.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
require.NoError(t, err, "describe-regions")
require.NotEmpty(t, regions.Regions, "no regions returned")
azOut, err := fix.AWS.EC2.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{})
require.NoError(t, err, "describe-availability-zones")
require.NotEmpty(t, azOut.AvailabilityZones, "no AZs returned")
fix.AZName = aws.StringValue(azOut.AvailabilityZones[0].ZoneName)
azState := aws.StringValue(azOut.AvailabilityZones[0].State)
require.Equalf(t, "available", azState, "AZ %s state %q (want available)", fix.AZName, azState)
harness.Detail(t, "az", fix.AZName, "region", aws.StringValue(azOut.AvailabilityZones[0].RegionName))
harness.OnFailure(t, func() {
harness.DumpCmd(t, fix.Artifacts, "phase1-az.txt",
"aws", "ec2", "describe-availability-zones")
})
}
// phase1b_ClusterStats exercises the spx CLI cluster surface (`get nodes`,
// `top nodes`, `get vms`). Single-node only — multinode mode is tested by a
// different scenario. Maps to run-e2e.sh ~95–127.
func phase1b_ClusterStats(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 1b — Cluster Stats CLI")
if fix.Env.Mode != harness.ModeSingle {
t.Skipf("Phase 1b is single-node only (mode=%s)", fix.Env.Mode)
}
harness.Step(t, "spx get nodes")
nodes := harness.SpxGetNodes(t)
assert.Contains(t, nodes, "Ready", "spx get nodes should report a Ready node\n%s", nodes)
harness.Step(t, "spx top nodes")
top := harness.SpxTopNodes(t)
// Bash matches `0/` in the resource stat column (e.g. "0/4 vCPU"). The
// presence of that fraction marker is what proves the stats column
// actually rendered.
assert.Contains(t, top, "0/", "spx top nodes should report resource stats\n%s", top)
harness.Step(t, "spx get vms (empty before any launches)")
vms := harness.SpxGetVMs(t)
assert.Contains(t, vms, "No VMs found",
"spx get vms should be empty before Phase 5\n%s", vms)
}
// phase2_Discovery re-asserts describe-regions / describe-availability-zones
// (cheap but it's where the bash records them as a phase boundary) and picks
// the nano instance type + architecture used throughout the rest of the run.
// Maps to run-e2e.sh ~128–164.
func phase2_Discovery(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 2 — Discovery & Metadata")
regions, err := fix.AWS.EC2.DescribeRegions(&ec2.DescribeRegionsInput{})
require.NoError(t, err)
require.NotEmpty(t, regions.Regions, "describe-regions empty")
azOut, err := fix.AWS.EC2.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{})
require.NoError(t, err)
require.NotEmpty(t, azOut.AvailabilityZones, "describe-availability-zones empty")
harness.Step(t, "discovering nano instance type")
out, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
require.NoError(t, err)
require.NotEmpty(t, out.InstanceTypes, "describe-instance-types empty")
// Bash picks the first match for `nano` and reads the architecture out
// of the same row. Replicate that ordering — daemons emit types in a
// stable order so picking the first nano keeps parity with the bash
// driver.
for _, it := range out.InstanceTypes {
name := aws.StringValue(it.InstanceType)
if !strings.Contains(name, "nano") {
continue
}
fix.InstanceType = name
if it.ProcessorInfo != nil && len(it.ProcessorInfo.SupportedArchitectures) > 0 {
fix.Arch = aws.StringValue(it.ProcessorInfo.SupportedArchitectures[0])
}
break
}
if fix.InstanceType == "" {
t.Fatalf("no nano instance type available; saw %d types", len(out.InstanceTypes))
}
if fix.Arch == "" {
t.Fatalf("instance type %s missing ProcessorInfo.SupportedArchitectures", fix.InstanceType)
}
harness.Detail(t, "instance_type", fix.InstanceType, "arch", fix.Arch, "az", fix.AZName)
}
// phase2b_SerialConsole flips serial-console-access on then off and verifies
// each transition with both the action-returned bool and a follow-up
// get-status round-trip. Maps to run-e2e.sh ~166–202.
func phase2b_SerialConsole(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 2b — Serial Console Access")
harness.Step(t, "default state should be disabled")
if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); got {
t.Fatalf("expected serial console default disabled, got enabled")
}
harness.Step(t, "enable")
if got := harness.EnableSerialConsoleAccess(t, fix.AWS); !got {
t.Fatalf("enable: action returned enabled=false")
}
if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); !got {
t.Fatalf("enable: subsequent get-status returned disabled")
}
harness.Detail(t, "state", "enabled")
harness.Step(t, "disable")
if got := harness.DisableSerialConsoleAccess(t, fix.AWS); got {
t.Fatalf("disable: action returned enabled=true")
}
if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); got {
t.Fatalf("disable: subsequent get-status returned enabled")
}
harness.Detail(t, "state", "disabled")
}
+96
View File
@@ -0,0 +1,96 @@
//go:build e2e
package single
import (
"regexp"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/require"
)
// amiIDRE matches "ami-<hex>" anywhere in spx import output. The CLI prints
// the new AMI ID on a "✅ Image import complete. Image-ID (AMI): ami-..."
// line; capturing the ID alone keeps us tolerant of the surrounding format
// changing (the bash script uses a separate describe-images call entirely
// because it didn't trust the CLI output).
var amiIDRE = regexp.MustCompile(`ami-[0-9a-f]+`)
// phase4_Image imports (or rediscovers) the architecture-appropriate Ubuntu
// AMI staged by bootstrap-install.sh and stashes the ID on the fixture for
// Phase 5+. Maps to run-e2e.sh ~233–255.
func phase4_Image(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 4 — Image Management")
require.NotEmpty(t, fix.Arch, "Phase 2 must populate fix.Arch before Phase 4")
// Bash hard-coded the mapping to ubuntu-24.04-{x86_64,arm64}; keep
// parity. fix.Arch comes from DescribeInstanceTypes so it'll be the
// canonical AWS arch label.
imgName := "ubuntu-24.04-" + fix.Arch
harness.Detail(t, "image", imgName, "arch", fix.Arch)
harness.Step(t, "spx admin images import --name %s", imgName)
// wantErr=true: bootstrap-install.sh has already imported this AMI in
// most environments, so the import returns non-zero on duplicate. We
// don't care about the exit code — we care about either parsing an
// AMI ID out of the output or successfully filtering DescribeImages
// by the canonical ami-<name> tag afterwards.
out := harness.SpxRun(t, true, "admin", "images", "import", "--name", imgName)
// Try to parse the AMI ID directly out of the CLI's success line.
amiID := amiIDRE.FindString(out)
if amiID != "" {
harness.Detail(t, "spx_parsed_ami", amiID)
} else {
harness.Step(t, "spx import did not yield AMI ID; falling back to describe-images")
}
// Either confirm the parsed ID or look it up by name.
// Bash filters on `Name=name,Values=ami-${IMAGE_NAME}` — the leading
// `ami-` prefix is part of the AMI's *name* in spinifex (admin.go:487),
// not the AMI ID. Replicate exactly.
filterName := "ami-" + imgName
describeOut, err := fix.AWS.EC2.DescribeImages(&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
{Name: aws.String("name"), Values: []*string{aws.String(filterName)}},
},
})
require.NoErrorf(t, err, "describe-images filter name=%s", filterName)
require.NotEmptyf(t, describeOut.Images,
"no AMI matched name=%s — bootstrap-install.sh did not stage it and the import failed:\n%s",
filterName, truncate(out, 2000))
resolvedID := aws.StringValue(describeOut.Images[0].ImageId)
require.NotEmpty(t, resolvedID, "describe-images returned blank ImageId")
if amiID != "" && amiID != resolvedID {
// Parsed an AMI ID from spx output but it doesn't match what
// describe-images by name returns — surface the mismatch loudly
// rather than silently preferring one source over the other.
t.Fatalf("spx output AMI %s != describe-images AMI %s for name %s",
amiID, resolvedID, filterName)
}
fix.AMIID = resolvedID
harness.Step(t, "describe-images by AMI ID (verify exactly one match)")
byID, err := fix.AWS.EC2.DescribeImages(&ec2.DescribeImagesInput{
ImageIds: []*string{aws.String(fix.AMIID)},
})
require.NoErrorf(t, err, "describe-images %s", fix.AMIID)
require.Lenf(t, byID.Images, 1, "expected exactly 1 image for %s, got %d", fix.AMIID, len(byID.Images))
require.Equal(t, fix.AMIID, aws.StringValue(byID.Images[0].ImageId), "round-trip AMI ID mismatch")
harness.Detail(t, "ami", fix.AMIID, "name", aws.StringValue(byID.Images[0].Name))
}
// truncate is used to keep failure messages from dumping arbitrarily large
// spx CLI output verbatim into the test log.
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "...[truncated]"
}
+110
View File
@@ -0,0 +1,110 @@
//go:build e2e
package single
import (
"crypto/rand"
"crypto/rsa"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
// phase3_KeyPairs exercises CreateKeyPair / ImportKeyPair / DescribeKeyPairs /
// DeleteKeyPair. test-key-1 is kept around for Phase 5 (instance launch);
// test-key-2 is created via import then deleted within this phase. Maps to
// run-e2e.sh ~204–231.
func phase3_KeyPairs(t *testing.T, fix *Fixture) {
harness.Phase(t, "Phase 3 — SSH Key Management")
const key1 = "test-key-1"
const key2 = "test-key-2"
// Best-effort clean-up of leftovers from a prior failed run so this
// phase is idempotent — bash relies on `set -e` failing loudly, but
// in Go we'd rather take the deterministic path and recreate.
_, _ = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key1)})
_, _ = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key2)})
harness.Step(t, "create-key-pair %q", key1)
created, err := fix.AWS.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{
KeyName: aws.String(key1),
})
require.NoError(t, err, "create-key-pair %s", key1)
material := aws.StringValue(created.KeyMaterial)
require.NotEmpty(t, material, "create-key-pair returned empty KeyMaterial")
require.True(t, strings.HasPrefix(material, "-----BEGIN"),
"KeyMaterial must be a PEM block (got prefix %q)", firstN(material, 32))
pemPath := filepath.Join(t.TempDir(), key1+".pem")
require.NoError(t, os.WriteFile(pemPath, []byte(material), 0o600), "write PEM")
fix.KeyName = key1
fix.KeyPath = pemPath
harness.Detail(t, "key", key1, "pem", pemPath)
// TODO(stage-G): register a t.Cleanup that deletes key1 once the whole
// TestSingleNode test finishes. Phase 5+ still need this key, so we
// can't bind it to the Phase 3 subtest. Stage G's teardown_test.go
// will own the final cleanup.
harness.Step(t, "import-key-pair %q (local-generated RSA)", key2)
pubMaterial := generateImportPubKey(t)
_, err = fix.AWS.EC2.ImportKeyPair(&ec2.ImportKeyPairInput{
KeyName: aws.String(key2),
PublicKeyMaterial: pubMaterial,
})
require.NoError(t, err, "import-key-pair %s", key2)
harness.Detail(t, "imported", key2)
harness.Step(t, "describe-key-pairs (both present)")
listed := describeKeyNames(t, fix)
assert.Contains(t, listed, key1, "describe-key-pairs missing %s (got %v)", key1, listed)
assert.Contains(t, listed, key2, "describe-key-pairs missing %s (got %v)", key2, listed)
harness.Step(t, "delete %q", key2)
_, err = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key2)})
require.NoError(t, err, "delete-key-pair %s", key2)
harness.Step(t, "describe-key-pairs (only %s remains)", key1)
remaining := describeKeyNames(t, fix)
assert.Contains(t, remaining, key1, "describe-key-pairs lost %s after deleting %s", key1, key2)
assert.NotContains(t, remaining, key2, "describe-key-pairs still lists %s after delete", key2)
}
// generateImportPubKey returns an OpenSSH-formatted public key (matching the
// `ssh-keygen -t rsa` output the bash script feeds into import-key-pair).
// 2048-bit to match the bash key length.
func generateImportPubKey(t *testing.T) []byte {
t.Helper()
priv, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err, "generate RSA key")
pub, err := ssh.NewPublicKey(&priv.PublicKey)
require.NoError(t, err, "ssh.NewPublicKey")
return ssh.MarshalAuthorizedKey(pub)
}
func describeKeyNames(t *testing.T, fix *Fixture) []string {
t.Helper()
out, err := fix.AWS.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{})
require.NoError(t, err, "describe-key-pairs")
names := make([]string, 0, len(out.KeyPairs))
for _, kp := range out.KeyPairs {
names = append(names, aws.StringValue(kp.KeyName))
}
return names
}
func firstN(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n]
}
+127
View File
@@ -0,0 +1,127 @@
//go:build e2e
// Package single is the Go port of run-e2e.sh — the single-node E2E suite
// that drives the full EC2/IAM lifecycle against a locally-bootstrapped
// Spinifex cluster. Each phase from the bash driver runs as a sequential
// subtest under TestSingleNode against one shared Fixture; ordering is
// contractual because Phase 5+ depend on the AMI / key pair / instance
// staged by Phase 1–4.
package single
import (
"bufio"
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// Fixture carries state across the sequential Phase subtests of
// TestSingleNode. Mirrors the env vars run-e2e.sh threads between phases
// (AMI_ID, INSTANCE_ID, KEY_NAME, etc).
type Fixture struct {
Env *harness.Env
AWS *harness.AWSClient
Artifacts string
Arch string // x86_64 | arm64
AMIID string // Phase 4
InstanceType string // selected nano type (Phase 2)
AZName string // Phase 2
KeyName string // Phase 3
KeyPath string // PEM written by Phase 3
Instance *ec2.Instance // Phase 5 primary (Stage C populates)
InstanceID string
RootVolumeID string
SSHHost string
SSHPort int
DefaultVPCID string
DefaultSGID string
DefaultSubnetID string
VolumeID string // Phase 5b
SnapshotID string // Phase 5c
CopySnapshotID string // Phase 5c
CustomAMIID string // Phase 5e
CustomAMISnapID string // Phase 5e backing
OVNAvailable bool // gates 8b-e
PoolMode bool // gates 8b / 8d
}
// TestSingleNode is the Go port of run-e2e.sh. Phases run as sequential
// subtests against a single shared Fixture — ordering matches the bash
// driver because Phase 5+ depend on the AMI/key/instance staged by 1–4.
func TestSingleNode(t *testing.T) {
env := harness.LoadEnv(t)
if env.Mode != harness.ModeSingle {
t.Skipf("TestSingleNode requires SPINIFEX_MODE=single (got %q)", env.Mode)
}
fix := &Fixture{
Env: env,
AWS: harness.NewAWSClient(t, env),
Artifacts: harness.ArtifactDir(t, env),
}
fix.PoolMode = detectPoolMode(env)
// Phase ordering is contractual — do not parallelise.
t.Run("Phase1_Environment", func(t *testing.T) { phase1_Environment(t, fix) })
t.Run("Phase1b_ClusterStats", func(t *testing.T) { phase1b_ClusterStats(t, fix) })
t.Run("Phase2_Discovery", func(t *testing.T) { phase2_Discovery(t, fix) })
t.Run("Phase2b_SerialConsole", func(t *testing.T) { phase2b_SerialConsole(t, fix) })
t.Run("Phase3_KeyPairs", func(t *testing.T) { phase3_KeyPairs(t, fix) })
t.Run("Phase4_Image", func(t *testing.T) { phase4_Image(t, fix) })
// Stages C–G will append more t.Run calls here.
harness.OnFailure(t, func() {
harness.DumpCmd(t, fix.Artifacts, "ec2-describe-instances.txt",
"aws", "ec2", "describe-instances")
})
}
// detectPoolMode reads external_mode from spinifex.toml. Defaults to false
// (dev_networking) which is the single-node CI fixture. Mirrors the parser
// used by lb/lb_test.go's skipIfDevNetworking but reads the positive side —
// any non-empty external_mode value ("pool" / "nat") means external IPAM
// is in play.
func detectPoolMode(env *harness.Env) bool {
cfg := os.ExpandEnv("$HOME/spinifex/config/spinifex.toml")
if env.ConfigDir != "" {
cfg = filepath.Join(env.ConfigDir, "spinifex.toml")
}
f, err := os.Open(cfg)
if err != nil {
return false
}
defer f.Close()
scanner := bufio.NewScanner(f)
inNetwork := false
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "[") {
inNetwork = line == "[network]"
continue
}
if !inNetwork {
continue
}
if !strings.HasPrefix(line, "external_mode") {
continue
}
// external_mode = "pool" — quoted value, anything non-empty == pool mode.
if i := bytes.IndexByte([]byte(line), '='); i >= 0 {
val := strings.TrimSpace(line[i+1:])
val = strings.Trim(val, "\"'")
return val != ""
}
}
return false
}