diff --git a/tests/e2e/single/discovery_test.go b/tests/e2e/single/discovery_test.go new file mode 100644 index 000000000..44ac96fdc --- /dev/null +++ b/tests/e2e/single/discovery_test.go @@ -0,0 +1,190 @@ +//go:build e2e + +package single + +import ( + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// phase1_Environment verifies KVM, AWS gateway TLS reachability, daemon NATS +// readiness, and the basic region/AZ discovery calls. Maps to run-e2e.sh +// lines ~51–93. +func phase1_Environment(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 1 — Environment") + + // /dev/kvm — skip rather than fatal in local-dev to match the bash + // "exit 1" only when KVM is genuinely required (CI). Local devs without + // nested virt would otherwise be unable to even compile-run this suite. + harness.Step(t, "checking /dev/kvm") + st, err := os.Stat("/dev/kvm") + if err != nil { + t.Skipf("/dev/kvm not present (%v); single-node suite needs KVM", err) + } + if st.Mode()&0o200 == 0 { + // Bash treated non-writable as fatal — replicate. + t.Fatalf("/dev/kvm exists but is not writable (mode=%v)", st.Mode()) + } + harness.Detail(t, "kvm", "writable") + + // AWS gateway TLS handshake. Bash does `curl -sk` so it accepts any + // non-zero status as success; we use HTTPSGet with a nil pool so the + // system trust store is used (the cert suite already validates CA + // installation), and treat a non-zero HTTP status as a successful + // handshake. + harness.Step(t, "waiting for AWS gateway") + host := "127.0.0.1" + if len(fix.Env.ServiceIPs) > 0 { + host = fix.Env.ServiceIPs[0] + } + gwURL := fmt.Sprintf("https://%s:%d/", host, fix.Env.AWSGWPort) + harness.Eventually(t, func() bool { + code, _, err := harness.HTTPSGet(gwURL, nil, fix.Env.DefaultTimeout) + return err == nil && code != 0 + }, 30*time.Second, 1*time.Second, "AWS gateway not reachable at "+gwURL) + harness.Detail(t, "gateway", gwURL) + + // Daemon NATS readiness: bash polls describe-instance-types via the + // AWS CLI until it returns a non-empty list. Replicate via the SDK so + // we exercise the same NATS subscription path. + harness.Step(t, "waiting for daemon NATS subscriptions") + harness.EventuallyErr(t, func() error { + out, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) + if err != nil { + return err + } + if len(out.InstanceTypes) == 0 { + return errors.New("describe-instance-types: empty result") + } + return nil + }, 30*time.Second, 1*time.Second) + + // Region + AZ smoke calls. + harness.Step(t, "describe-regions / describe-availability-zones") + regions, err := fix.AWS.EC2.DescribeRegions(&ec2.DescribeRegionsInput{}) + require.NoError(t, err, "describe-regions") + require.NotEmpty(t, regions.Regions, "no regions returned") + + azOut, err := fix.AWS.EC2.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}) + require.NoError(t, err, "describe-availability-zones") + require.NotEmpty(t, azOut.AvailabilityZones, "no AZs returned") + fix.AZName = aws.StringValue(azOut.AvailabilityZones[0].ZoneName) + azState := aws.StringValue(azOut.AvailabilityZones[0].State) + require.Equalf(t, "available", azState, "AZ %s state %q (want available)", fix.AZName, azState) + harness.Detail(t, "az", fix.AZName, "region", aws.StringValue(azOut.AvailabilityZones[0].RegionName)) + + harness.OnFailure(t, func() { + harness.DumpCmd(t, fix.Artifacts, "phase1-az.txt", + "aws", "ec2", "describe-availability-zones") + }) +} + +// phase1b_ClusterStats exercises the spx CLI cluster surface (`get nodes`, +// `top nodes`, `get vms`). Single-node only — multinode mode is tested by a +// different scenario. Maps to run-e2e.sh ~95–127. +func phase1b_ClusterStats(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 1b — Cluster Stats CLI") + if fix.Env.Mode != harness.ModeSingle { + t.Skipf("Phase 1b is single-node only (mode=%s)", fix.Env.Mode) + } + + harness.Step(t, "spx get nodes") + nodes := harness.SpxGetNodes(t) + assert.Contains(t, nodes, "Ready", "spx get nodes should report a Ready node\n%s", nodes) + + harness.Step(t, "spx top nodes") + top := harness.SpxTopNodes(t) + // Bash matches `0/` in the resource stat column (e.g. "0/4 vCPU"). The + // presence of that fraction marker is what proves the stats column + // actually rendered. + assert.Contains(t, top, "0/", "spx top nodes should report resource stats\n%s", top) + + harness.Step(t, "spx get vms (empty before any launches)") + vms := harness.SpxGetVMs(t) + assert.Contains(t, vms, "No VMs found", + "spx get vms should be empty before Phase 5\n%s", vms) +} + +// phase2_Discovery re-asserts describe-regions / describe-availability-zones +// (cheap but it's where the bash records them as a phase boundary) and picks +// the nano instance type + architecture used throughout the rest of the run. +// Maps to run-e2e.sh ~128–164. +func phase2_Discovery(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 2 — Discovery & Metadata") + + regions, err := fix.AWS.EC2.DescribeRegions(&ec2.DescribeRegionsInput{}) + require.NoError(t, err) + require.NotEmpty(t, regions.Regions, "describe-regions empty") + + azOut, err := fix.AWS.EC2.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}) + require.NoError(t, err) + require.NotEmpty(t, azOut.AvailabilityZones, "describe-availability-zones empty") + + harness.Step(t, "discovering nano instance type") + out, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) + require.NoError(t, err) + require.NotEmpty(t, out.InstanceTypes, "describe-instance-types empty") + + // Bash picks the first match for `nano` and reads the architecture out + // of the same row. Replicate that ordering — daemons emit types in a + // stable order so picking the first nano keeps parity with the bash + // driver. + for _, it := range out.InstanceTypes { + name := aws.StringValue(it.InstanceType) + if !strings.Contains(name, "nano") { + continue + } + fix.InstanceType = name + if it.ProcessorInfo != nil && len(it.ProcessorInfo.SupportedArchitectures) > 0 { + fix.Arch = aws.StringValue(it.ProcessorInfo.SupportedArchitectures[0]) + } + break + } + if fix.InstanceType == "" { + t.Fatalf("no nano instance type available; saw %d types", len(out.InstanceTypes)) + } + if fix.Arch == "" { + t.Fatalf("instance type %s missing ProcessorInfo.SupportedArchitectures", fix.InstanceType) + } + harness.Detail(t, "instance_type", fix.InstanceType, "arch", fix.Arch, "az", fix.AZName) +} + +// phase2b_SerialConsole flips serial-console-access on then off and verifies +// each transition with both the action-returned bool and a follow-up +// get-status round-trip. Maps to run-e2e.sh ~166–202. +func phase2b_SerialConsole(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 2b — Serial Console Access") + + harness.Step(t, "default state should be disabled") + if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); got { + t.Fatalf("expected serial console default disabled, got enabled") + } + + harness.Step(t, "enable") + if got := harness.EnableSerialConsoleAccess(t, fix.AWS); !got { + t.Fatalf("enable: action returned enabled=false") + } + if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); !got { + t.Fatalf("enable: subsequent get-status returned disabled") + } + harness.Detail(t, "state", "enabled") + + harness.Step(t, "disable") + if got := harness.DisableSerialConsoleAccess(t, fix.AWS); got { + t.Fatalf("disable: action returned enabled=true") + } + if got := harness.GetSerialConsoleAccessEnabled(t, fix.AWS); got { + t.Fatalf("disable: subsequent get-status returned enabled") + } + harness.Detail(t, "state", "disabled") +} diff --git a/tests/e2e/single/image_test.go b/tests/e2e/single/image_test.go new file mode 100644 index 000000000..e2a629802 --- /dev/null +++ b/tests/e2e/single/image_test.go @@ -0,0 +1,96 @@ +//go:build e2e + +package single + +import ( + "regexp" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// amiIDRE matches "ami-" anywhere in spx import output. The CLI prints +// the new AMI ID on a "✅ Image import complete. Image-ID (AMI): ami-..." +// line; capturing the ID alone keeps us tolerant of the surrounding format +// changing (the bash script uses a separate describe-images call entirely +// because it didn't trust the CLI output). +var amiIDRE = regexp.MustCompile(`ami-[0-9a-f]+`) + +// phase4_Image imports (or rediscovers) the architecture-appropriate Ubuntu +// AMI staged by bootstrap-install.sh and stashes the ID on the fixture for +// Phase 5+. Maps to run-e2e.sh ~233–255. +func phase4_Image(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 4 — Image Management") + require.NotEmpty(t, fix.Arch, "Phase 2 must populate fix.Arch before Phase 4") + + // Bash hard-coded the mapping to ubuntu-24.04-{x86_64,arm64}; keep + // parity. fix.Arch comes from DescribeInstanceTypes so it'll be the + // canonical AWS arch label. + imgName := "ubuntu-24.04-" + fix.Arch + harness.Detail(t, "image", imgName, "arch", fix.Arch) + + harness.Step(t, "spx admin images import --name %s", imgName) + // wantErr=true: bootstrap-install.sh has already imported this AMI in + // most environments, so the import returns non-zero on duplicate. We + // don't care about the exit code — we care about either parsing an + // AMI ID out of the output or successfully filtering DescribeImages + // by the canonical ami- tag afterwards. + out := harness.SpxRun(t, true, "admin", "images", "import", "--name", imgName) + + // Try to parse the AMI ID directly out of the CLI's success line. + amiID := amiIDRE.FindString(out) + if amiID != "" { + harness.Detail(t, "spx_parsed_ami", amiID) + } else { + harness.Step(t, "spx import did not yield AMI ID; falling back to describe-images") + } + + // Either confirm the parsed ID or look it up by name. + // Bash filters on `Name=name,Values=ami-${IMAGE_NAME}` — the leading + // `ami-` prefix is part of the AMI's *name* in spinifex (admin.go:487), + // not the AMI ID. Replicate exactly. + filterName := "ami-" + imgName + describeOut, err := fix.AWS.EC2.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("name"), Values: []*string{aws.String(filterName)}}, + }, + }) + require.NoErrorf(t, err, "describe-images filter name=%s", filterName) + require.NotEmptyf(t, describeOut.Images, + "no AMI matched name=%s — bootstrap-install.sh did not stage it and the import failed:\n%s", + filterName, truncate(out, 2000)) + + resolvedID := aws.StringValue(describeOut.Images[0].ImageId) + require.NotEmpty(t, resolvedID, "describe-images returned blank ImageId") + + if amiID != "" && amiID != resolvedID { + // Parsed an AMI ID from spx output but it doesn't match what + // describe-images by name returns — surface the mismatch loudly + // rather than silently preferring one source over the other. + t.Fatalf("spx output AMI %s != describe-images AMI %s for name %s", + amiID, resolvedID, filterName) + } + fix.AMIID = resolvedID + + harness.Step(t, "describe-images by AMI ID (verify exactly one match)") + byID, err := fix.AWS.EC2.DescribeImages(&ec2.DescribeImagesInput{ + ImageIds: []*string{aws.String(fix.AMIID)}, + }) + require.NoErrorf(t, err, "describe-images %s", fix.AMIID) + require.Lenf(t, byID.Images, 1, "expected exactly 1 image for %s, got %d", fix.AMIID, len(byID.Images)) + require.Equal(t, fix.AMIID, aws.StringValue(byID.Images[0].ImageId), "round-trip AMI ID mismatch") + + harness.Detail(t, "ami", fix.AMIID, "name", aws.StringValue(byID.Images[0].Name)) +} + +// truncate is used to keep failure messages from dumping arbitrarily large +// spx CLI output verbatim into the test log. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "...[truncated]" +} diff --git a/tests/e2e/single/keypair_test.go b/tests/e2e/single/keypair_test.go new file mode 100644 index 000000000..2cb53df62 --- /dev/null +++ b/tests/e2e/single/keypair_test.go @@ -0,0 +1,110 @@ +//go:build e2e + +package single + +import ( + "crypto/rand" + "crypto/rsa" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" +) + +// phase3_KeyPairs exercises CreateKeyPair / ImportKeyPair / DescribeKeyPairs / +// DeleteKeyPair. test-key-1 is kept around for Phase 5 (instance launch); +// test-key-2 is created via import then deleted within this phase. Maps to +// run-e2e.sh ~204–231. +func phase3_KeyPairs(t *testing.T, fix *Fixture) { + harness.Phase(t, "Phase 3 — SSH Key Management") + + const key1 = "test-key-1" + const key2 = "test-key-2" + + // Best-effort clean-up of leftovers from a prior failed run so this + // phase is idempotent — bash relies on `set -e` failing loudly, but + // in Go we'd rather take the deterministic path and recreate. + _, _ = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key1)}) + _, _ = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key2)}) + + harness.Step(t, "create-key-pair %q", key1) + created, err := fix.AWS.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{ + KeyName: aws.String(key1), + }) + require.NoError(t, err, "create-key-pair %s", key1) + material := aws.StringValue(created.KeyMaterial) + require.NotEmpty(t, material, "create-key-pair returned empty KeyMaterial") + require.True(t, strings.HasPrefix(material, "-----BEGIN"), + "KeyMaterial must be a PEM block (got prefix %q)", firstN(material, 32)) + + pemPath := filepath.Join(t.TempDir(), key1+".pem") + require.NoError(t, os.WriteFile(pemPath, []byte(material), 0o600), "write PEM") + fix.KeyName = key1 + fix.KeyPath = pemPath + harness.Detail(t, "key", key1, "pem", pemPath) + + // TODO(stage-G): register a t.Cleanup that deletes key1 once the whole + // TestSingleNode test finishes. Phase 5+ still need this key, so we + // can't bind it to the Phase 3 subtest. Stage G's teardown_test.go + // will own the final cleanup. + + harness.Step(t, "import-key-pair %q (local-generated RSA)", key2) + pubMaterial := generateImportPubKey(t) + _, err = fix.AWS.EC2.ImportKeyPair(&ec2.ImportKeyPairInput{ + KeyName: aws.String(key2), + PublicKeyMaterial: pubMaterial, + }) + require.NoError(t, err, "import-key-pair %s", key2) + harness.Detail(t, "imported", key2) + + harness.Step(t, "describe-key-pairs (both present)") + listed := describeKeyNames(t, fix) + assert.Contains(t, listed, key1, "describe-key-pairs missing %s (got %v)", key1, listed) + assert.Contains(t, listed, key2, "describe-key-pairs missing %s (got %v)", key2, listed) + + harness.Step(t, "delete %q", key2) + _, err = fix.AWS.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(key2)}) + require.NoError(t, err, "delete-key-pair %s", key2) + + harness.Step(t, "describe-key-pairs (only %s remains)", key1) + remaining := describeKeyNames(t, fix) + assert.Contains(t, remaining, key1, "describe-key-pairs lost %s after deleting %s", key1, key2) + assert.NotContains(t, remaining, key2, "describe-key-pairs still lists %s after delete", key2) +} + +// generateImportPubKey returns an OpenSSH-formatted public key (matching the +// `ssh-keygen -t rsa` output the bash script feeds into import-key-pair). +// 2048-bit to match the bash key length. +func generateImportPubKey(t *testing.T) []byte { + t.Helper() + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "generate RSA key") + pub, err := ssh.NewPublicKey(&priv.PublicKey) + require.NoError(t, err, "ssh.NewPublicKey") + return ssh.MarshalAuthorizedKey(pub) +} + +func describeKeyNames(t *testing.T, fix *Fixture) []string { + t.Helper() + out, err := fix.AWS.EC2.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{}) + require.NoError(t, err, "describe-key-pairs") + names := make([]string, 0, len(out.KeyPairs)) + for _, kp := range out.KeyPairs { + names = append(names, aws.StringValue(kp.KeyName)) + } + return names +} + +func firstN(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/tests/e2e/single/main_test.go b/tests/e2e/single/main_test.go new file mode 100644 index 000000000..6a2e374a9 --- /dev/null +++ b/tests/e2e/single/main_test.go @@ -0,0 +1,127 @@ +//go:build e2e + +// Package single is the Go port of run-e2e.sh — the single-node E2E suite +// that drives the full EC2/IAM lifecycle against a locally-bootstrapped +// Spinifex cluster. Each phase from the bash driver runs as a sequential +// subtest under TestSingleNode against one shared Fixture; ordering is +// contractual because Phase 5+ depend on the AMI / key pair / instance +// staged by Phase 1–4. +package single + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// Fixture carries state across the sequential Phase subtests of +// TestSingleNode. Mirrors the env vars run-e2e.sh threads between phases +// (AMI_ID, INSTANCE_ID, KEY_NAME, etc). +type Fixture struct { + Env *harness.Env + AWS *harness.AWSClient + Artifacts string + + Arch string // x86_64 | arm64 + AMIID string // Phase 4 + InstanceType string // selected nano type (Phase 2) + AZName string // Phase 2 + KeyName string // Phase 3 + KeyPath string // PEM written by Phase 3 + + Instance *ec2.Instance // Phase 5 primary (Stage C populates) + InstanceID string + RootVolumeID string + SSHHost string + SSHPort int + DefaultVPCID string + DefaultSGID string + DefaultSubnetID string + + VolumeID string // Phase 5b + SnapshotID string // Phase 5c + CopySnapshotID string // Phase 5c + CustomAMIID string // Phase 5e + CustomAMISnapID string // Phase 5e backing + + OVNAvailable bool // gates 8b-e + PoolMode bool // gates 8b / 8d +} + +// TestSingleNode is the Go port of run-e2e.sh. Phases run as sequential +// subtests against a single shared Fixture — ordering matches the bash +// driver because Phase 5+ depend on the AMI/key/instance staged by 1–4. +func TestSingleNode(t *testing.T) { + env := harness.LoadEnv(t) + if env.Mode != harness.ModeSingle { + t.Skipf("TestSingleNode requires SPINIFEX_MODE=single (got %q)", env.Mode) + } + + fix := &Fixture{ + Env: env, + AWS: harness.NewAWSClient(t, env), + Artifacts: harness.ArtifactDir(t, env), + } + fix.PoolMode = detectPoolMode(env) + + // Phase ordering is contractual — do not parallelise. + t.Run("Phase1_Environment", func(t *testing.T) { phase1_Environment(t, fix) }) + t.Run("Phase1b_ClusterStats", func(t *testing.T) { phase1b_ClusterStats(t, fix) }) + t.Run("Phase2_Discovery", func(t *testing.T) { phase2_Discovery(t, fix) }) + t.Run("Phase2b_SerialConsole", func(t *testing.T) { phase2b_SerialConsole(t, fix) }) + t.Run("Phase3_KeyPairs", func(t *testing.T) { phase3_KeyPairs(t, fix) }) + t.Run("Phase4_Image", func(t *testing.T) { phase4_Image(t, fix) }) + + // Stages C–G will append more t.Run calls here. + + harness.OnFailure(t, func() { + harness.DumpCmd(t, fix.Artifacts, "ec2-describe-instances.txt", + "aws", "ec2", "describe-instances") + }) +} + +// detectPoolMode reads external_mode from spinifex.toml. Defaults to false +// (dev_networking) which is the single-node CI fixture. Mirrors the parser +// used by lb/lb_test.go's skipIfDevNetworking but reads the positive side — +// any non-empty external_mode value ("pool" / "nat") means external IPAM +// is in play. +func detectPoolMode(env *harness.Env) bool { + cfg := os.ExpandEnv("$HOME/spinifex/config/spinifex.toml") + if env.ConfigDir != "" { + cfg = filepath.Join(env.ConfigDir, "spinifex.toml") + } + f, err := os.Open(cfg) + if err != nil { + return false + } + defer f.Close() + + scanner := bufio.NewScanner(f) + inNetwork := false + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "[") { + inNetwork = line == "[network]" + continue + } + if !inNetwork { + continue + } + if !strings.HasPrefix(line, "external_mode") { + continue + } + // external_mode = "pool" — quoted value, anything non-empty == pool mode. + if i := bytes.IndexByte([]byte(line), '='); i >= 0 { + val := strings.TrimSpace(line[i+1:]) + val = strings.Trim(val, "\"'") + return val != "" + } + } + return false +}