test(e2e): delete seven tests with no coverage

TestFinalClusterStats and TestInstanceLaunch assert nothing beyond
what other callers of their shared helpers already imply. The
multinode baseline, volume, guest-SSH, and preflight tests duplicate
single-node coverage or exercise per-guest behaviour that isn't
cluster-specific. TestMultinodeInstanceDistribution can't fail: three
independent RunInstances calls with no placement group make a strict
spread assertion flaky and a loose one meaningless.

Preflight checks (/dev/kvm writability, peer SSH reachability) move
into requireMultiNodeFixture so the package fixture singleton still
fails fast with a clear message instead of disappearing along with
the dedicated test. The remaining callers of needInstanceTrio
(CrossNodeOps, CrossNodeGateway, NodeFailure, VolumeDurability) still
trigger the shared trio launch on their own, so removing the
distribution and volume-lifecycle tests doesn't leave it unbuilt.

Also slims the manifest-lint baseline entry for the deleted
multinode volume test.
This commit is contained in:
joshsiv-mulga
2026-07-20 15:20:23 +10:00
parent a0b3b8af32
commit 237e78cb7c
12 changed files with 53 additions and 349 deletions
-1
View File
@@ -35,7 +35,6 @@ create tests/e2e/multinode/placement_nat_test.go CreateVpc #1
create tests/e2e/multinode/placement_nat_test.go RunInstances #1
create tests/e2e/multinode/placement_nat_test.go RunInstances #2
create tests/e2e/multinode/volume_durability_test.go CreateVolume #1
create tests/e2e/multinode/volume_test.go CreateVolume #1
create tests/e2e/multinode/vpc_test.go CreateSubnet #1
create tests/e2e/multinode/vpc_test.go CreateVpc #1
create tests/e2e/multinode/vpc_test.go RunInstances #1
+1 -46
View File
@@ -5,7 +5,6 @@ package multinode
import (
"context"
"fmt"
"net"
"strings"
"testing"
"time"
@@ -17,11 +16,9 @@ import (
"github.com/stretchr/testify/require"
)
// Fresh-install reachability baselines. Run before needInstanceTrio so the
// Fresh-install reachability baseline. Run before needInstanceTrio so the
// default SG/subnet/route-table are in their pristine state.
//
// - runMultinodeDefaultSGReachabilityBaseline: proves the SG (not routing)
// gates external tcp/22 — blocked before authorize, reachable after.
// - runMultinodeSameSGCrossHostComms: proves the default SG self-reference
// rule spans chassis — ICMP between two instances on different nodes
// succeeds with no added ingress rule.
@@ -95,48 +92,6 @@ func sshCapture(pem, user, host string, port int, cmd string) (string, error) {
return string(out), err
}
// runMultinodeDefaultSGReachabilityBaseline asserts the default-deny SG gate
// on the public-IP datapath: blocked before authorize, reachable after.
func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Baseline: default-deny SG blocks external reach until authorized")
vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS)
instType, arch := needInstanceTypeArch(t, fix)
amiID := needAMI(t, fix, arch)
keyName, keyPath := needKeyPair(t, fix)
sgID := harness.EnsureSG(t, fix.Harness, vpcID, "baseline-denysg")
id := baselineLaunch(t, fix, amiID, instType, keyName, subnetID, []string{sgID})
inst := harness.WaitForInstanceState(t, fix.AWS, id, "running")
pubIP := aws.StringValue(inst.PublicIpAddress)
if pubIP == "" || pubIP == "None" {
t.Fatalf("instance %s has no public IP; the datapath it depends on is "+
"broken or the subnet does not auto-assign one (hostfwd fallback is disabled)", id)
}
harness.Detail(t, "instance", id, "public_ip", pubIP, "sg", sgID)
harness.Step(t, "asserting tcp/22 stays blocked under default-deny SG")
deadline := time.Now().Add(30 * time.Second)
for time.Now().Before(deadline) {
conn, derr := net.DialTimeout("tcp", net.JoinHostPort(pubIP, "22"), 3*time.Second)
if derr == nil {
_ = conn.Close()
t.Fatalf("tcp/22 to %s connected with NO ingress rule — default-deny SG must block external traffic", pubIP)
}
time.Sleep(3 * time.Second)
}
harness.Step(t, "authorizing tcp/22, expecting reachability")
harness.AuthorizeSSHIngress(t, fix.AWS, sgID)
harness.GuestSSHReady(t, pubIP, 22, "ubuntu", keyPath,
harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second))
out, err := sshCapture(keyPath, "ubuntu", pubIP, 22, "id")
require.NoErrorf(t, err, "ssh id after authorize: %s", out)
assert.Containsf(t, out, "ubuntu", "ssh id after authorize\n%s", out)
}
// runMultinodeSameSGCrossHostComms launches two instances on different nodes sharing
// the default SG and asserts ICMP-ping succeeds across hosts. ICMP is permitted only
// by the default SG self-reference rule, proving it is enforced across chassis.
-37
View File
@@ -1,37 +0,0 @@
//go:build e2e
package multinode
import (
"strings"
"testing"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// runInstanceDistribution launches the trio and checks at least two cluster nodes host them.
// Distribution is non-deterministic so all-on-one-node is a WARN not a failure, mirroring
// the bash driver behaviour.
func runInstanceDistribution(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Instance Distribution")
ids := needInstanceTrio(t, fix)
harness.Detail(t, "instances", ids)
harness.Step(t, "check distribution across nodes")
counts := harness.AssertSpreadAcrossNodes(t, fix.Cluster, ids, 1)
harness.Detail(t, "spread", counts, "unique_hosts", len(counts))
if len(counts) < 2 {
t.Logf("WARN: all %d instances on one node (%v) — non-fatal, scheduler quirk", len(ids), counts)
}
// spx get vms is best-effort: CLI ↔ NATS can race cluster join without
// affecting the data path. Accept non-zero exit; log missing IDs as WARN.
harness.Step(t, "spx get vms includes every launched instance (best-effort)")
vms := harness.SpxRunBestEffort(t, "get", "vms")
for _, id := range ids {
if !strings.Contains(vms, id) {
t.Logf("WARN: spx get vms did not list %s", id)
}
}
}
-81
View File
@@ -1,81 +0,0 @@
//go:build e2e
package multinode
import (
"context"
"fmt"
"os/exec"
"strconv"
"strings"
"testing"
"time"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// runGuestSSH is the Go port of guest SSH probes
// (run-multinode-e2e.sh:626-728). For every instance in the trio, resolve an
// SSH endpoint (PublicIpAddress or hosting-node hostfwd), wait until SSH
// answers, and assert `id` reports ubuntu (the AMI's stock default user, now
// that the seed is retired and cloud-init bootstraps from the Ec2 datasource).
// Also asserts the AWS-form hostname and runs `lsblk` as a root-device smoke.
func runGuestSSH(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Guest SSH")
ids := needInstanceTrio(t, fix)
_, pemPath := needKeyPair(t, fix)
for _, id := range ids {
harness.Step(t, "guest_ssh probe %s", id)
host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, id)
harness.Detail(t, "instance", id, "ssh_host", host, "ssh_port", port)
harness.GuestSSHReady(t, host, port, "ubuntu", pemPath,
harness.WithTimeout(2*time.Minute), harness.WithPoll(2*time.Second))
idOut := sshRun(t, pemPath, "ubuntu", host, port, "id")
harness.Detail(t, "instance", id, "id", strings.TrimSpace(idOut))
if !strings.Contains(idOut, "ubuntu") {
t.Fatalf("instance %s id output missing ubuntu:\n%s", id, idOut)
}
// Cutover guard: cloud-init sets the AWS-form hostname ip-<dashed-ip> from
// IMDS local-hostname, not the retired seed's spinifex-vm-<id>.
hostname := strings.TrimSpace(sshRun(t, pemPath, "ubuntu", host, port, "hostname"))
harness.Detail(t, "instance", id, "hostname", hostname)
if !strings.HasPrefix(hostname, "ip-") {
t.Fatalf("instance %s hostname %q is not the AWS form ip-<dashed-ip> "+
"rendered from IMDS — seed may not be retired", id, hostname)
}
lsblk := sshRun(t, pemPath, "ubuntu", host, port, "lsblk")
// lsblk header line + at least one device row.
harness.Detail(t, "instance", id, "lsblk_lines", strings.Count(lsblk, "\n"))
}
}
// sshRun executes cmd over SSH against (user@host:port) using pemPath as
// the private key. Returns combined stdout+stderr; fatals on non-zero
// exit. Mirrors the bash ssh invocation in phase 4 (-o flags identical).
func sshRun(t *testing.T, pemPath, user, host string, port int, cmd string) string {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
args := []string{
"-i", pemPath,
"-p", strconv.Itoa(port),
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=5",
"-o", "BatchMode=yes",
"-o", "LogLevel=ERROR",
fmt.Sprintf("%s@%s", user, host),
cmd,
}
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
if err != nil {
t.Fatalf("ssh %s@%s:%d %q failed: %v\n%s", user, host, port, cmd, err, string(out))
}
return string(out)
}
+1 -1
View File
@@ -42,7 +42,7 @@ func needKeyPair(t *testing.T, fix *Fixture) (name, pemPath string) {
return harness.EnsureKeyPair(t, fix.Harness, fix.Artifacts)
}
// Package-scoped trio. runInstanceDistribution launches; downstream tests reuse the same IDs.
// Package-scoped trio. Whichever Test* runs first triggers the launch; every other caller of needInstanceTrio reuses the same IDs.
var (
trioOnce sync.Once
trioIDs []string
+41
View File
@@ -9,9 +9,12 @@
package multinode
import (
"context"
"fmt"
"os"
"sync"
"testing"
"time"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
@@ -55,6 +58,19 @@ func requireMultiNodeFixture(t *testing.T) *Fixture {
pkgFixErr = cerr
return
}
// Pre-flight: catch a misconfigured image or broken peer networking
// before any AWS API churn, with a clear message rather than letting
// every later Test* fail on it independently.
if err := checkKVMWritable(); err != nil {
pkgFixErr = err
return
}
if err := checkPeersReachable(cluster); err != nil {
pkgFixErr = err
return
}
awsCli := harness.NewAWSClient(t, env)
h, herr := harness.NewProcessFixture(awsCli)
if herr != nil {
@@ -84,6 +100,31 @@ func requireMultiNodeFixture(t *testing.T) *Fixture {
return pkgFix
}
// checkKVMWritable fails fast if /dev/kvm is not writable on this node, which
// would otherwise surface confusingly deep inside the first instance launch.
func checkKVMWritable() error {
f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0)
if err != nil {
return fmt.Errorf("preflight: /dev/kvm not writable: %w", err)
}
return f.Close()
}
// checkPeersReachable SSHes `hostname` to every remote cluster node (skipping
// node1, the runner itself — looping back over SSH tests nothing about peer
// reachability) so a broken peer network fails here instead of mid-suite.
func checkPeersReachable(cluster *harness.Cluster) error {
ssh := harness.NewPeerSSH()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for _, n := range cluster.Nodes[1:] {
if _, err := ssh.Run(ctx, n.Addr, "hostname"); err != nil {
return fmt.Errorf("preflight: peer_ssh %s (%s): %w", n.Name, n.Addr, err)
}
}
return nil
}
// Fixture carries the per-process state shared across every Test* in this package.
// Per-phase resource IDs are memoized on Harness via need* / harness.Ensure* helpers.
type Fixture struct {
-37
View File
@@ -1,37 +0,0 @@
//go:build e2e
package multinode
import (
"context"
"os"
"testing"
"time"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// runPreflight checks /dev/kvm is writable on node1 and SSH `hostname` succeeds on every peer.
// Catches misconfigured images or broken peer networking before any AWS API churn.
func runPreflight(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Pre-flight")
harness.Step(t, "verify /dev/kvm")
f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0)
if err != nil {
t.Fatalf("/dev/kvm not writable: %v", err)
}
_ = f.Close()
harness.Step(t, "peer_ssh hostname on every remote node")
ssh := harness.NewPeerSSH()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Skip node1 (the runner): SSH back to its own loopback tests nothing about peer reachability.
for _, n := range fix.Cluster.Nodes[1:] {
if _, err := ssh.Run(ctx, n.Addr, "hostname"); err != nil {
t.Fatalf("peer_ssh %s (%s): %v", n.Name, n.Addr, err)
}
harness.Detail(t, "node", n.Name, "addr", n.Addr, "ssh", "ok")
}
}
+9 -31
View File
@@ -9,19 +9,15 @@ import (
// Top-level Test* wrappers, each delegating to a runX function. Parallel tests
// share the read-only singleton trio; sequential tests mutate cluster state
// (stop/start nodes, predastore, EIP pool) and are ordered so the cluster is
// fully restabilised before GuestSSH probes every trio member.
// TestMultinodePreflight runs sequentially first to initialise the package fixture singleton.
func TestMultinodePreflight(t *testing.T) {
runPreflight(t, requireMultiNodeFixture(t))
}
// Baseline tests run before needInstanceTrio so the default SG/subnet/route table
// are in pristine state. Both own dedicated SGs and self-cleaning instances.
func TestMultinodeDefaultSGReachabilityBaseline(t *testing.T) {
runMultinodeDefaultSGReachabilityBaseline(t, requireMultiNodeFixture(t))
}
// fully restabilised before later tests probe the trio.
//
// Pre-flight (/dev/kvm writability, peer SSH reachability) runs inside
// requireMultiNodeFixture itself, so the package fixture singleton fails
// fast with a clear message before any Test* body executes.
// Baseline test owns a dedicated SG and self-cleaning instances; it runs
// before needInstanceTrio so the default SG/subnet/route table are in
// pristine state.
func TestMultinodeSameSGCrossHostComms(t *testing.T) {
runMultinodeSameSGCrossHostComms(t, requireMultiNodeFixture(t))
}
@@ -31,11 +27,6 @@ func TestMultinodeClusterHealth(t *testing.T) {
runClusterHealth(t, requireMultiNodeFixture(t))
}
func TestMultinodeInstanceDistribution(t *testing.T) {
t.Parallel()
runInstanceDistribution(t, requireMultiNodeFixture(t))
}
// TestMultinodeDNS is sequential because it launches guests and briefly stops
// one Northstar unit while exercising resolver failover.
func TestMultinodeDNS(t *testing.T) {
@@ -49,11 +40,6 @@ func TestMultinodeJetStreamReplicas(t *testing.T) {
runJetStreamReplicas(t, requireMultiNodeFixture(t))
}
// TestMultinodeVolumeLifecycle is sequential: touches predastore state.
func TestMultinodeVolumeLifecycle(t *testing.T) {
runVolumeLifecycle(t, requireMultiNodeFixture(t))
}
// TestMultinodeVolumeDurability is sequential and declared before CrossNodeOps
// (which destabilises trio[0]'s sshd) so its guest-SSH probes hit settled VMs.
// Touches predastore state.
@@ -67,8 +53,7 @@ func TestMultinodeCrossNodeGateway(t *testing.T) {
runCrossNodeGateway(t, requireMultiNodeFixture(t))
}
// TestMultinodeCrossNodeOps is sequential: stops/starts trio[0]; GuestSSH is declared
// after this so it probes the trio only after the cluster has restabilised.
// TestMultinodeCrossNodeOps is sequential: stops/starts trio[0].
func TestMultinodeCrossNodeOps(t *testing.T) {
runCrossNodeOps(t, requireMultiNodeFixture(t))
}
@@ -92,13 +77,6 @@ func TestMultinodeSpread(t *testing.T) {
runSpread(t, requireMultiNodeFixture(t))
}
// TestMultinodeGuestSSH is sequential and declared last so it runs after the cluster has
// fully restabilised. CrossNodeOps stops/starts trio[0] and only waits for "running" — not
// sshd — so GuestSSH must not probe until the guest has settled.
func TestMultinodeGuestSSH(t *testing.T) {
runGuestSSH(t, requireMultiNodeFixture(t))
}
// TestMultinodeVPCNetworking owns 10.200.0.0/16 (no EIP use); safe to run in parallel.
func TestMultinodeVPCNetworking(t *testing.T) {
t.Parallel()
-55
View File
@@ -1,55 +0,0 @@
//go:build e2e
package multinode
import (
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/tests/e2e/harness"
"github.com/stretchr/testify/require"
)
// runVolumeLifecycle creates a 10GiB volume, attaches it to trio[0] at /dev/sdf,
// detaches, and deletes. Managed inline (not via EnsureVolume) to exercise the full
// CRUD cycle — Ensure* cleanup would race the in-test delete.
func runVolumeLifecycle(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Volume Lifecycle")
az := needAZ(t, fix)
trio := needInstanceTrio(t, fix)
require.NotEmpty(t, trio, "trio required")
target := trio[0]
harness.Step(t, "create 10GiB volume in %s", az)
create, err := fix.AWS.EC2.CreateVolume(&ec2.CreateVolumeInput{
AvailabilityZone: aws.String(az),
Size: aws.Int64(10),
})
require.NoError(t, err, "create-volume")
volID := aws.StringValue(create.VolumeId)
require.NotEmpty(t, volID, "create-volume returned empty VolumeId")
harness.Detail(t, "volume", volID)
harness.Step(t, "attach %s to %s as /dev/sdf", volID, target)
_, err = fix.AWS.EC2.AttachVolume(&ec2.AttachVolumeInput{
VolumeId: aws.String(volID),
InstanceId: aws.String(target),
Device: aws.String("/dev/sdf"),
})
require.NoError(t, err, "attach-volume")
harness.WaitForVolumeState(t, fix.AWS, volID, "in-use",
harness.WithTimeout(60*time.Second), harness.WithPoll(2*time.Second))
harness.Step(t, "detach %s", volID)
_, err = fix.AWS.EC2.DetachVolume(&ec2.DetachVolumeInput{VolumeId: aws.String(volID)})
require.NoError(t, err, "detach-volume")
harness.WaitForVolumeState(t, fix.AWS, volID, "available",
harness.WithTimeout(60*time.Second), harness.WithPoll(2*time.Second))
harness.Step(t, "delete %s", volID)
_, err = fix.AWS.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(volID)})
require.NoError(t, err, "delete-volume")
}
-18
View File
@@ -17,24 +17,6 @@ import (
"github.com/stretchr/testify/require"
)
// runInstanceLaunch bootstraps the suite's primary instance and asserts
// that the live row exposes the AMI / type / key the launch was given.
// Delegates the launch + memoization to needInstance so any sibling Test*
// hits the same cached row. Maps to run-e2e.sh ~257–343.
func runInstanceLaunch(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — Instance Lifecycle")
def := harness.EnsureDefaultVPC(t, fix.Harness)
require.NotEmpty(t, def.SGID, "default SG ID required")
harness.Detail(t, "vpc", def.VPCID, "sg", def.SGID, "subnet", def.SubnetID)
inst, rootVolumeID := needInstance(t, fix)
instanceID := aws.StringValue(inst.InstanceId)
require.NotEmpty(t, instanceID, "needInstance returned empty InstanceId")
harness.Detail(t, "instance", instanceID, "root_volume", rootVolumeID,
"root_device", aws.StringValue(inst.RootDeviceName))
}
// runInstanceClusterStats re-runs `spx get vms` now that a VM is running.
// Single-node only — multinode uses a different cluster surface. Maps to
// run-e2e.sh ~345–353.
-32
View File
@@ -1,32 +0,0 @@
//go:build e2e
package single
import (
"strings"
"testing"
"github.com/mulgadc/spinifex/tests/e2e/harness"
)
// runFinalClusterStats is a single-node sanity pass against `spx get
// vms`. Replaces the umbrella's phase9 + phase9a (deleted) — resource
// cleanup is now owned by harness.Fixture.Close, invoked from TestMain
// after every Test* in the package has run.
func runFinalClusterStats(t *testing.T, fix *Fixture) {
harness.Phase(t, "Single — Final Cluster Stats")
if fix.Env.Mode != harness.ModeSingle {
t.Skipf("Phase 9b is single-node only (mode=%s)", fix.Env.Mode)
}
out := harness.SpxGetVMs(t)
harness.Detail(t, "spx_get_vms_bytes", len(out))
// Best-effort sanity: log (don't fail) if the output unexpectedly lists
// any i-XXXXXX rows. Per-test fixture cleanups should have terminated
// every instance the suite launched in single-node mode, but other
// suites running in parallel would surface here.
if strings.Contains(out, "i-") {
t.Logf("phase9b: spx get vms still references instance-like rows (may be from concurrent suites):\n%s", out)
}
}
+1 -10
View File
@@ -11,7 +11,7 @@ import "testing"
// Parallel bucket (t.Parallel): read-only or own-everything tests that never
// mutate the singleton instance, default VPC, or default SG.
// Sequential: tests that boot, mutate, or snapshot the singleton VM; IAM tests
// that share package-scoped IAM state; TestFinalClusterStats (last gate).
// that share package-scoped IAM state.
// --- Bucket #1: parallel-safe ---
@@ -78,10 +78,6 @@ func TestGuestDNSResolution(t *testing.T) {
runGuestDNSResolution(t, requireSingleNodeFixture(t))
}
func TestInstanceLaunch(t *testing.T) {
runInstanceLaunch(t, requireSingleNodeFixture(t))
}
func TestInstanceClusterStats(t *testing.T) {
runInstanceClusterStats(t, requireSingleNodeFixture(t))
}
@@ -195,8 +191,3 @@ func TestInstanceEIP(t *testing.T) {
func TestSGToSGDatapath(t *testing.T) {
runSGToSGDatapath(t, requireSingleNodeFixture(t))
}
// TestFinalClusterStats runs as the last sequential test.
func TestFinalClusterStats(t *testing.T) {
runFinalClusterStats(t, requireSingleNodeFixture(t))
}