mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-16 01:22:00 +00:00
Josh Siviour
92fa467c90
* refactor(e2e/multinode): flatten phase 11 + parallelize bucket #1 + drop phase numbers Split monolithic phase11_PlacementAndNATGateway (~430 lines, 8 inline sub-steps) into 6 top-level Tests, each owning its own VPC graph via t.Cleanup and layered setup helpers (spreadSetupVPC, spreadSetupBastion, spreadSetupPrivateTrio, spreadSetupNATGW). The cleanup-ordering test flips *bool latches as each explicit teardown step succeeds so the LIFO unwind is a no-op on the success path. Add t.Parallel() to read-only / independent Tests: ClusterHealth, InstanceDistribution, GuestSSH, VPCNetworking. Deviates from the mulga-siv-127 bead's bucket-#1 listing by keeping CrossNodeGateway sequential (asserts a snapshot equality across gateways that the parallel VPC test would break by churning instance count mid-assert) and CrossNodeOps sequential (stops/starts trio[0], would race GuestSSH). Both deviations documented in tests_test.go + plan doc. Drop the bash phase numbering (phaseN_X funcs + "Phase N" log strings) from the rest of the suite — 10 funcs renamed to runX. Phase numbers were a port-time crutch; the Test names now describe each scenario and phase numbers ageing into noise as bash is retired in Stage H. * refactor(e2e/multinode): collapse spread Tests + control execution order Two changes wedged together since they touch the same files: 1. Collapse the 6 top-level TestMultinodeSpread* Tests into a single TestMultinodeSpread with 6 t.Run sub-tests sharing one VPC + bastion + private trio + NAT GW setup chain. The 6-test split paid setup cost 6× — bastion launch + private trio launch + NAT GW provision per Test pushed phase-11 wall-clock near 60min vs ~15min monolithic. Shared- setup layout keeps JUnit granularity (one row per sub-test) while restoring the monolithic budget. Setup helpers register t.Cleanup against the outer test so teardown runs once at the end (LIFO: NAT GW → trio → bastion → VPC). 2. Move the TestMultinodeSpread wrapper into tests_test.go so file-source order controls execution position. Previously the func lived in placement_nat_test.go which sorts before preflight_test.go and tests_test.go alphabetically — Spread ran FIRST in the sequential bucket, defeating Preflight's fail-fast on a sick cluster. Wrapper now runs after NodeRecovery so the cluster has been fully degrade-tested before Spread's 4-VM + NAT GW + custom-VPC graph spins up.
68 lines
2.8 KiB
Go
68 lines
2.8 KiB
Go
//go:build e2e
|
|
|
|
package multinode
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// runNodeFailure is the Go port of node-failure injection
|
|
// (run-multinode-e2e.sh:898-957). Stops spinifex.target on node2, asserts the
|
|
// cluster degrades cleanly: surviving nodes still serve DescribeInstances,
|
|
// NATS reports 1 peer instead of 2, and the trio remains addressable.
|
|
//
|
|
// Registers a t.Cleanup that restarts node2 unconditionally so a cancelled
|
|
// recovery test doesn't leave the cluster broken for downstream tests.
|
|
// runNodeRecovery itself runs the recovery assertions explicitly.
|
|
func runNodeFailure(t *testing.T, fix *Fixture) {
|
|
harness.Phase(t, "Multinode — Node Failure")
|
|
|
|
// Trio must already exist — node2 going down shouldn't cause a race with
|
|
// trio launches issuing through that node's gateway.
|
|
_ = needInstanceTrio(t, fix)
|
|
require.GreaterOrEqualf(t, len(fix.Cluster.Nodes), 3, "node failure requires a 3-node cluster, have %d", len(fix.Cluster.Nodes))
|
|
node2 := fix.Cluster.Nodes[1]
|
|
node3 := fix.Cluster.Nodes[2]
|
|
local := fix.Cluster.Nodes[0]
|
|
|
|
harness.Step(t, "stop spinifex.target on %s (%s)", node2.Name, node2.Addr)
|
|
harness.StopNode(t, node2)
|
|
t.Cleanup(func() {
|
|
// Best-effort: ensure node2 comes back even if assertions below fail
|
|
// or the run is cancelled. runNodeRecovery calls StartNode again —
|
|
// systemctl start is idempotent.
|
|
harness.StartNode(t, node2)
|
|
})
|
|
|
|
// NATS detection of a peer drop is delayed by the heartbeat — bash
|
|
// uses `sleep 10`. WaitNATSPeers polls so we converge faster on a
|
|
// healthy run while still tolerating a slow detection.
|
|
harness.Step(t, "wait NATS to report 1 peer (degraded)")
|
|
fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second))
|
|
|
|
harness.Step(t, "DescribeInstanceTypes still answers via %s", local.Name)
|
|
localCli := harness.AWSClientForGateway(t, fix.Env, local)
|
|
_, err := localCli.EC2.DescribeInstanceTypes(nil)
|
|
require.NoErrorf(t, err, "%s describe-instance-types during node2 failure", local.Name)
|
|
|
|
harness.Step(t, "DescribeInstanceTypes still answers via %s", node3.Name)
|
|
node3Cli := harness.AWSClientForGateway(t, fix.Env, node3)
|
|
_, err = node3Cli.EC2.DescribeInstanceTypes(nil)
|
|
require.NoErrorf(t, err, "%s describe-instance-types during node2 failure", node3.Name)
|
|
|
|
harness.Step(t, "DescribeInstances still returns >0 from %s", local.Name)
|
|
out, err := localCli.EC2.DescribeInstances(&ec2.DescribeInstancesInput{})
|
|
require.NoError(t, err, "describe-instances during failure")
|
|
total := 0
|
|
for _, r := range out.Reservations {
|
|
total += len(r.Instances)
|
|
}
|
|
harness.Detail(t, "instances_visible", total)
|
|
require.Greaterf(t, total, 0, "no instances visible from %s during node2 failure", local.Name)
|
|
}
|