mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
92fa467c90
* refactor(e2e/multinode): flatten phase 11 + parallelize bucket #1 + drop phase numbers Split monolithic phase11_PlacementAndNATGateway (~430 lines, 8 inline sub-steps) into 6 top-level Tests, each owning its own VPC graph via t.Cleanup and layered setup helpers (spreadSetupVPC, spreadSetupBastion, spreadSetupPrivateTrio, spreadSetupNATGW). The cleanup-ordering test flips *bool latches as each explicit teardown step succeeds so the LIFO unwind is a no-op on the success path. Add t.Parallel() to read-only / independent Tests: ClusterHealth, InstanceDistribution, GuestSSH, VPCNetworking. Deviates from the mulga-siv-127 bead's bucket-#1 listing by keeping CrossNodeGateway sequential (asserts a snapshot equality across gateways that the parallel VPC test would break by churning instance count mid-assert) and CrossNodeOps sequential (stops/starts trio[0], would race GuestSSH). Both deviations documented in tests_test.go + plan doc. Drop the bash phase numbering (phaseN_X funcs + "Phase N" log strings) from the rest of the suite — 10 funcs renamed to runX. Phase numbers were a port-time crutch; the Test names now describe each scenario and phase numbers ageing into noise as bash is retired in Stage H. * refactor(e2e/multinode): collapse spread Tests + control execution order Two changes wedged together since they touch the same files: 1. Collapse the 6 top-level TestMultinodeSpread* Tests into a single TestMultinodeSpread with 6 t.Run sub-tests sharing one VPC + bastion + private trio + NAT GW setup chain. The 6-test split paid setup cost 6× — bastion launch + private trio launch + NAT GW provision per Test pushed phase-11 wall-clock near 60min vs ~15min monolithic. Shared- setup layout keeps JUnit granularity (one row per sub-test) while restoring the monolithic budget. Setup helpers register t.Cleanup against the outer test so teardown runs once at the end (LIFO: NAT GW → trio → bastion → VPC). 2. Move the TestMultinodeSpread wrapper into tests_test.go so file-source order controls execution position. Previously the func lived in placement_nat_test.go which sorts before preflight_test.go and tests_test.go alphabetically — Spread ran FIRST in the sequential bucket, defeating Preflight's fail-fast on a sick cluster. Wrapper now runs after NodeRecovery so the cluster has been fully degrade-tested before Spread's 4-VM + NAT GW + custom-VPC graph spins up.
78 lines
2.6 KiB
Go
78 lines
2.6 KiB
Go
//go:build e2e
|
|
|
|
package multinode
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// runCrossNodeOps is the Go port of cross-node stop/start
|
|
// (run-multinode-e2e.sh:854-895). Picks the first trio instance, finds the
|
|
// node currently hosting it, then drives stop+start through gateways on
|
|
// OTHER nodes — proving the cross-node control path actually reaches the
|
|
// hosting daemon, not just the local one.
|
|
//
|
|
// State guarantee: instance ends running. Other Test*s may share the trio
|
|
// (guest SSH, etc.) so leaving it stopped would cascade-fail downstream
|
|
// tests. Bash has the same guarantee implicitly via the start poll at end.
|
|
func runCrossNodeOps(t *testing.T, fix *Fixture) {
|
|
harness.Phase(t, "Multinode — Cross-Node Stop/Start")
|
|
|
|
trio := needInstanceTrio(t, fix)
|
|
require.NotEmpty(t, trio, "trio required")
|
|
target := trio[0]
|
|
|
|
hostNode := harness.InstanceHostingNode(t, fix.Cluster, target)
|
|
require.NotNil(t, hostNode, "no node hosts %s", target)
|
|
harness.Detail(t, "target", target, "hosting_node", hostNode.Name)
|
|
|
|
stopGW, startGW := otherTwoGateways(fix.Cluster, hostNode)
|
|
harness.Detail(t, "stop_via", stopGW.Name, "start_via", startGW.Name)
|
|
|
|
harness.Step(t, "stop %s via %s", target, stopGW.Name)
|
|
stopCli := harness.AWSClientForGateway(t, fix.Env, *stopGW)
|
|
_, err := stopCli.EC2.StopInstances(&ec2.StopInstancesInput{
|
|
InstanceIds: []*string{aws.String(target)},
|
|
})
|
|
require.NoErrorf(t, err, "stop-instances via %s", stopGW.Name)
|
|
harness.WaitForInstanceState(t, stopCli, target, "stopped",
|
|
harness.WithTimeout(60*time.Second), harness.WithPoll(2*time.Second))
|
|
|
|
harness.Step(t, "start %s via %s", target, startGW.Name)
|
|
startCli := harness.AWSClientForGateway(t, fix.Env, *startGW)
|
|
_, err = startCli.EC2.StartInstances(&ec2.StartInstancesInput{
|
|
InstanceIds: []*string{aws.String(target)},
|
|
})
|
|
require.NoErrorf(t, err, "start-instances via %s", startGW.Name)
|
|
harness.WaitForInstanceState(t, startCli, target, "running",
|
|
harness.WithTimeout(60*time.Second), harness.WithPoll(2*time.Second))
|
|
}
|
|
|
|
// otherTwoGateways returns two distinct nodes neither of which is host.
|
|
// Falls back to (other, other) on a 2-node cluster (no third gateway
|
|
// available) — matches the bash fallback.
|
|
func otherTwoGateways(c *harness.Cluster, host *harness.Node) (stop, start *harness.Node) {
|
|
for i := range c.Nodes {
|
|
n := &c.Nodes[i]
|
|
if n.Index == host.Index {
|
|
continue
|
|
}
|
|
if stop == nil {
|
|
stop = n
|
|
continue
|
|
}
|
|
start = n
|
|
break
|
|
}
|
|
if start == nil {
|
|
start = stop
|
|
}
|
|
return stop, start
|
|
}
|