mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
92fa467c90
* refactor(e2e/multinode): flatten phase 11 + parallelize bucket #1 + drop phase numbers Split monolithic phase11_PlacementAndNATGateway (~430 lines, 8 inline sub-steps) into 6 top-level Tests, each owning its own VPC graph via t.Cleanup and layered setup helpers (spreadSetupVPC, spreadSetupBastion, spreadSetupPrivateTrio, spreadSetupNATGW). The cleanup-ordering test flips *bool latches as each explicit teardown step succeeds so the LIFO unwind is a no-op on the success path. Add t.Parallel() to read-only / independent Tests: ClusterHealth, InstanceDistribution, GuestSSH, VPCNetworking. Deviates from the mulga-siv-127 bead's bucket-#1 listing by keeping CrossNodeGateway sequential (asserts a snapshot equality across gateways that the parallel VPC test would break by churning instance count mid-assert) and CrossNodeOps sequential (stops/starts trio[0], would race GuestSSH). Both deviations documented in tests_test.go + plan doc. Drop the bash phase numbering (phaseN_X funcs + "Phase N" log strings) from the rest of the suite — 10 funcs renamed to runX. Phase numbers were a port-time crutch; the Test names now describe each scenario and phase numbers ageing into noise as bash is retired in Stage H. * refactor(e2e/multinode): collapse spread Tests + control execution order Two changes wedged together since they touch the same files: 1. Collapse the 6 top-level TestMultinodeSpread* Tests into a single TestMultinodeSpread with 6 t.Run sub-tests sharing one VPC + bastion + private trio + NAT GW setup chain. The 6-test split paid setup cost 6× — bastion launch + private trio launch + NAT GW provision per Test pushed phase-11 wall-clock near 60min vs ~15min monolithic. Shared- setup layout keeps JUnit granularity (one row per sub-test) while restoring the monolithic budget. Setup helpers register t.Cleanup against the outer test so teardown runs once at the end (LIFO: NAT GW → trio → bastion → VPC). 2. Move the TestMultinodeSpread wrapper into tests_test.go so file-source order controls execution position. Previously the func lived in placement_nat_test.go which sorts before preflight_test.go and tests_test.go alphabetically — Spread ran FIRST in the sequential bucket, defeating Preflight's fail-fast on a sick cluster. Wrapper now runs after NodeRecovery so the cluster has been fully degrade-tested before Spread's 4-VM + NAT GW + custom-VPC graph spins up.
72 lines
2.3 KiB
Go
72 lines
2.3 KiB
Go
//go:build e2e
|
|
|
|
package multinode
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
)
|
|
|
|
// runGuestSSH is the Go port of guest SSH probes
|
|
// (run-multinode-e2e.sh:626-728). For every instance in the trio, resolve an
|
|
// SSH endpoint (PublicIpAddress or hosting-node hostfwd), wait until SSH
|
|
// answers, and assert `id` reports ec2-user. Also runs `lsblk` as a smoke
|
|
// probe for the root-device wiring.
|
|
func runGuestSSH(t *testing.T, fix *Fixture) {
|
|
harness.Phase(t, "Multinode — Guest SSH")
|
|
|
|
ids := needInstanceTrio(t, fix)
|
|
_, pemPath := needKeyPair(t, fix)
|
|
|
|
for _, id := range ids {
|
|
harness.Step(t, "guest_ssh probe %s", id)
|
|
host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, id)
|
|
harness.Detail(t, "instance", id, "ssh_host", host, "ssh_port", port)
|
|
|
|
harness.GuestSSHReady(t, host, port, "ec2-user", pemPath,
|
|
harness.WithTimeout(2*time.Minute), harness.WithPoll(2*time.Second))
|
|
|
|
idOut := sshRun(t, pemPath, "ec2-user", host, port, "id")
|
|
harness.Detail(t, "instance", id, "id", strings.TrimSpace(idOut))
|
|
if !strings.Contains(idOut, "ec2-user") {
|
|
t.Fatalf("instance %s id output missing ec2-user:\n%s", id, idOut)
|
|
}
|
|
|
|
lsblk := sshRun(t, pemPath, "ec2-user", host, port, "lsblk")
|
|
// lsblk header line + at least one device row.
|
|
harness.Detail(t, "instance", id, "lsblk_lines", strings.Count(lsblk, "\n"))
|
|
}
|
|
}
|
|
|
|
// sshRun executes cmd over SSH against (user@host:port) using pemPath as
|
|
// the private key. Returns combined stdout+stderr; fatals on non-zero
|
|
// exit. Mirrors the bash ssh invocation in phase 4 (-o flags identical).
|
|
func sshRun(t *testing.T, pemPath, user, host string, port int, cmd string) string {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
args := []string{
|
|
"-i", pemPath,
|
|
"-p", strconv.Itoa(port),
|
|
"-o", "StrictHostKeyChecking=no",
|
|
"-o", "UserKnownHostsFile=/dev/null",
|
|
"-o", "ConnectTimeout=5",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "LogLevel=ERROR",
|
|
fmt.Sprintf("%s@%s", user, host),
|
|
cmd,
|
|
}
|
|
out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput()
|
|
if err != nil {
|
|
t.Fatalf("ssh %s@%s:%d %q failed: %v\n%s", user, host, port, cmd, err, string(out))
|
|
}
|
|
return string(out)
|
|
}
|