diff --git a/tests/e2e/harness/multinode_node_lifecycle.go b/tests/e2e/harness/multinode_node_lifecycle.go index dd23edff7..035c74242 100644 --- a/tests/e2e/harness/multinode_node_lifecycle.go +++ b/tests/e2e/harness/multinode_node_lifecycle.go @@ -5,19 +5,36 @@ package harness import ( "context" "fmt" + "strings" "testing" "time" ) -// StopNode stops every spinifex unit on a remote node via systemctl. Non-fatal: -// the cluster shutdown sequence can racily kill the SSH connection. +// nodeServiceUnits are the PartOf=spinifex.target units StopNode stops to +// simulate a node outage. spinifex-shutdown.service is deliberately excluded: +// its ExecStop drains guests to stopped, which a hard outage must not do. +var nodeServiceUnits = []string{ + "spinifex-ui.service", + "spinifex-vpcd.service", + "spinifex-awsgw.service", + "spinifex-daemon.service", + "spinifex-viperblock.service", + "spinifex-predastore.service", + "spinifex-nats.service", +} + +// StopNode simulates a hard node outage by stopping the spinifex service units +// directly (not spinifex.target), so guests keep running (daemon +// KillMode=process) and the target's drain ExecStop never fires. Non-fatal: +// the shutdown sequence can racily kill the SSH connection. func StopNode(t *testing.T, node Node) { t.Helper() ssh := NewPeerSSH() - // 3min covers slow 6+ unit shutdowns (predastore, NATS, awsgw, vpcd, daemon, ui). + // 3min covers slow shutdowns of all seven units (predastore, NATS, awsgw, vpcd, daemon, ui). ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() - if _, err := ssh.Run(ctx, node.Addr, "sudo systemctl stop spinifex.target"); err != nil { + cmd := "sudo systemctl stop " + strings.Join(nodeServiceUnits, " ") + if _, err := ssh.Run(ctx, node.Addr, cmd); err != nil { t.Logf("StopNode %s: %v (proceeding — bash treats this as non-fatal)", node.Name, err) } } diff --git a/tests/e2e/multinode/node_failure_test.go b/tests/e2e/multinode/node_failure_test.go index d9fccd99a..c5a0bbfc2 100644 --- a/tests/e2e/multinode/node_failure_test.go +++ b/tests/e2e/multinode/node_failure_test.go @@ -11,9 +11,10 @@ import ( "github.com/stretchr/testify/require" ) -// runNodeFailure stops spinifex.target on node2 and asserts clean degradation: -// surviving nodes still serve DescribeInstances and NATS reports 1 peer. -// Unconditionally restarts node2 in t.Cleanup so downstream tests are unaffected. +// runNodeFailure simulates a hard outage on node2 (stops its spinifex services, +// guests keep running) and asserts clean degradation: surviving nodes still +// serve DescribeInstances and NATS reports 1 peer. Unconditionally restarts +// node2 in t.Cleanup so downstream tests are unaffected. func runNodeFailure(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Node Failure") @@ -25,7 +26,7 @@ func runNodeFailure(t *testing.T, fix *Fixture) { node3 := fix.Cluster.Nodes[2] local := fix.Cluster.Nodes[0] - harness.Step(t, "stop spinifex.target on %s (%s)", node2.Name, node2.Addr) + harness.Step(t, "stop spinifex services on %s (%s) — guests stay running", node2.Name, node2.Addr) harness.StopNode(t, node2) t.Cleanup(func() { harness.StartNode(t, node2) // idempotent; runNodeRecovery may also call this