This commit is contained in:
Julian Sommer
2026-06-24 16:37:58 +10:00
parent 78739cc93b
commit fa066a64d7
2 changed files with 26 additions and 8 deletions
+21 -4
View File
@@ -5,19 +5,36 @@ package harness
import (
"context"
"fmt"
"strings"
"testing"
"time"
)
// StopNode stops every spinifex unit on a remote node via systemctl. Non-fatal:
// the cluster shutdown sequence can racily kill the SSH connection.
// nodeServiceUnits are the PartOf=spinifex.target units StopNode stops to
// simulate a node outage. spinifex-shutdown.service is deliberately excluded:
// its ExecStop drains guests to stopped, which a hard outage must not do.
var nodeServiceUnits = []string{
"spinifex-ui.service",
"spinifex-vpcd.service",
"spinifex-awsgw.service",
"spinifex-daemon.service",
"spinifex-viperblock.service",
"spinifex-predastore.service",
"spinifex-nats.service",
}
// StopNode simulates a hard node outage by stopping the spinifex service units
// directly (not spinifex.target), so guests keep running (daemon
// KillMode=process) and the target's drain ExecStop never fires. Non-fatal:
// the shutdown sequence can racily kill the SSH connection.
func StopNode(t *testing.T, node Node) {
t.Helper()
ssh := NewPeerSSH()
// 3min covers slow 6+ unit shutdowns (predastore, NATS, awsgw, vpcd, daemon, ui).
// 3min covers slow shutdowns of all seven units (predastore, NATS, awsgw, vpcd, daemon, ui).
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
if _, err := ssh.Run(ctx, node.Addr, "sudo systemctl stop spinifex.target"); err != nil {
cmd := "sudo systemctl stop " + strings.Join(nodeServiceUnits, " ")
if _, err := ssh.Run(ctx, node.Addr, cmd); err != nil {
t.Logf("StopNode %s: %v (proceeding — bash treats this as non-fatal)", node.Name, err)
}
}
+5 -4
View File
@@ -11,9 +11,10 @@ import (
"github.com/stretchr/testify/require"
)
// runNodeFailure stops spinifex.target on node2 and asserts clean degradation:
// surviving nodes still serve DescribeInstances and NATS reports 1 peer.
// Unconditionally restarts node2 in t.Cleanup so downstream tests are unaffected.
// runNodeFailure simulates a hard outage on node2 (stops its spinifex services,
// guests keep running) and asserts clean degradation: surviving nodes still
// serve DescribeInstances and NATS reports 1 peer. Unconditionally restarts
// node2 in t.Cleanup so downstream tests are unaffected.
func runNodeFailure(t *testing.T, fix *Fixture) {
harness.Phase(t, "Multinode — Node Failure")
@@ -25,7 +26,7 @@ func runNodeFailure(t *testing.T, fix *Fixture) {
node3 := fix.Cluster.Nodes[2]
local := fix.Cluster.Nodes[0]
harness.Step(t, "stop spinifex.target on %s (%s)", node2.Name, node2.Addr)
harness.Step(t, "stop spinifex services on %s (%s) — guests stay running", node2.Name, node2.Addr)
harness.StopNode(t, node2)
t.Cleanup(func() {
harness.StartNode(t, node2) // idempotent; runNodeRecovery may also call this