mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
fix
This commit is contained in:
@@ -5,19 +5,36 @@ package harness
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StopNode stops every spinifex unit on a remote node via systemctl. Non-fatal:
|
||||
// the cluster shutdown sequence can racily kill the SSH connection.
|
||||
// nodeServiceUnits are the PartOf=spinifex.target units StopNode stops to
|
||||
// simulate a node outage. spinifex-shutdown.service is deliberately excluded:
|
||||
// its ExecStop drains guests to stopped, which a hard outage must not do.
|
||||
var nodeServiceUnits = []string{
|
||||
"spinifex-ui.service",
|
||||
"spinifex-vpcd.service",
|
||||
"spinifex-awsgw.service",
|
||||
"spinifex-daemon.service",
|
||||
"spinifex-viperblock.service",
|
||||
"spinifex-predastore.service",
|
||||
"spinifex-nats.service",
|
||||
}
|
||||
|
||||
// StopNode simulates a hard node outage by stopping the spinifex service units
|
||||
// directly (not spinifex.target), so guests keep running (daemon
|
||||
// KillMode=process) and the target's drain ExecStop never fires. Non-fatal:
|
||||
// the shutdown sequence can racily kill the SSH connection.
|
||||
func StopNode(t *testing.T, node Node) {
|
||||
t.Helper()
|
||||
ssh := NewPeerSSH()
|
||||
// 3min covers slow 6+ unit shutdowns (predastore, NATS, awsgw, vpcd, daemon, ui).
|
||||
// 3min covers slow shutdowns of all seven units (predastore, NATS, awsgw, vpcd, daemon, ui).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
if _, err := ssh.Run(ctx, node.Addr, "sudo systemctl stop spinifex.target"); err != nil {
|
||||
cmd := "sudo systemctl stop " + strings.Join(nodeServiceUnits, " ")
|
||||
if _, err := ssh.Run(ctx, node.Addr, cmd); err != nil {
|
||||
t.Logf("StopNode %s: %v (proceeding — bash treats this as non-fatal)", node.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,9 +11,10 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// runNodeFailure stops spinifex.target on node2 and asserts clean degradation:
|
||||
// surviving nodes still serve DescribeInstances and NATS reports 1 peer.
|
||||
// Unconditionally restarts node2 in t.Cleanup so downstream tests are unaffected.
|
||||
// runNodeFailure simulates a hard outage on node2 (stops its spinifex services,
|
||||
// guests keep running) and asserts clean degradation: surviving nodes still
|
||||
// serve DescribeInstances and NATS reports 1 peer. Unconditionally restarts
|
||||
// node2 in t.Cleanup so downstream tests are unaffected.
|
||||
func runNodeFailure(t *testing.T, fix *Fixture) {
|
||||
harness.Phase(t, "Multinode — Node Failure")
|
||||
|
||||
@@ -25,7 +26,7 @@ func runNodeFailure(t *testing.T, fix *Fixture) {
|
||||
node3 := fix.Cluster.Nodes[2]
|
||||
local := fix.Cluster.Nodes[0]
|
||||
|
||||
harness.Step(t, "stop spinifex.target on %s (%s)", node2.Name, node2.Addr)
|
||||
harness.Step(t, "stop spinifex services on %s (%s) — guests stay running", node2.Name, node2.Addr)
|
||||
harness.StopNode(t, node2)
|
||||
t.Cleanup(func() {
|
||||
harness.StartNode(t, node2) // idempotent; runNodeRecovery may also call this
|
||||
|
||||
Reference in New Issue
Block a user