fix(e2e): skip stopped node in multinode NATS peer poll

StopNode now stops spinifex-nats.service, so WaitNATSPeers polling that
node's dead 127.0.0.1:8222 monitor port fails with curl exit 7 and never
converges. Add WithSkipNodes to exclude the stopped node from the poll;
TestMultinodeNodeFailure skips node2.
This commit is contained in:
joshsiv-mulga
2026-06-25 08:05:44 +10:00
parent d32712b727
commit 5fa12ec358
3 changed files with 23 additions and 3 deletions
+5
View File
@@ -30,6 +30,8 @@ type natsRoutezResponse struct {
// WaitNATSPeers polls every node's NATS /routez until each reports at least want
// distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222
// only, so queries run via PeerSSH + curl rather than dialling node.Addr directly.
// Nodes passed to WithSkipNodes are excluded — use it when a node has been
// stopped so its dead monitor port doesn't fail the poll.
func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) {
t.Helper()
cfg := applyOpts(pollCfg{timeout: 60 * time.Second, interval: 2 * time.Second}, opts...)
@@ -37,6 +39,9 @@ func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) {
EventuallyErr(t, func() error {
for _, n := range c.Nodes {
if _, skip := cfg.skipNodes[n.Name]; skip {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
out, err := ssh.Run(ctx, n.Addr,
fmt.Sprintf("curl -fsS http://127.0.0.1:%d/routez", natsMonitorPort))
+16 -2
View File
@@ -16,8 +16,9 @@ import (
type PollOpt func(*pollCfg)
type pollCfg struct {
timeout time.Duration
interval time.Duration
timeout time.Duration
interval time.Duration
skipNodes map[string]struct{}
}
// WithTimeout overrides the resource-default timeout.
@@ -26,6 +27,19 @@ func WithTimeout(d time.Duration) PollOpt { return func(c *pollCfg) { c.timeout
// WithPoll overrides the resource-default polling interval.
func WithPoll(d time.Duration) PollOpt { return func(c *pollCfg) { c.interval = d } }
// WithSkipNodes excludes named nodes from a per-node cluster poll. Used when a
// test has intentionally stopped a node (e.g. StopNode) and must not poll it.
func WithSkipNodes(names ...string) PollOpt {
return func(c *pollCfg) {
if c.skipNodes == nil {
c.skipNodes = make(map[string]struct{}, len(names))
}
for _, n := range names {
c.skipNodes[n] = struct{}{}
}
}
}
func applyOpts(def pollCfg, opts ...PollOpt) pollCfg {
for _, o := range opts {
o(&def)
+2 -1
View File
@@ -33,8 +33,9 @@ func runNodeFailure(t *testing.T, fix *Fixture) {
})
// NATS peer-drop detection is delayed by heartbeat; poll so we converge faster than a fixed sleep.
// Skip node2 — StopNode stopped its NATS, so its monitor port is dead.
harness.Step(t, "wait NATS to report 1 peer (degraded)")
fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second))
fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second), harness.WithSkipNodes(node2.Name))
harness.Step(t, "DescribeInstanceTypes still answers via %s", local.Name)
localCli := harness.AWSClientForGateway(t, fix.Env, local)