From 5fa12ec3588591dd9f0ba2e377c4a752ea0d3f9a Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Thu, 25 Jun 2026 08:05:44 +1000 Subject: [PATCH] fix(e2e): skip stopped node in multinode NATS peer poll StopNode now stops spinifex-nats.service, so WaitNATSPeers polling that node's dead 127.0.0.1:8222 monitor port fails with curl exit 7 and never converges. Add WithSkipNodes to exclude the stopped node from the poll; TestMultinodeNodeFailure skips node2. --- tests/e2e/harness/multinode_cluster.go | 5 +++++ tests/e2e/harness/poll.go | 18 ++++++++++++++++-- tests/e2e/multinode/node_failure_test.go | 3 ++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/e2e/harness/multinode_cluster.go b/tests/e2e/harness/multinode_cluster.go index 2388d5b45..4870b6f67 100644 --- a/tests/e2e/harness/multinode_cluster.go +++ b/tests/e2e/harness/multinode_cluster.go @@ -30,6 +30,8 @@ type natsRoutezResponse struct { // WaitNATSPeers polls every node's NATS /routez until each reports at least want // distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222 // only, so queries run via PeerSSH + curl rather than dialling node.Addr directly. +// Nodes passed to WithSkipNodes are excluded — use it when a node has been +// stopped so its dead monitor port doesn't fail the poll. func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) { t.Helper() cfg := applyOpts(pollCfg{timeout: 60 * time.Second, interval: 2 * time.Second}, opts...) @@ -37,6 +39,9 @@ func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) { EventuallyErr(t, func() error { for _, n := range c.Nodes { + if _, skip := cfg.skipNodes[n.Name]; skip { + continue + } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) out, err := ssh.Run(ctx, n.Addr, fmt.Sprintf("curl -fsS http://127.0.0.1:%d/routez", natsMonitorPort)) diff --git a/tests/e2e/harness/poll.go b/tests/e2e/harness/poll.go index dbae1aee9..4bb295f18 100644 --- a/tests/e2e/harness/poll.go +++ b/tests/e2e/harness/poll.go @@ -16,8 +16,9 @@ import ( type PollOpt func(*pollCfg) type pollCfg struct { - timeout time.Duration - interval time.Duration + timeout time.Duration + interval time.Duration + skipNodes map[string]struct{} } // WithTimeout overrides the resource-default timeout. @@ -26,6 +27,19 @@ func WithTimeout(d time.Duration) PollOpt { return func(c *pollCfg) { c.timeout // WithPoll overrides the resource-default polling interval. func WithPoll(d time.Duration) PollOpt { return func(c *pollCfg) { c.interval = d } } +// WithSkipNodes excludes named nodes from a per-node cluster poll. Used when a +// test has intentionally stopped a node (e.g. StopNode) and must not poll it. +func WithSkipNodes(names ...string) PollOpt { + return func(c *pollCfg) { + if c.skipNodes == nil { + c.skipNodes = make(map[string]struct{}, len(names)) + } + for _, n := range names { + c.skipNodes[n] = struct{}{} + } + } +} + func applyOpts(def pollCfg, opts ...PollOpt) pollCfg { for _, o := range opts { o(&def) diff --git a/tests/e2e/multinode/node_failure_test.go b/tests/e2e/multinode/node_failure_test.go index c5a0bbfc2..811f96c49 100644 --- a/tests/e2e/multinode/node_failure_test.go +++ b/tests/e2e/multinode/node_failure_test.go @@ -33,8 +33,9 @@ func runNodeFailure(t *testing.T, fix *Fixture) { }) // NATS peer-drop detection is delayed by heartbeat; poll so we converge faster than a fixed sleep. + // Skip node2 — StopNode stopped its NATS, so its monitor port is dead. harness.Step(t, "wait NATS to report 1 peer (degraded)") - fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second)) + fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second), harness.WithSkipNodes(node2.Name)) harness.Step(t, "DescribeInstanceTypes still answers via %s", local.Name) localCli := harness.AWSClientForGateway(t, fix.Env, local)