mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
fix(e2e): skip stopped node in multinode NATS peer poll
StopNode now stops spinifex-nats.service, so WaitNATSPeers polling that node's dead 127.0.0.1:8222 monitor port fails with curl exit 7 and never converges. Add WithSkipNodes to exclude the stopped node from the poll; TestMultinodeNodeFailure skips node2.
This commit is contained in:
@@ -30,6 +30,8 @@ type natsRoutezResponse struct {
|
||||
// WaitNATSPeers polls every node's NATS /routez until each reports at least want
|
||||
// distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222
|
||||
// only, so queries run via PeerSSH + curl rather than dialling node.Addr directly.
|
||||
// Nodes passed to WithSkipNodes are excluded — use it when a node has been
|
||||
// stopped so its dead monitor port doesn't fail the poll.
|
||||
func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) {
|
||||
t.Helper()
|
||||
cfg := applyOpts(pollCfg{timeout: 60 * time.Second, interval: 2 * time.Second}, opts...)
|
||||
@@ -37,6 +39,9 @@ func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) {
|
||||
|
||||
EventuallyErr(t, func() error {
|
||||
for _, n := range c.Nodes {
|
||||
if _, skip := cfg.skipNodes[n.Name]; skip {
|
||||
continue
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
out, err := ssh.Run(ctx, n.Addr,
|
||||
fmt.Sprintf("curl -fsS http://127.0.0.1:%d/routez", natsMonitorPort))
|
||||
|
||||
@@ -16,8 +16,9 @@ import (
|
||||
type PollOpt func(*pollCfg)
|
||||
|
||||
type pollCfg struct {
|
||||
timeout time.Duration
|
||||
interval time.Duration
|
||||
timeout time.Duration
|
||||
interval time.Duration
|
||||
skipNodes map[string]struct{}
|
||||
}
|
||||
|
||||
// WithTimeout overrides the resource-default timeout.
|
||||
@@ -26,6 +27,19 @@ func WithTimeout(d time.Duration) PollOpt { return func(c *pollCfg) { c.timeout
|
||||
// WithPoll overrides the resource-default polling interval.
|
||||
func WithPoll(d time.Duration) PollOpt { return func(c *pollCfg) { c.interval = d } }
|
||||
|
||||
// WithSkipNodes excludes named nodes from a per-node cluster poll. Used when a
|
||||
// test has intentionally stopped a node (e.g. StopNode) and must not poll it.
|
||||
func WithSkipNodes(names ...string) PollOpt {
|
||||
return func(c *pollCfg) {
|
||||
if c.skipNodes == nil {
|
||||
c.skipNodes = make(map[string]struct{}, len(names))
|
||||
}
|
||||
for _, n := range names {
|
||||
c.skipNodes[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyOpts(def pollCfg, opts ...PollOpt) pollCfg {
|
||||
for _, o := range opts {
|
||||
o(&def)
|
||||
|
||||
@@ -33,8 +33,9 @@ func runNodeFailure(t *testing.T, fix *Fixture) {
|
||||
})
|
||||
|
||||
// NATS peer-drop detection is delayed by heartbeat; poll so we converge faster than a fixed sleep.
|
||||
// Skip node2 — StopNode stopped its NATS, so its monitor port is dead.
|
||||
harness.Step(t, "wait NATS to report 1 peer (degraded)")
|
||||
fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second))
|
||||
fix.Cluster.WaitNATSPeers(t, 1, harness.WithTimeout(30*time.Second), harness.WithPoll(2*time.Second), harness.WithSkipNodes(node2.Name))
|
||||
|
||||
harness.Step(t, "DescribeInstanceTypes still answers via %s", local.Name)
|
||||
localCli := harness.AWSClientForGateway(t, fix.Env, local)
|
||||
|
||||
Reference in New Issue
Block a user