From 97f57d3826cdf223bbf761543dd2f07f252a42d7 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Fri, 1 May 2026 12:10:59 +1000 Subject: [PATCH] fix(vpcd,viperblockd): retry NATS connect for ISO firstboot resilience MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both services used one-shot utils.ConnectNATS in launchService. On ISO firstboot, when nats.service is starting in parallel, the dial can race ahead of NATS being ready and exit non-zero. systemd's Restart=on-failure with default StartLimitBurst=5 in 10s burns through the budget in ~25s and the unit enters failed permanently — DHCP manager never starts, vpc.dhcp.acquire requests hit no-responders and the install hangs nondeterministically. Match daemon/awsgw, which already use ConnectNATSWithRetry (5min exponential backoff). The newDHCPManagerWithRetry guard (mulga-siv-40) only helps after the dial succeeds. --- spinifex/services/viperblockd/viperblockd.go | 2 +- spinifex/services/vpcd/vpcd.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 2c5b287b0..75a8e7e20 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -171,7 +171,7 @@ func (svc *Service) Reload() (err error) { func launchService(cfg *Config) (err error) { // Connect to NATS - nc, err := utils.ConnectNATS(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) + nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) return err diff --git a/spinifex/services/vpcd/vpcd.go b/spinifex/services/vpcd/vpcd.go index 41cd0abbc..cf58131c3 100644 --- a/spinifex/services/vpcd/vpcd.go +++ b/spinifex/services/vpcd/vpcd.go @@ -301,7 +301,7 @@ func launchService(cfg *Config) error { slog.Info("OVN preflight passed (br-int exists, ovn-controller running)") // Connect to NATS - nc, err := utils.ConnectNATS(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) + nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) if err != nil { slog.Error("Failed to connect to NATS", "err", err) return err