Files
spinifex/cmd/installer/firstboot/firstboot_test.go
T
Josh Siviour dad5d702e1 fix: stop lbagent heartbeats on positive LB deletion, and actually enable the JetStream watchdog (#643)
* fix(lbagent): stop heartbeating a load balancer once it is confirmed deleted

A LoadBalancerNotFound response is the gateway's positive confirmation that
the store has no record for the LB, not a connectivity failure — unlike a
dial error, timeout, or 5xx, which leave its existence unknown and must keep
retrying. Past the startup registration grace window (or after any prior
successful heartbeat), it can no longer be the record's own replication
race, so the agent now stops rather than retrying forever and filling the
log with heartbeat noise after teardown.

* fix(nats): enable the JetStream ENOSPC-latch watchdog timer

nats-server disables JetStream process-wide on ENOSPC (server/jetstream.go
handleOutOfSpace -> ShutdownJetStream) and never re-enables it itself —
that decision is nats-server's own internal state, not spinifex's, so a
restart is the only way back. The spinifex-nats-watchdog.timer + service
already probe /healthz?js-enabled-only=true and try-restart on a sustained
failure, but the timer's WantedBy=timers.target never took effect: neither
setup.sh nor firstboot enabled it, so the watchdog shipped inert and a full
disk needed a manual restart forever. Enable --now it alongside the rest of
the unit set.
2026-08-02 08:21:46 +10:00

168 lines
4.6 KiB
Go

package firstboot
import (
"os"
"path/filepath"
"strings"
"testing"
)
func makeRootDirs(t *testing.T, root string) {
t.Helper()
for _, d := range []string{
"usr/local/bin",
"etc/systemd/system",
"etc/systemd/system/multi-user.target.wants",
} {
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", d, err)
}
}
}
func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
if strings.Contains(string(script), "curl") {
t.Error("script should not contain curl when InstallCallback is empty")
}
}
func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
cfg := Config{
Hostname: "test-node",
ClusterRole: "init",
InstallCallback: callbackURL,
}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
content := string(script)
if !strings.Contains(content, "curl") {
t.Error("script missing curl command")
}
if !strings.Contains(content, callbackURL) {
t.Errorf("script missing callback URL %q", callbackURL)
}
}
func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
content := readScript(t, root)
if !strings.Contains(content, "setup-ovn.sh --management") {
t.Error("script should run setup-ovn --management when firstboot owns formation")
}
if !strings.Contains(content, "systemctl start ovn-central") {
t.Error("script should pre-start ovn-central when firstboot owns formation")
}
}
func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
content := readScript(t, root)
if strings.Contains(content, "setup-ovn.sh --management") {
t.Error("script must not run setup-ovn --management when a controller owns OVN")
}
if strings.Contains(content, "systemctl start ovn-central") {
t.Error("script must not pre-start ovn-central when a controller owns OVN")
}
if !strings.Contains(content, "setup-ovn deferred") {
t.Error("script should note setup-ovn is deferred under SkipFormation")
}
}
func readScript(t *testing.T, root string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
return string(b)
}
// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
// only takes effect once enabled — without this line the watchdog is dropped
// onto disk but never runs, and a full disk needs a manual nats restart forever.
func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
content := readScript(t, root)
if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
}
}
func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
cfg := Config{
Hostname: "node1",
ClusterRole: "init",
InstallCallback: callbackURL,
}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
content := string(script)
doneIdx := strings.Index(content, "touch \"$DONE_MARKER\"")
curlIdx := strings.Index(content, "curl")
if doneIdx < 0 {
t.Fatal("done marker not found in script")
}
if curlIdx < 0 {
t.Fatal("curl not found in script")
}
if curlIdx < doneIdx {
t.Error("curl must appear after done marker write")
}
}