mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
dad5d702e1
* fix(lbagent): stop heartbeating a load balancer once it is confirmed deleted A LoadBalancerNotFound response is the gateway's positive confirmation that the store has no record for the LB, not a connectivity failure — unlike a dial error, timeout, or 5xx, which leave its existence unknown and must keep retrying. Past the startup registration grace window (or after any prior successful heartbeat), it can no longer be the record's own replication race, so the agent now stops rather than retrying forever and filling the log with heartbeat noise after teardown. * fix(nats): enable the JetStream ENOSPC-latch watchdog timer nats-server disables JetStream process-wide on ENOSPC (server/jetstream.go handleOutOfSpace -> ShutdownJetStream) and never re-enables it itself — that decision is nats-server's own internal state, not spinifex's, so a restart is the only way back. The spinifex-nats-watchdog.timer + service already probe /healthz?js-enabled-only=true and try-restart on a sustained failure, but the timer's WantedBy=timers.target never took effect: neither setup.sh nor firstboot enabled it, so the watchdog shipped inert and a full disk needed a manual restart forever. Enable --now it alongside the rest of the unit set.
168 lines
4.6 KiB
Go
168 lines
4.6 KiB
Go
package firstboot
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func makeRootDirs(t *testing.T, root string) {
|
|
t.Helper()
|
|
for _, d := range []string{
|
|
"usr/local/bin",
|
|
"etc/systemd/system",
|
|
"etc/systemd/system/multi-user.target.wants",
|
|
} {
|
|
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", d, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
|
|
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
|
|
if err != nil {
|
|
t.Fatalf("read script: %v", err)
|
|
}
|
|
if strings.Contains(string(script), "curl") {
|
|
t.Error("script should not contain curl when InstallCallback is empty")
|
|
}
|
|
}
|
|
|
|
func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
|
|
cfg := Config{
|
|
Hostname: "test-node",
|
|
ClusterRole: "init",
|
|
InstallCallback: callbackURL,
|
|
}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
|
|
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
|
|
if err != nil {
|
|
t.Fatalf("read script: %v", err)
|
|
}
|
|
content := string(script)
|
|
|
|
if !strings.Contains(content, "curl") {
|
|
t.Error("script missing curl command")
|
|
}
|
|
if !strings.Contains(content, callbackURL) {
|
|
t.Errorf("script missing callback URL %q", callbackURL)
|
|
}
|
|
}
|
|
|
|
func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
content := readScript(t, root)
|
|
|
|
if !strings.Contains(content, "setup-ovn.sh --management") {
|
|
t.Error("script should run setup-ovn --management when firstboot owns formation")
|
|
}
|
|
if !strings.Contains(content, "systemctl start ovn-central") {
|
|
t.Error("script should pre-start ovn-central when firstboot owns formation")
|
|
}
|
|
}
|
|
|
|
func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
content := readScript(t, root)
|
|
|
|
if strings.Contains(content, "setup-ovn.sh --management") {
|
|
t.Error("script must not run setup-ovn --management when a controller owns OVN")
|
|
}
|
|
if strings.Contains(content, "systemctl start ovn-central") {
|
|
t.Error("script must not pre-start ovn-central when a controller owns OVN")
|
|
}
|
|
if !strings.Contains(content, "setup-ovn deferred") {
|
|
t.Error("script should note setup-ovn is deferred under SkipFormation")
|
|
}
|
|
}
|
|
|
|
func readScript(t *testing.T, root string) string {
|
|
t.Helper()
|
|
b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
|
|
if err != nil {
|
|
t.Fatalf("read script: %v", err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// TestWriteScriptEnablesNatsWatchdogTimer pins that firstboot activates the
|
|
// JetStream ENOSPC-latch watchdog timer. The unit file's WantedBy=timers.target
|
|
// only takes effect once enabled — without this line the watchdog is dropped
|
|
// onto disk but never runs, and a full disk needs a manual nats restart forever.
|
|
func TestWriteScriptEnablesNatsWatchdogTimer(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
content := readScript(t, root)
|
|
|
|
if !strings.Contains(content, "systemctl enable --now spinifex-nats-watchdog.timer") {
|
|
t.Error("firstboot script must enable --now spinifex-nats-watchdog.timer")
|
|
}
|
|
}
|
|
|
|
func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
|
|
root := t.TempDir()
|
|
makeRootDirs(t, root)
|
|
|
|
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
|
|
cfg := Config{
|
|
Hostname: "node1",
|
|
ClusterRole: "init",
|
|
InstallCallback: callbackURL,
|
|
}
|
|
if err := Write(root, cfg); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
|
|
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
|
|
if err != nil {
|
|
t.Fatalf("read script: %v", err)
|
|
}
|
|
content := string(script)
|
|
|
|
doneIdx := strings.Index(content, "touch \"$DONE_MARKER\"")
|
|
curlIdx := strings.Index(content, "curl")
|
|
if doneIdx < 0 {
|
|
t.Fatal("done marker not found in script")
|
|
}
|
|
if curlIdx < 0 {
|
|
t.Fatal("curl not found in script")
|
|
}
|
|
if curlIdx < doneIdx {
|
|
t.Error("curl must appear after done marker write")
|
|
}
|
|
}
|