Files
spinifex/cmd/installer/firstboot/firstboot_test.go
T
Josh Siviour 0823bf8c16 feat: OVN OVSDB RAFT clustering for multinode Spinifex (epic mulga-siv-486) (#458)
* feat: add OVSDB RAFT cluster primitives to setup-ovn.sh

Add --db-cluster-local-addr / --db-cluster-remote-addr flags so a
management node can run the OVN NB/SB databases as a native OVSDB RAFT
cluster instead of a single non-clustered ovn-central. When a local addr
is given, write a shared OVN_CTL_OPTS to /etc/default/ovn-central (NB
RAFT 6643 / SB RAFT 6644) and restart the per-DB units directly, since
the ovn-central aggregator is ExecStart=/bin/true and won't restart its
children. Empty remote addr creates the cluster (init node); a set remote
addr joins it. NB/SB client set-connection runs only on the create node
(RAFT replicates it cluster-wide).

The single-node --management path is unchanged when no cluster addrs are
supplied, so dev boxes and existing single-node clusters keep working.
--ovn-remote already passes through verbatim, so a comma-separated SB
endpoint list works for compute nodes; documented in usage.

* feat: OVN client multi-endpoint failover for RAFT clusters

Accept a comma-separated OVN NB/SB endpoint list so vpcd can target a
clustered DB. config.ParseEndpoints splits the list (trimming whitespace,
dropping empties); live.go Connect passes one libovsdb WithEndpoint per
endpoint, so the client tries them in order, fails over on disconnect, and
follows RAFT leader redirects. Single-endpoint strings still yield one
endpoint, so single-node dev is unchanged.

ovn-sbctl --db= already accepts the raw comma list, so the SB CLI
consumers (discoverChassis, gateway_claim) need no change.

* feat: write clustered OVN NB/SB endpoints across the RAFT quorum

spx admin init/join now emit comma-separated OVN NB/SB endpoint lists for
the first three quorum nodes (formation.BuildOVNDBAddrs) instead of loopback
or a single leader address, so vpcd fails over across the cluster.

firstboot defers OVN bring-up to the provisioning controller when it owns
formation (SkipFormation): a standalone single-node .db would otherwise block
the controller's clustered create/join.

* test: add multinode OVN RAFT cluster + failover e2e; document clustered topology

ovn_raft_test.go asserts NB/SB cluster/status reports a 3-server quorum with
one leader on every DB node, then stops the NB leader's ovn-central and proves
an NB write still lands via a surviving endpoint.

Docs: vpc-networking gains the multi-endpoint vpcd config and a RAFT
cluster-status debugging section; network-connections lists the 6643/6644 RAFT
ports.
2026-07-09 09:43:29 +10:00

149 lines
3.9 KiB
Go

package firstboot
import (
"os"
"path/filepath"
"strings"
"testing"
)
func makeRootDirs(t *testing.T, root string) {
t.Helper()
for _, d := range []string{
"usr/local/bin",
"etc/systemd/system",
"etc/systemd/system/multi-user.target.wants",
} {
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", d, err)
}
}
}
func TestWriteScriptNoCallbackWhenEmpty(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
if strings.Contains(string(script), "curl") {
t.Error("script should not contain curl when InstallCallback is empty")
}
}
func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
cfg := Config{
Hostname: "test-node",
ClusterRole: "init",
InstallCallback: callbackURL,
}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
content := string(script)
if !strings.Contains(content, "curl") {
t.Error("script missing curl command")
}
if !strings.Contains(content, callbackURL) {
t.Errorf("script missing callback URL %q", callbackURL)
}
}
func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init"}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
content := readScript(t, root)
if !strings.Contains(content, "setup-ovn.sh --management") {
t.Error("script should run setup-ovn --management when firstboot owns formation")
}
if !strings.Contains(content, "systemctl start ovn-central") {
t.Error("script should pre-start ovn-central when firstboot owns formation")
}
}
func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
content := readScript(t, root)
if strings.Contains(content, "setup-ovn.sh --management") {
t.Error("script must not run setup-ovn --management when a controller owns OVN")
}
if strings.Contains(content, "systemctl start ovn-central") {
t.Error("script must not pre-start ovn-central when a controller owns OVN")
}
if !strings.Contains(content, "setup-ovn deferred") {
t.Error("script should note setup-ovn is deferred under SkipFormation")
}
}
func readScript(t *testing.T, root string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
return string(b)
}
func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
root := t.TempDir()
makeRootDirs(t, root)
const callbackURL = "http://192.168.1.12/boot/done?mac=aa:bb:cc:dd:ee:ff"
cfg := Config{
Hostname: "node1",
ClusterRole: "init",
InstallCallback: callbackURL,
}
if err := Write(root, cfg); err != nil {
t.Fatalf("Write: %v", err)
}
script, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
if err != nil {
t.Fatalf("read script: %v", err)
}
content := string(script)
doneIdx := strings.Index(content, "touch \"$DONE_MARKER\"")
curlIdx := strings.Index(content, "curl")
if doneIdx < 0 {
t.Fatal("done marker not found in script")
}
if curlIdx < 0 {
t.Fatal("curl not found in script")
}
if curlIdx < doneIdx {
t.Error("curl must appear after done marker write")
}
}