diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 41766d073..4f9929d1b 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -32,12 +32,12 @@ import ( "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata" - "github.com/mulgadc/spinifex/spinifex/providers/ebs" "github.com/mulgadc/spinifex/spinifex/formation" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/network/host" "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/providers/ebs" "github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu" "github.com/mulgadc/spinifex/spinifex/runtime/host/dns" "github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire" @@ -1443,10 +1443,10 @@ func runAdminInit(cmd *cobra.Command, args []string) { } // Assemble the external pool blocks rendered into spinifex.toml. - var externalPools []admin.PoolData + var externalPools []formation.PoolData switch externalMode { case "nat": - externalPools = append(externalPools, admin.PoolData{ + externalPools = append(externalPools, formation.PoolData{ Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP, PrefixLen: 24, DNSServers: dnsServers, GwLrpRangeStart: host.NATTransitGwLrpStart, GwLrpRangeEnd: host.NATTransitGwLrpEnd, @@ -1458,14 +1458,14 @@ func runAdminInit(cmd *cobra.Command, args []string) { if externalSource == "dhcp" && isNonBridgeableUplink(externalBindBridge) { dhcpMAC = "interface" } - externalPools = append(externalPools, admin.PoolData{ + externalPools = append(externalPools, formation.PoolData{ Name: "wan", Source: externalSource, BindBridge: externalBindBridge, DHCPMAC: dhcpMAC, Start: poolStart, End: poolEnd, Gateway: natPublicGateway, GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers, }) } case "pool": - externalPools = append(externalPools, admin.PoolData{ + externalPools = append(externalPools, formation.PoolData{ Name: "wan", Source: externalSource, BindBridge: externalBindBridge, Start: poolStart, End: poolEnd, Gateway: externalGateway, GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers, @@ -1742,14 +1742,14 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } - var predastoreNodes []admin.PredastoreNodeConfig + var predastoreNodes []formation.PredastoreNodeConfig for i, ip := range ips { ip = strings.TrimSpace(ip) if net.ParseIP(ip) == nil { fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP in --predastore-nodes: %s\n", ip) os.Exit(1) } - predastoreNodes = append(predastoreNodes, admin.PredastoreNodeConfig{ + predastoreNodes = append(predastoreNodes, formation.PredastoreNodeConfig{ ID: i + 1, Host: ip, }) diff --git a/cmd/spinifex/cmd/admin_nat_test.go b/cmd/spinifex/cmd/admin_nat_test.go index 7a2115b40..4e521d9f0 100644 --- a/cmd/spinifex/cmd/admin_nat_test.go +++ b/cmd/spinifex/cmd/admin_nat_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/formation" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -30,7 +31,7 @@ func TestSpinifexTomlTemplate_NATMode(t *testing.T) { ExternalMode: "nat", BridgeMode: "nat", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24, }}, } @@ -57,7 +58,7 @@ func TestSpinifexTomlTemplate_GwLrpRange(t *testing.T) { OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642", ExternalMode: "nat", BridgeMode: "nat", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24, GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254", }}, @@ -82,7 +83,7 @@ func TestSpinifexTomlTemplate_GwLrpRangeOmittedWhenUnset(t *testing.T) { OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642", ExternalMode: "pool", ExternalIface: "enp0s3", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.250", Gateway: "192.168.1.1", PrefixLen: 24, @@ -114,7 +115,7 @@ func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) { ExternalMode: "pool", ExternalIface: "enp0s3", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.250", Gateway: "192.168.1.1", PrefixLen: 24, @@ -161,7 +162,7 @@ func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) { ExternalMode: "nat", BridgeMode: "nat", - Pools: []admin.PoolData{ + Pools: []formation.PoolData{ {Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24}, { Name: "wan", Source: "dhcp", BindBridge: "wlan0", diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index 46d9d0385..7bc442130 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -317,7 +317,7 @@ external_interface = "enp0s3" // iam.nats_url must not embed a bare 0.0.0.0 — the predastore process dials // its own NATS via loopback when listening on wildcard. func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) { - nodes := []admin.PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"}, @@ -690,7 +690,7 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) { OVNSBAddr: "tcp:127.0.0.1:6642", ExternalMode: "pool", ExternalIface: "eth0", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24, }}, } @@ -721,7 +721,7 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) { OVNSBAddr: "tcp:127.0.0.1:6642", ExternalMode: "pool", ExternalIface: "eth0", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.250", Gateway: "192.168.1.1", PrefixLen: 24, @@ -744,7 +744,7 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) { settings := &admin.ConfigSettings{} nc := &formation.NetworkConfig{ ExternalMode: "pool", - Pools: []admin.PoolData{{ + Pools: []formation.PoolData{{ Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24, }}, } @@ -762,7 +762,7 @@ func TestApplyNetworkConfig_PropagatesEveryPool(t *testing.T) { settings := &admin.ConfigSettings{} nc := &formation.NetworkConfig{ ExternalMode: "nat", - Pools: []admin.PoolData{ + Pools: []formation.PoolData{ {Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24, GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254"}, {Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.200", @@ -788,8 +788,8 @@ func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) str return string(b) } -func predastoreMultinodeNodes() []admin.PredastoreNodeConfig { - return []admin.PredastoreNodeConfig{ +func predastoreMultinodeNodes() []formation.PredastoreNodeConfig { + return []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"}, } } diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 353af8d21..c511a4d1d 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -23,6 +23,7 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/formation" toml "github.com/pelletier/go-toml/v2" "gopkg.in/ini.v1" ) @@ -90,10 +91,10 @@ type ConfigSettings struct { OVNSBAddr string // External networking for public subnets - ExternalMode string // "pool", "nat" (routed), or "" (disabled) - ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") - BridgeMode string // vpcd bridge_mode; only written for "nat" (bridged modes auto-detect) - Pools []PoolData // External pools rendered in order (nat mode: transit first, then optional public pool) + ExternalMode string // "pool", "nat" (routed), or "" (disabled) + ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") + BridgeMode string // vpcd bridge_mode; only written for "nat" (bridged modes auto-detect) + Pools []formation.PoolData // External pools rendered in order (nat mode: transit first, then optional public pool) // OperatorEmail is the address collected at install time. Written under [operator] // in spinifex.toml so it survives wipes. Empty means no identity was supplied. @@ -147,32 +148,6 @@ type ConfigSettings struct { ResolverAllowFrom []string } -// PoolData is one [[network.external_pools]] block rendered into spinifex.toml. -// The json tags are load bearing: formation sends these to every joiner, so a -// renamed field silently drops a pool rather than failing to compile. -type PoolData struct { - Name string `json:"name"` // Pool name (e.g., "wan", "nat-transit") - Source string `json:"source,omitempty"` // IP source: "static" or "dhcp" - BindBridge string `json:"bind_bridge,omitempty"` // Linux bridge / interface for upstream DORA (source=dhcp only) - Start string `json:"start,omitempty"` // First IP in range (static only) - End string `json:"end,omitempty"` // Last IP in range (static only) - Gateway string `json:"gateway,omitempty"` // WAN gateway IP - GatewayIP string `json:"gateway_ip,omitempty"` // Explicit SNAT IP (overrides default of first IP in range) - PrefixLen int `json:"prefix_len,omitempty"` // Subnet prefix length (default 24) - DNSServers []string `json:"dns_servers,omitempty"` // DNS servers for VM DHCP (auto-detected from host) - DHCPMAC string `json:"dhcp_mac,omitempty"` // DHCP client MAC strategy: "derived" (default) or "interface" - // GwLrpRangeStart/End reserve gateway-LRP IPs for OVN routers. When empty - // the allocator auto-derives the top 16 host IPs of the pool subnet. - GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"` - GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"` -} - -// PredastoreNodeConfig describes a single Predastore node for multi-node config generation. -type PredastoreNodeConfig struct { - ID int - Host string -} - type ConfigFile struct { Name string Path string @@ -1008,7 +983,7 @@ const PredastoreAdminPort = 8660 // PredastoreTopology derives the cluster nodes for a set of machines: each // machine hosts a gate, one blob node and one meta node. Node IDs are unique // across the whole file, so gates take 1..n, blobs n+1..2n and metas 2n+1..3n. -func PredastoreTopology(nodes []PredastoreNodeConfig) []PredastoreClusterNode { +func PredastoreTopology(nodes []formation.PredastoreNodeConfig) []PredastoreClusterNode { out := make([]PredastoreClusterNode, 0, len(nodes)*3) for _, n := range nodes { out = append(out, PredastoreClusterNode{ID: n.ID, HostID: n.ID, Role: predastoreRoleGate, Port: predastoreGatePort}) @@ -1035,13 +1010,13 @@ func PredastoreTopology(nodes []PredastoreNodeConfig) []PredastoreClusterNode { // system key write access to it, and adds the read-only entry the resolver // authenticates with. A zero value omits all three, yielding a config no // northstar service can use. -func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, dataDir, bindIP string, compactionIntervalSeconds int, northstar NorthstarCredentials) (string, error) { +func GenerateMultiNodePredastoreConfig(templateStr string, nodes []formation.PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, dataDir, bindIP string, compactionIntervalSeconds int, northstar NorthstarCredentials) (string, error) { if len(nodes) < 2 { return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes)) } data := struct { - Nodes []PredastoreNodeConfig + Nodes []formation.PredastoreNodeConfig ClusterNodes []PredastoreClusterNode AccessKey string SecretKey string @@ -1083,7 +1058,7 @@ func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNod // FindNodeIDByIP returns the node ID for the given IP in the node list, // or 0 if the IP is not found. -func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int { +func FindNodeIDByIP(nodes []formation.PredastoreNodeConfig, ip string) int { for _, n := range nodes { if n.Host == ip { return n.ID diff --git a/spinifex/admin/admin_test.go b/spinifex/admin/admin_test.go index c3857110a..70e49ea41 100644 --- a/spinifex/admin/admin_test.go +++ b/spinifex/admin/admin_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/formation" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -816,7 +817,7 @@ id = {{.ID}} addr = "{{.Host}}" data_dir = "{{$.PredastoreDataDir}}" {{end}}` - nodes := []PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"}, @@ -838,7 +839,7 @@ host_id = {{.HostID}} role = "{{.Role}}" port = {{.Port}} {{end}}` - nodes := []PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"}, @@ -855,7 +856,7 @@ port = {{.Port}} } func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) { - topology := PredastoreTopology([]PredastoreNodeConfig{ + topology := PredastoreTopology([]formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, }) @@ -877,7 +878,7 @@ func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) { // bucket and no credential the resolver could authenticate with. func TestGenerateMultiNodePredastoreConfig_NorthstarCredentialsReachTemplate(t *testing.T) { tmpl := `access = "{{.NorthstarAccessKey}}" secret = "{{.NorthstarSecretKey}}" bucket = "{{.NorthstarBucket}}"` - nodes := []PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, } @@ -892,7 +893,7 @@ func TestGenerateMultiNodePredastoreConfig_NorthstarCredentialsReachTemplate(t * // production template's guards key off to omit the stanzas entirely. func TestGenerateMultiNodePredastoreConfig_NoNorthstarCredentials(t *testing.T) { tmpl := `access = "{{.NorthstarAccessKey}}" secret = "{{.NorthstarSecretKey}}" bucket = "{{.NorthstarBucket}}"` - nodes := []PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, } @@ -906,7 +907,7 @@ func TestGenerateMultiNodePredastoreConfig_NoNorthstarCredentials(t *testing.T) func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) { tmpl := "{{range .Nodes}}{{.ID}}{{end}}" - _, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{ + _, err := GenerateMultiNodePredastoreConfig(tmpl, []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, }, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{}) assert.Error(t, err) @@ -914,7 +915,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) { } func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) { - _, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{ + _, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []formation.PredastoreNodeConfig{ {ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"}, }, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{}) assert.Error(t, err) @@ -924,7 +925,7 @@ func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) { // --- FindNodeIDByIP --- func TestFindNodeIDByIP(t *testing.T) { - nodes := []PredastoreNodeConfig{ + nodes := []formation.PredastoreNodeConfig{ {ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"}, diff --git a/spinifex/formation/formation.go b/spinifex/formation/formation.go index fc822a7b7..9fbb9e954 100644 --- a/spinifex/formation/formation.go +++ b/spinifex/formation/formation.go @@ -16,7 +16,6 @@ import ( "time" "github.com/mulgadc/bluebottle/pkg/tlsconfig" - "github.com/mulgadc/spinifex/spinifex/admin" ) // NodeInfo describes a node participating in cluster formation. @@ -34,6 +33,36 @@ type NodeInfo struct { Services []string `json:"services,omitempty"` } +// PredastoreNodeConfig identifies one formed node in the generated Predastore +// topology. Node IDs are assigned from the deterministic formation membership +// order so every joiner renders the same topology. +type PredastoreNodeConfig struct { + ID int + Host string +} + +// PoolData is one external-network pool propagated to joining nodes during +// formation and rendered into spinifex.toml. +// +// The JSON field names are part of the formation protocol: changing one would +// silently drop a pool on a joining node rather than fail compilation. +type PoolData struct { + Name string `json:"name"` + Source string `json:"source,omitempty"` + BindBridge string `json:"bind_bridge,omitempty"` + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + Gateway string `json:"gateway,omitempty"` + GatewayIP string `json:"gateway_ip,omitempty"` + PrefixLen int `json:"prefix_len,omitempty"` + DNSServers []string `json:"dns_servers,omitempty"` + DHCPMAC string `json:"dhcp_mac,omitempty"` + // GwLrpRangeStart/End reserve gateway-LRP IPs for OVN routers. When empty + // the allocator derives the top 16 host IPs of the pool subnet. + GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"` + GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"` +} + // JoinRequest is the payload POSTed by joining nodes. type JoinRequest struct { NodeInfo @@ -70,8 +99,8 @@ type NetworkConfig struct { // order. A list rather than one flattened pool because routed NAT has two — // the transit segment and any public pool — and the transit pool carries a // gateway-LRP range that has no equivalent in pool mode. - Pools []admin.PoolData `json:"pools,omitempty"` - PoolDNSServers []string `json:"pool_dns_servers,omitempty"` + Pools []PoolData `json:"pools,omitempty"` + PoolDNSServers []string `json:"pool_dns_servers,omitempty"` // IPSecEnabled propagates the cluster-wide intra-AZ IPsec toggle so the // joining node provisions strongSwan and flips OVS ipsec_encapsulation to diff --git a/spinifex/formation/helpers.go b/spinifex/formation/helpers.go index 7bf6a354b..3439842f1 100644 --- a/spinifex/formation/helpers.go +++ b/spinifex/formation/helpers.go @@ -5,8 +5,6 @@ import ( "maps" "slices" "strings" - - "github.com/mulgadc/spinifex/spinifex/admin" ) // ovnDBQuorum is the number of nodes that host the clustered OVN NB/SB @@ -52,7 +50,7 @@ func BuildClusterRoutes(nodes map[string]NodeInfo) []string { // BuildPredastoreNodes returns sorted PredastoreNodeConfig (1-based IDs) for // nodes running the "predastore" service. -func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfig { +func BuildPredastoreNodes(nodes map[string]NodeInfo) []PredastoreNodeConfig { predaNodes := make(map[string]NodeInfo) for k, n := range nodes { if hasService(n.Services, "predastore") { @@ -60,9 +58,9 @@ func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfi } } sorted := sortedNodes(predaNodes) - out := make([]admin.PredastoreNodeConfig, len(sorted)) + out := make([]PredastoreNodeConfig, len(sorted)) for i, n := range sorted { - out[i] = admin.PredastoreNodeConfig{ + out[i] = PredastoreNodeConfig{ ID: i + 1, Host: n.BindIP, }