gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 09:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: make formation own its protocol payloads

Browse Source
This commit is contained in:
joshsiv-mulga
2026-09-29 15:47:23 +10:00
parent 967f6904ab
commit 248f0e0054
7 changed files with 73 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/spinifex/cmd/admin.go
+7 -7
View File
@@ -32,12 +32,12 @@ import (
"github.com/mulgadc/spinifex/spinifex/admin"
"github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
"github.com/mulgadc/spinifex/spinifex/formation"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/network/host"
"github.com/mulgadc/spinifex/spinifex/objectstore"
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
"github.com/mulgadc/spinifex/spinifex/runtime/host/dns"
"github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire"
@@ -1443,10 +1443,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
}
// Assemble the external pool blocks rendered into spinifex.toml.
var externalPools []admin.PoolData
var externalPools []formation.PoolData
switch externalMode {
case "nat":
externalPools = append(externalPools, admin.PoolData{
externalPools = append(externalPools, formation.PoolData{
Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP,
PrefixLen: 24, DNSServers: dnsServers,
GwLrpRangeStart: host.NATTransitGwLrpStart, GwLrpRangeEnd: host.NATTransitGwLrpEnd,
@@ -1458,14 +1458,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
if externalSource == "dhcp" && isNonBridgeableUplink(externalBindBridge) {
dhcpMAC = "interface"
}
externalPools = append(externalPools, admin.PoolData{
externalPools = append(externalPools, formation.PoolData{
Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
DHCPMAC: dhcpMAC, Start: poolStart, End: poolEnd, Gateway: natPublicGateway,
GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
})
}
case "pool":
externalPools = append(externalPools, admin.PoolData{
externalPools = append(externalPools, formation.PoolData{
Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
Start: poolStart, End: poolEnd, Gateway: externalGateway,
GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
@@ -1742,14 +1742,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
os.Exit(1)
}
var predastoreNodes []admin.PredastoreNodeConfig
var predastoreNodes []formation.PredastoreNodeConfig
for i, ip := range ips {
ip = strings.TrimSpace(ip)
if net.ParseIP(ip) == nil {
fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP in --predastore-nodes: %s\n", ip)
os.Exit(1)
}
predastoreNodes = append(predastoreNodes, admin.PredastoreNodeConfig{
predastoreNodes = append(predastoreNodes, formation.PredastoreNodeConfig{
ID: i + 1,
Host: ip,
})
cmd/spinifex/cmd/admin_nat_test.go
+6 -5
View File
@@ -7,6 +7,7 @@ import (
"testing"
"github.com/mulgadc/spinifex/spinifex/admin"
"github.com/mulgadc/spinifex/spinifex/formation"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -30,7 +31,7 @@ func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
ExternalMode: "nat",
BridgeMode: "nat",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
}},
}
@@ -57,7 +58,7 @@ func TestSpinifexTomlTemplate_GwLrpRange(t *testing.T) {
OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
ExternalMode: "nat", BridgeMode: "nat",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254",
}},
@@ -82,7 +83,7 @@ func TestSpinifexTomlTemplate_GwLrpRangeOmittedWhenUnset(t *testing.T) {
OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
ExternalMode: "pool", ExternalIface: "enp0s3",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "wan", Source: "static",
Start: "192.168.1.150", End: "192.168.1.250",
Gateway: "192.168.1.1", PrefixLen: 24,
@@ -114,7 +115,7 @@ func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
ExternalMode: "pool",
ExternalIface: "enp0s3",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "wan", Source: "static",
Start: "192.168.1.150", End: "192.168.1.250",
Gateway: "192.168.1.1", PrefixLen: 24,
@@ -161,7 +162,7 @@ func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) {
ExternalMode: "nat",
BridgeMode: "nat",
Pools: []admin.PoolData{
Pools: []formation.PoolData{
{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24},
{
Name: "wan", Source: "dhcp", BindBridge: "wlan0",
cmd/spinifex/cmd/admin_test.go
+7 -7
View File
@@ -317,7 +317,7 @@ external_interface = "enp0s3"
// iam.nats_url must not embed a bare 0.0.0.0 — the predastore process dials
// its own NATS via loopback when listening on wildcard.
func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
nodes := []admin.PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
{ID: 3, Host: "10.0.0.3"},
@@ -690,7 +690,7 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
OVNSBAddr: "tcp:127.0.0.1:6642",
ExternalMode: "pool",
ExternalIface: "eth0",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
}},
}
@@ -721,7 +721,7 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
OVNSBAddr: "tcp:127.0.0.1:6642",
ExternalMode: "pool",
ExternalIface: "eth0",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "wan", Source: "static",
Start: "192.168.1.150", End: "192.168.1.250",
Gateway: "192.168.1.1", PrefixLen: 24,
@@ -744,7 +744,7 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
settings := &admin.ConfigSettings{}
nc := &formation.NetworkConfig{
ExternalMode: "pool",
Pools: []admin.PoolData{{
Pools: []formation.PoolData{{
Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
}},
}
@@ -762,7 +762,7 @@ func TestApplyNetworkConfig_PropagatesEveryPool(t *testing.T) {
settings := &admin.ConfigSettings{}
nc := &formation.NetworkConfig{
ExternalMode: "nat",
Pools: []admin.PoolData{
Pools: []formation.PoolData{
{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254"},
{Name: "wan", Source: "static", Start: "192.168.1.150", End: "192.168.1.200",
@@ -788,8 +788,8 @@ func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) str
return string(b)
}
func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
return []admin.PredastoreNodeConfig{
func predastoreMultinodeNodes() []formation.PredastoreNodeConfig {
return []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"},
}
}
spinifex/admin/admin.go
+9 -34
View File
@@ -23,6 +23,7 @@ import (
"time"
"github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/formation"
toml "github.com/pelletier/go-toml/v2"
"gopkg.in/ini.v1"
)
@@ -90,10 +91,10 @@ type ConfigSettings struct {
OVNSBAddr string
// External networking for public subnets
ExternalMode string // "pool", "nat" (routed), or "" (disabled)
ExternalIface string // WAN NIC name (e.g., "eth0", "eth1")
BridgeMode string // vpcd bridge_mode; only written for "nat" (bridged modes auto-detect)
Pools []PoolData // External pools rendered in order (nat mode: transit first, then optional public pool)
ExternalMode string // "pool", "nat" (routed), or "" (disabled)
ExternalIface string // WAN NIC name (e.g., "eth0", "eth1")
BridgeMode string // vpcd bridge_mode; only written for "nat" (bridged modes auto-detect)
Pools []formation.PoolData // External pools rendered in order (nat mode: transit first, then optional public pool)
// OperatorEmail is the address collected at install time. Written under [operator]
// in spinifex.toml so it survives wipes. Empty means no identity was supplied.
@@ -147,32 +148,6 @@ type ConfigSettings struct {
ResolverAllowFrom []string
}
// PoolData is one [[network.external_pools]] block rendered into spinifex.toml.
// The json tags are load bearing: formation sends these to every joiner, so a
// renamed field silently drops a pool rather than failing to compile.
type PoolData struct {
Name string `json:"name"` // Pool name (e.g., "wan", "nat-transit")
Source string `json:"source,omitempty"` // IP source: "static" or "dhcp"
BindBridge string `json:"bind_bridge,omitempty"` // Linux bridge / interface for upstream DORA (source=dhcp only)
Start string `json:"start,omitempty"` // First IP in range (static only)
End string `json:"end,omitempty"` // Last IP in range (static only)
Gateway string `json:"gateway,omitempty"` // WAN gateway IP
GatewayIP string `json:"gateway_ip,omitempty"` // Explicit SNAT IP (overrides default of first IP in range)
PrefixLen int `json:"prefix_len,omitempty"` // Subnet prefix length (default 24)
DNSServers []string `json:"dns_servers,omitempty"` // DNS servers for VM DHCP (auto-detected from host)
DHCPMAC string `json:"dhcp_mac,omitempty"` // DHCP client MAC strategy: "derived" (default) or "interface"
// GwLrpRangeStart/End reserve gateway-LRP IPs for OVN routers. When empty
// the allocator auto-derives the top 16 host IPs of the pool subnet.
GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"`
GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"`
}
// PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
type PredastoreNodeConfig struct {
ID int
Host string
}
type ConfigFile struct {
Name string
Path string
@@ -1008,7 +983,7 @@ const PredastoreAdminPort = 8660
// PredastoreTopology derives the cluster nodes for a set of machines: each
// machine hosts a gate, one blob node and one meta node. Node IDs are unique
// across the whole file, so gates take 1..n, blobs n+1..2n and metas 2n+1..3n.
func PredastoreTopology(nodes []PredastoreNodeConfig) []PredastoreClusterNode {
func PredastoreTopology(nodes []formation.PredastoreNodeConfig) []PredastoreClusterNode {
out := make([]PredastoreClusterNode, 0, len(nodes)*3)
for _, n := range nodes {
out = append(out, PredastoreClusterNode{ID: n.ID, HostID: n.ID, Role: predastoreRoleGate, Port: predastoreGatePort})
@@ -1035,13 +1010,13 @@ func PredastoreTopology(nodes []PredastoreNodeConfig) []PredastoreClusterNode {
// system key write access to it, and adds the read-only entry the resolver
// authenticates with. A zero value omits all three, yielding a config no
// northstar service can use.
func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, dataDir, bindIP string, compactionIntervalSeconds int, northstar NorthstarCredentials) (string, error) {
func GenerateMultiNodePredastoreConfig(templateStr string, nodes []formation.PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, dataDir, bindIP string, compactionIntervalSeconds int, northstar NorthstarCredentials) (string, error) {
if len(nodes) < 2 {
return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes))
}
data := struct {
Nodes []PredastoreNodeConfig
Nodes []formation.PredastoreNodeConfig
ClusterNodes []PredastoreClusterNode
AccessKey string
SecretKey string
@@ -1083,7 +1058,7 @@ func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNod
// FindNodeIDByIP returns the node ID for the given IP in the node list,
// or 0 if the IP is not found.
func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int {
func FindNodeIDByIP(nodes []formation.PredastoreNodeConfig, ip string) int {
for _, n := range nodes {
if n.Host == ip {
return n.ID
spinifex/admin/admin_test.go
+9 -8
View File
@@ -12,6 +12,7 @@ import (
"testing"
"github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/formation"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -816,7 +817,7 @@ id = {{.ID}}
addr = "{{.Host}}"
data_dir = "{{$.PredastoreDataDir}}"
{{end}}`
nodes := []PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
{ID: 3, Host: "10.0.0.3"},
@@ -838,7 +839,7 @@ host_id = {{.HostID}}
role = "{{.Role}}"
port = {{.Port}}
{{end}}`
nodes := []PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
{ID: 3, Host: "10.0.0.3"},
@@ -855,7 +856,7 @@ port = {{.Port}}
}
func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) {
topology := PredastoreTopology([]PredastoreNodeConfig{
topology := PredastoreTopology([]formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
})
@@ -877,7 +878,7 @@ func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) {
// bucket and no credential the resolver could authenticate with.
func TestGenerateMultiNodePredastoreConfig_NorthstarCredentialsReachTemplate(t *testing.T) {
tmpl := `access = "{{.NorthstarAccessKey}}" secret = "{{.NorthstarSecretKey}}" bucket = "{{.NorthstarBucket}}"`
nodes := []PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
}
@@ -892,7 +893,7 @@ func TestGenerateMultiNodePredastoreConfig_NorthstarCredentialsReachTemplate(t *
// production template's guards key off to omit the stanzas entirely.
func TestGenerateMultiNodePredastoreConfig_NoNorthstarCredentials(t *testing.T) {
tmpl := `access = "{{.NorthstarAccessKey}}" secret = "{{.NorthstarSecretKey}}" bucket = "{{.NorthstarBucket}}"`
nodes := []PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
}
@@ -906,7 +907,7 @@ func TestGenerateMultiNodePredastoreConfig_NoNorthstarCredentials(t *testing.T)
func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
tmpl := "{{range .Nodes}}{{.ID}}{{end}}"
_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
_, err := GenerateMultiNodePredastoreConfig(tmpl, []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
}, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{})
assert.Error(t, err)
@@ -914,7 +915,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
}
func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{
_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []formation.PredastoreNodeConfig{
{ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"},
}, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{})
assert.Error(t, err)
@@ -924,7 +925,7 @@ func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
// --- FindNodeIDByIP ---
func TestFindNodeIDByIP(t *testing.T) {
nodes := []PredastoreNodeConfig{
nodes := []formation.PredastoreNodeConfig{
{ID: 1, Host: "10.0.0.1"},
{ID: 2, Host: "10.0.0.2"},
{ID: 3, Host: "10.0.0.3"},
spinifex/formation/formation.go
+32 -3
View File
@@ -16,7 +16,6 @@ import (
"time"
"github.com/mulgadc/bluebottle/pkg/tlsconfig"
"github.com/mulgadc/spinifex/spinifex/admin"
)
// NodeInfo describes a node participating in cluster formation.
@@ -34,6 +33,36 @@ type NodeInfo struct {
Services []string `json:"services,omitempty"`
}
// PredastoreNodeConfig identifies one formed node in the generated Predastore
// topology. Node IDs are assigned from the deterministic formation membership
// order so every joiner renders the same topology.
type PredastoreNodeConfig struct {
ID int
Host string
}
// PoolData is one external-network pool propagated to joining nodes during
// formation and rendered into spinifex.toml.
//
// The JSON field names are part of the formation protocol: changing one would
// silently drop a pool on a joining node rather than fail compilation.
type PoolData struct {
Name string `json:"name"`
Source string `json:"source,omitempty"`
BindBridge string `json:"bind_bridge,omitempty"`
Start string `json:"start,omitempty"`
End string `json:"end,omitempty"`
Gateway string `json:"gateway,omitempty"`
GatewayIP string `json:"gateway_ip,omitempty"`
PrefixLen int `json:"prefix_len,omitempty"`
DNSServers []string `json:"dns_servers,omitempty"`
DHCPMAC string `json:"dhcp_mac,omitempty"`
// GwLrpRangeStart/End reserve gateway-LRP IPs for OVN routers. When empty
// the allocator derives the top 16 host IPs of the pool subnet.
GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"`
GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"`
}
// JoinRequest is the payload POSTed by joining nodes.
type JoinRequest struct {
NodeInfo
@@ -70,8 +99,8 @@ type NetworkConfig struct {
// order. A list rather than one flattened pool because routed NAT has two —
// the transit segment and any public pool — and the transit pool carries a
// gateway-LRP range that has no equivalent in pool mode.
Pools []admin.PoolData `json:"pools,omitempty"`
PoolDNSServers []string `json:"pool_dns_servers,omitempty"`
Pools []PoolData `json:"pools,omitempty"`
PoolDNSServers []string `json:"pool_dns_servers,omitempty"`
// IPSecEnabled propagates the cluster-wide intra-AZ IPsec toggle so the
// joining node provisions strongSwan and flips OVS ipsec_encapsulation to
spinifex/formation/helpers.go
+3 -5
View File
@@ -5,8 +5,6 @@ import (
"maps"
"slices"
"strings"
"github.com/mulgadc/spinifex/spinifex/admin"
)
// ovnDBQuorum is the number of nodes that host the clustered OVN NB/SB
@@ -52,7 +50,7 @@ func BuildClusterRoutes(nodes map[string]NodeInfo) []string {
// BuildPredastoreNodes returns sorted PredastoreNodeConfig (1-based IDs) for
// nodes running the "predastore" service.
func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfig {
func BuildPredastoreNodes(nodes map[string]NodeInfo) []PredastoreNodeConfig {
predaNodes := make(map[string]NodeInfo)
for k, n := range nodes {
if hasService(n.Services, "predastore") {
@@ -60,9 +58,9 @@ func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfi
}
}
sorted := sortedNodes(predaNodes)
out := make([]admin.PredastoreNodeConfig, len(sorted))
out := make([]PredastoreNodeConfig, len(sorted))
for i, n := range sorted {
out[i] = admin.PredastoreNodeConfig{
out[i] = PredastoreNodeConfig{
ID: i + 1,
Host: n.BindIP,
}