From 7cdf630cf73cafd22eb81454a0a7dfb684e2f111 Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Wed, 25 Mar 2026 17:06:21 +1000 Subject: [PATCH] feat: propagate network config via formation server to joining nodes (mulga-759) admin init builds a NetworkConfig from external networking flags and passes it to the formation server. When joining nodes poll /formation/status, they receive the cluster-wide [network] section (external mode, pools, gateway, DNS, bootstrap VPC IDs). Each joining node auto-detects its own external_interface and wan_bridge locally. Fixes multi-node clusters having no [network] section in spinifex.toml on secondary nodes, which caused no public IP assignment and broke guest SSH in E2E tests. --- cmd/spinifex/cmd/admin.go | 91 +++++++++++++++++++++++++++- spinifex/formation/formation.go | 74 ++++++++++++++-------- spinifex/formation/formation_test.go | 28 ++++----- 3 files changed, 152 insertions(+), 41 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index c62c9e74b..c08365af4 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -765,9 +765,35 @@ func runAdminInit(cmd *cobra.Command, args []string) { isMultiNode := nodes >= 2 && bindIP != "0.0.0.0" if isMultiNode { + // Build cluster-wide network config for propagation to joining nodes + var networkConfig *formation.NetworkConfig + if externalMode != "" { + bootstrapVpcId := utils.GenerateResourceID("vpc") + bootstrapSubnetId := utils.GenerateResourceID("subnet") + bootstrapIgwId := utils.GenerateResourceID("igw") + networkConfig = &formation.NetworkConfig{ + ExternalMode: externalMode, + ExternalDHCP: useExternalDHCP, + PoolName: "wan", + PoolSource: externalSource, + PoolStart: poolStart, + PoolEnd: poolEnd, + PoolGateway: externalGateway, + PoolGatewayIP: gatewayIP, + PoolPrefixLen: externalPrefixLen, + PoolDNSServers: dnsServers, + BootstrapAccountId: admin.DefaultAccountID(), + BootstrapVpcId: bootstrapVpcId, + BootstrapSubnetId: bootstrapSubnetId, + BootstrapIgwId: bootstrapIgwId, + BootstrapCidr: handlers_ec2_vpc.DefaultVPCCidr, + BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr, + } + } + runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind, - port, nodes, formationTimeoutStr, services) + port, nodes, formationTimeoutStr, services, networkConfig) return } @@ -954,7 +980,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { // then generates configs with complete cluster topology. func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName, configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string, - port, expectedNodes int, formationTimeoutStr string, services []string) { + port, expectedNodes int, formationTimeoutStr string, services []string, networkConfig *formation.NetworkConfig) { formationTimeout, err := time.ParseDuration(formationTimeoutStr) if err != nil { fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err) @@ -998,7 +1024,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, AdminSecretKey: bootstrapResult.AdminSecretKey, } - fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData)) + fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig) // Include master key in formation server for distribution to joining nodes fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey)) @@ -1106,6 +1132,34 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, OVNSBAddr: "tcp:127.0.0.1:6642", } + // Apply cluster-wide network config to init node's config + if networkConfig != nil { + configSettings.ExternalMode = networkConfig.ExternalMode + configSettings.ExternalDHCP = networkConfig.ExternalDHCP + configSettings.PoolName = networkConfig.PoolName + configSettings.PoolSource = networkConfig.PoolSource + configSettings.PoolStart = networkConfig.PoolStart + configSettings.PoolEnd = networkConfig.PoolEnd + configSettings.PoolGateway = networkConfig.PoolGateway + configSettings.PoolGatewayIP = networkConfig.PoolGatewayIP + configSettings.PoolPrefixLen = networkConfig.PoolPrefixLen + configSettings.PoolDNSServers = networkConfig.PoolDNSServers + + configSettings.BootstrapAccountId = networkConfig.BootstrapAccountId + configSettings.BootstrapVpcId = networkConfig.BootstrapVpcId + configSettings.BootstrapSubnetId = networkConfig.BootstrapSubnetId + configSettings.BootstrapIgwId = networkConfig.BootstrapIgwId + configSettings.BootstrapCidr = networkConfig.BootstrapCidr + configSettings.BootstrapSubnetCidr = networkConfig.BootstrapSubnetCidr + + // Auto-detect local network topology for per-node config + detected, err := admin.DetectNetwork() + if err == nil && detected.WAN != nil { + configSettings.ExternalIface = detected.WAN.Name + configSettings.WanBridge = detectedWanBridge(detected) + } + } + // Generate config files configs := []admin.ConfigFile{ {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, @@ -1437,6 +1491,37 @@ func runAdminJoin(cmd *cobra.Command, args []string) { OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP), } + // Apply cluster-wide network config from leader + if statusResp.NetworkConfig != nil { + nc := statusResp.NetworkConfig + configSettings.ExternalMode = nc.ExternalMode + configSettings.ExternalDHCP = nc.ExternalDHCP + configSettings.PoolName = nc.PoolName + configSettings.PoolSource = nc.PoolSource + configSettings.PoolStart = nc.PoolStart + configSettings.PoolEnd = nc.PoolEnd + configSettings.PoolGateway = nc.PoolGateway + configSettings.PoolGatewayIP = nc.PoolGatewayIP + configSettings.PoolPrefixLen = nc.PoolPrefixLen + configSettings.PoolDNSServers = nc.PoolDNSServers + + configSettings.BootstrapAccountId = nc.BootstrapAccountId + configSettings.BootstrapVpcId = nc.BootstrapVpcId + configSettings.BootstrapSubnetId = nc.BootstrapSubnetId + configSettings.BootstrapIgwId = nc.BootstrapIgwId + configSettings.BootstrapCidr = nc.BootstrapCidr + configSettings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr + + // Auto-detect local network topology for per-node config + if nc.ExternalMode != "" { + detected, err := admin.DetectNetwork() + if err == nil && detected.WAN != nil { + configSettings.ExternalIface = detected.WAN.Name + configSettings.WanBridge = detectedWanBridge(detected) + } + } + } + // Generate config files configs := []admin.ConfigFile{ {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, diff --git a/spinifex/formation/formation.go b/spinifex/formation/formation.go index 868fb7292..c9cdd4070 100644 --- a/spinifex/formation/formation.go +++ b/spinifex/formation/formation.go @@ -38,14 +38,37 @@ type JoinResponse struct { // StatusResponse is returned by the formation server status endpoint. type StatusResponse struct { - Complete bool `json:"complete"` - Joined int `json:"joined"` - Expected int `json:"expected"` - Nodes map[string]NodeInfo `json:"nodes,omitempty"` - Credentials *SharedCredentials `json:"credentials,omitempty"` - CACert string `json:"ca_cert,omitempty"` - CAKey string `json:"ca_key,omitempty"` - MasterKey string `json:"master_key,omitempty"` + Complete bool `json:"complete"` + Joined int `json:"joined"` + Expected int `json:"expected"` + Nodes map[string]NodeInfo `json:"nodes,omitempty"` + Credentials *SharedCredentials `json:"credentials,omitempty"` + CACert string `json:"ca_cert,omitempty"` + CAKey string `json:"ca_key,omitempty"` + MasterKey string `json:"master_key,omitempty"` + NetworkConfig *NetworkConfig `json:"network_config,omitempty"` +} + +// NetworkConfig holds the cluster-wide external networking configuration +// propagated from the init node to joining nodes during formation. +type NetworkConfig struct { + ExternalMode string `json:"external_mode"` + ExternalDHCP bool `json:"external_dhcp,omitempty"` + PoolName string `json:"pool_name"` + PoolSource string `json:"pool_source,omitempty"` + PoolStart string `json:"pool_start,omitempty"` + PoolEnd string `json:"pool_end,omitempty"` + PoolGateway string `json:"pool_gateway"` + PoolGatewayIP string `json:"pool_gateway_ip,omitempty"` + PoolPrefixLen int `json:"pool_prefix_len"` + PoolDNSServers []string `json:"pool_dns_servers,omitempty"` + + BootstrapAccountId string `json:"bootstrap_account_id,omitempty"` + BootstrapVpcId string `json:"bootstrap_vpc_id,omitempty"` + BootstrapSubnetId string `json:"bootstrap_subnet_id,omitempty"` + BootstrapIgwId string `json:"bootstrap_igw_id,omitempty"` + BootstrapCidr string `json:"bootstrap_cidr,omitempty"` + BootstrapSubnetCidr string `json:"bootstrap_subnet_cidr,omitempty"` } // SharedCredentials contains the cluster-wide credentials distributed during formation. @@ -67,26 +90,28 @@ type SharedCredentials struct { // nodes have joined, the done channel is closed and full cluster data (credentials, // CA, node list) becomes available via GET /formation/status. type FormationServer struct { - mu sync.RWMutex - expected int - nodes map[string]NodeInfo - credentials *SharedCredentials - caCert string - caKey string - masterKey string - done chan struct{} - server *http.Server + mu sync.RWMutex + expected int + nodes map[string]NodeInfo + credentials *SharedCredentials + caCert string + caKey string + masterKey string + networkConfig *NetworkConfig + done chan struct{} + server *http.Server } // NewFormationServer creates a new formation server expecting the given number of nodes. -func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string) *FormationServer { +func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string, networkConfig *NetworkConfig) *FormationServer { return &FormationServer{ - expected: expected, - nodes: make(map[string]NodeInfo), - credentials: creds, - caCert: caCert, - caKey: caKey, - done: make(chan struct{}), + expected: expected, + nodes: make(map[string]NodeInfo), + credentials: creds, + caCert: caCert, + caKey: caKey, + networkConfig: networkConfig, + done: make(chan struct{}), } } @@ -252,6 +277,7 @@ func (fs *FormationServer) handleStatus(w http.ResponseWriter, r *http.Request) resp.CACert = fs.caCert resp.CAKey = fs.caKey resp.MasterKey = fs.masterKey + resp.NetworkConfig = fs.networkConfig } writeJSON(w, http.StatusOK, resp) diff --git a/spinifex/formation/formation_test.go b/spinifex/formation/formation_test.go index 21eb3ab31..5357f4107 100644 --- a/spinifex/formation/formation_test.go +++ b/spinifex/formation/formation_test.go @@ -35,7 +35,7 @@ func testNode(name, ip string) NodeInfo { func TestNewFormationServer(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem") + fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil) assert.Equal(t, 3, fs.expected) assert.Empty(t, fs.nodes) @@ -44,7 +44,7 @@ func TestNewFormationServer(t *testing.T) { func TestRegisterNode(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) err := fs.RegisterNode(testNode("node1", "10.0.0.1")) require.NoError(t, err) @@ -56,7 +56,7 @@ func TestRegisterNode(t *testing.T) { func TestRegisterDuplicateName(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) err := fs.RegisterNode(testNode("node1", "10.0.0.2")) @@ -66,7 +66,7 @@ func TestRegisterDuplicateName(t *testing.T) { func TestRegisterDuplicateIP(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) err := fs.RegisterNode(testNode("node2", "10.0.0.1")) @@ -76,7 +76,7 @@ func TestRegisterDuplicateIP(t *testing.T) { func TestIsComplete(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "", "") + fs := NewFormationServer(2, testCreds(), "", "", nil) assert.False(t, fs.IsComplete()) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) @@ -87,7 +87,7 @@ func TestIsComplete(t *testing.T) { func TestWaitForCompletion(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "", "") + fs := NewFormationServer(2, testCreds(), "", "", nil) go func() { time.Sleep(50 * time.Millisecond) @@ -102,7 +102,7 @@ func TestWaitForCompletion(t *testing.T) { func TestWaitForCompletionTimeout(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1"))) @@ -122,7 +122,7 @@ func testServer(t *testing.T, fs *FormationServer) *httptest.Server { func TestJoinEndpoint(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) ts := testServer(t, fs) defer ts.Close() @@ -142,7 +142,7 @@ func TestJoinEndpoint(t *testing.T) { func TestJoinEndpointDuplicate(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "", "") + fs := NewFormationServer(3, testCreds(), "", "", nil) ts := testServer(t, fs) defer ts.Close() @@ -161,7 +161,7 @@ func TestJoinEndpointDuplicate(t *testing.T) { func TestStatusEndpointIncomplete(t *testing.T) { t.Parallel() - fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key") + fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil) ts := testServer(t, fs) defer ts.Close() @@ -184,7 +184,7 @@ func TestStatusEndpointIncomplete(t *testing.T) { func TestStatusEndpointComplete(t *testing.T) { t.Parallel() creds := testCreds() - fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data") + fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil) ts := testServer(t, fs) defer ts.Close() @@ -209,7 +209,7 @@ func TestStatusEndpointComplete(t *testing.T) { func TestStatusEndpointMasterKey(t *testing.T) { t.Parallel() - fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key") + fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil) fs.SetMasterKey("dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk") ts := testServer(t, fs) defer ts.Close() @@ -237,7 +237,7 @@ func TestStatusEndpointMasterKey(t *testing.T) { func TestHealthEndpoint(t *testing.T) { t.Parallel() - fs := NewFormationServer(1, testCreds(), "", "") + fs := NewFormationServer(1, testCreds(), "", "", nil) ts := testServer(t, fs) defer ts.Close() @@ -292,7 +292,7 @@ func TestBuildPredastoreNodes(t *testing.T) { func TestFullFormationFlow(t *testing.T) { t.Parallel() creds := testCreds() - fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----") + fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil) ts := testServer(t, fs) defer ts.Close()