mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 03:11:46 +00:00
feat: propagate network config via formation server to joining nodes (mulga-759)
admin init builds a NetworkConfig from external networking flags and passes it to the formation server. When joining nodes poll /formation/status, they receive the cluster-wide [network] section (external mode, pools, gateway, DNS, bootstrap VPC IDs). Each joining node auto-detects its own external_interface and wan_bridge locally. Fixes multi-node clusters having no [network] section in spinifex.toml on secondary nodes, which caused no public IP assignment and broke guest SSH in E2E tests.
This commit is contained in:
Ben Duncan
@@ -765,9 +765,35 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
isMultiNode := nodes >= 2 && bindIP != "0.0.0.0"
|
||||
|
||||
if isMultiNode {
|
||||
// Build cluster-wide network config for propagation to joining nodes
|
||||
var networkConfig *formation.NetworkConfig
|
||||
if externalMode != "" {
|
||||
bootstrapVpcId := utils.GenerateResourceID("vpc")
|
||||
bootstrapSubnetId := utils.GenerateResourceID("subnet")
|
||||
bootstrapIgwId := utils.GenerateResourceID("igw")
|
||||
networkConfig = &formation.NetworkConfig{
|
||||
ExternalMode: externalMode,
|
||||
ExternalDHCP: useExternalDHCP,
|
||||
PoolName: "wan",
|
||||
PoolSource: externalSource,
|
||||
PoolStart: poolStart,
|
||||
PoolEnd: poolEnd,
|
||||
PoolGateway: externalGateway,
|
||||
PoolGatewayIP: gatewayIP,
|
||||
PoolPrefixLen: externalPrefixLen,
|
||||
PoolDNSServers: dnsServers,
|
||||
BootstrapAccountId: admin.DefaultAccountID(),
|
||||
BootstrapVpcId: bootstrapVpcId,
|
||||
BootstrapSubnetId: bootstrapSubnetId,
|
||||
BootstrapIgwId: bootstrapIgwId,
|
||||
BootstrapCidr: handlers_ec2_vpc.DefaultVPCCidr,
|
||||
BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
|
||||
}
|
||||
}
|
||||
|
||||
runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
|
||||
configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind,
|
||||
port, nodes, formationTimeoutStr, services)
|
||||
port, nodes, formationTimeoutStr, services, networkConfig)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -954,7 +980,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
// then generates configs with complete cluster topology.
|
||||
func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
|
||||
configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string,
|
||||
port, expectedNodes int, formationTimeoutStr string, services []string) {
|
||||
port, expectedNodes int, formationTimeoutStr string, services []string, networkConfig *formation.NetworkConfig) {
|
||||
formationTimeout, err := time.ParseDuration(formationTimeoutStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
|
||||
@@ -998,7 +1024,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
|
||||
AdminSecretKey: bootstrapResult.AdminSecretKey,
|
||||
}
|
||||
|
||||
fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData))
|
||||
fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig)
|
||||
|
||||
// Include master key in formation server for distribution to joining nodes
|
||||
fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey))
|
||||
@@ -1106,6 +1132,34 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
|
||||
OVNSBAddr: "tcp:127.0.0.1:6642",
|
||||
}
|
||||
|
||||
// Apply cluster-wide network config to init node's config
|
||||
if networkConfig != nil {
|
||||
configSettings.ExternalMode = networkConfig.ExternalMode
|
||||
configSettings.ExternalDHCP = networkConfig.ExternalDHCP
|
||||
configSettings.PoolName = networkConfig.PoolName
|
||||
configSettings.PoolSource = networkConfig.PoolSource
|
||||
configSettings.PoolStart = networkConfig.PoolStart
|
||||
configSettings.PoolEnd = networkConfig.PoolEnd
|
||||
configSettings.PoolGateway = networkConfig.PoolGateway
|
||||
configSettings.PoolGatewayIP = networkConfig.PoolGatewayIP
|
||||
configSettings.PoolPrefixLen = networkConfig.PoolPrefixLen
|
||||
configSettings.PoolDNSServers = networkConfig.PoolDNSServers
|
||||
|
||||
configSettings.BootstrapAccountId = networkConfig.BootstrapAccountId
|
||||
configSettings.BootstrapVpcId = networkConfig.BootstrapVpcId
|
||||
configSettings.BootstrapSubnetId = networkConfig.BootstrapSubnetId
|
||||
configSettings.BootstrapIgwId = networkConfig.BootstrapIgwId
|
||||
configSettings.BootstrapCidr = networkConfig.BootstrapCidr
|
||||
configSettings.BootstrapSubnetCidr = networkConfig.BootstrapSubnetCidr
|
||||
|
||||
// Auto-detect local network topology for per-node config
|
||||
detected, err := admin.DetectNetwork()
|
||||
if err == nil && detected.WAN != nil {
|
||||
configSettings.ExternalIface = detected.WAN.Name
|
||||
configSettings.WanBridge = detectedWanBridge(detected)
|
||||
}
|
||||
}
|
||||
|
||||
// Generate config files
|
||||
configs := []admin.ConfigFile{
|
||||
{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
|
||||
@@ -1437,6 +1491,37 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
|
||||
}
|
||||
|
||||
// Apply cluster-wide network config from leader
|
||||
if statusResp.NetworkConfig != nil {
|
||||
nc := statusResp.NetworkConfig
|
||||
configSettings.ExternalMode = nc.ExternalMode
|
||||
configSettings.ExternalDHCP = nc.ExternalDHCP
|
||||
configSettings.PoolName = nc.PoolName
|
||||
configSettings.PoolSource = nc.PoolSource
|
||||
configSettings.PoolStart = nc.PoolStart
|
||||
configSettings.PoolEnd = nc.PoolEnd
|
||||
configSettings.PoolGateway = nc.PoolGateway
|
||||
configSettings.PoolGatewayIP = nc.PoolGatewayIP
|
||||
configSettings.PoolPrefixLen = nc.PoolPrefixLen
|
||||
configSettings.PoolDNSServers = nc.PoolDNSServers
|
||||
|
||||
configSettings.BootstrapAccountId = nc.BootstrapAccountId
|
||||
configSettings.BootstrapVpcId = nc.BootstrapVpcId
|
||||
configSettings.BootstrapSubnetId = nc.BootstrapSubnetId
|
||||
configSettings.BootstrapIgwId = nc.BootstrapIgwId
|
||||
configSettings.BootstrapCidr = nc.BootstrapCidr
|
||||
configSettings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr
|
||||
|
||||
// Auto-detect local network topology for per-node config
|
||||
if nc.ExternalMode != "" {
|
||||
detected, err := admin.DetectNetwork()
|
||||
if err == nil && detected.WAN != nil {
|
||||
configSettings.ExternalIface = detected.WAN.Name
|
||||
configSettings.WanBridge = detectedWanBridge(detected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate config files
|
||||
configs := []admin.ConfigFile{
|
||||
{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
|
||||
|
||||
@@ -38,14 +38,37 @@ type JoinResponse struct {
|
||||
|
||||
// StatusResponse is returned by the formation server status endpoint.
|
||||
type StatusResponse struct {
|
||||
Complete bool `json:"complete"`
|
||||
Joined int `json:"joined"`
|
||||
Expected int `json:"expected"`
|
||||
Nodes map[string]NodeInfo `json:"nodes,omitempty"`
|
||||
Credentials *SharedCredentials `json:"credentials,omitempty"`
|
||||
CACert string `json:"ca_cert,omitempty"`
|
||||
CAKey string `json:"ca_key,omitempty"`
|
||||
MasterKey string `json:"master_key,omitempty"`
|
||||
Complete bool `json:"complete"`
|
||||
Joined int `json:"joined"`
|
||||
Expected int `json:"expected"`
|
||||
Nodes map[string]NodeInfo `json:"nodes,omitempty"`
|
||||
Credentials *SharedCredentials `json:"credentials,omitempty"`
|
||||
CACert string `json:"ca_cert,omitempty"`
|
||||
CAKey string `json:"ca_key,omitempty"`
|
||||
MasterKey string `json:"master_key,omitempty"`
|
||||
NetworkConfig *NetworkConfig `json:"network_config,omitempty"`
|
||||
}
|
||||
|
||||
// NetworkConfig holds the cluster-wide external networking configuration
|
||||
// propagated from the init node to joining nodes during formation.
|
||||
type NetworkConfig struct {
|
||||
ExternalMode string `json:"external_mode"`
|
||||
ExternalDHCP bool `json:"external_dhcp,omitempty"`
|
||||
PoolName string `json:"pool_name"`
|
||||
PoolSource string `json:"pool_source,omitempty"`
|
||||
PoolStart string `json:"pool_start,omitempty"`
|
||||
PoolEnd string `json:"pool_end,omitempty"`
|
||||
PoolGateway string `json:"pool_gateway"`
|
||||
PoolGatewayIP string `json:"pool_gateway_ip,omitempty"`
|
||||
PoolPrefixLen int `json:"pool_prefix_len"`
|
||||
PoolDNSServers []string `json:"pool_dns_servers,omitempty"`
|
||||
|
||||
BootstrapAccountId string `json:"bootstrap_account_id,omitempty"`
|
||||
BootstrapVpcId string `json:"bootstrap_vpc_id,omitempty"`
|
||||
BootstrapSubnetId string `json:"bootstrap_subnet_id,omitempty"`
|
||||
BootstrapIgwId string `json:"bootstrap_igw_id,omitempty"`
|
||||
BootstrapCidr string `json:"bootstrap_cidr,omitempty"`
|
||||
BootstrapSubnetCidr string `json:"bootstrap_subnet_cidr,omitempty"`
|
||||
}
|
||||
|
||||
// SharedCredentials contains the cluster-wide credentials distributed during formation.
|
||||
@@ -67,26 +90,28 @@ type SharedCredentials struct {
|
||||
// nodes have joined, the done channel is closed and full cluster data (credentials,
|
||||
// CA, node list) becomes available via GET /formation/status.
|
||||
type FormationServer struct {
|
||||
mu sync.RWMutex
|
||||
expected int
|
||||
nodes map[string]NodeInfo
|
||||
credentials *SharedCredentials
|
||||
caCert string
|
||||
caKey string
|
||||
masterKey string
|
||||
done chan struct{}
|
||||
server *http.Server
|
||||
mu sync.RWMutex
|
||||
expected int
|
||||
nodes map[string]NodeInfo
|
||||
credentials *SharedCredentials
|
||||
caCert string
|
||||
caKey string
|
||||
masterKey string
|
||||
networkConfig *NetworkConfig
|
||||
done chan struct{}
|
||||
server *http.Server
|
||||
}
|
||||
|
||||
// NewFormationServer creates a new formation server expecting the given number of nodes.
|
||||
func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string) *FormationServer {
|
||||
func NewFormationServer(expected int, creds *SharedCredentials, caCert, caKey string, networkConfig *NetworkConfig) *FormationServer {
|
||||
return &FormationServer{
|
||||
expected: expected,
|
||||
nodes: make(map[string]NodeInfo),
|
||||
credentials: creds,
|
||||
caCert: caCert,
|
||||
caKey: caKey,
|
||||
done: make(chan struct{}),
|
||||
expected: expected,
|
||||
nodes: make(map[string]NodeInfo),
|
||||
credentials: creds,
|
||||
caCert: caCert,
|
||||
caKey: caKey,
|
||||
networkConfig: networkConfig,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +277,7 @@ func (fs *FormationServer) handleStatus(w http.ResponseWriter, r *http.Request)
|
||||
resp.CACert = fs.caCert
|
||||
resp.CAKey = fs.caKey
|
||||
resp.MasterKey = fs.masterKey
|
||||
resp.NetworkConfig = fs.networkConfig
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
|
||||
@@ -35,7 +35,7 @@ func testNode(name, ip string) NodeInfo {
|
||||
|
||||
func TestNewFormationServer(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem")
|
||||
fs := NewFormationServer(3, testCreds(), "ca-cert-pem", "ca-key-pem", nil)
|
||||
|
||||
assert.Equal(t, 3, fs.expected)
|
||||
assert.Empty(t, fs.nodes)
|
||||
@@ -44,7 +44,7 @@ func TestNewFormationServer(t *testing.T) {
|
||||
|
||||
func TestRegisterNode(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
|
||||
err := fs.RegisterNode(testNode("node1", "10.0.0.1"))
|
||||
require.NoError(t, err)
|
||||
@@ -56,7 +56,7 @@ func TestRegisterNode(t *testing.T) {
|
||||
|
||||
func TestRegisterDuplicateName(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
|
||||
require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
|
||||
err := fs.RegisterNode(testNode("node1", "10.0.0.2"))
|
||||
@@ -66,7 +66,7 @@ func TestRegisterDuplicateName(t *testing.T) {
|
||||
|
||||
func TestRegisterDuplicateIP(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
|
||||
require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
|
||||
err := fs.RegisterNode(testNode("node2", "10.0.0.1"))
|
||||
@@ -76,7 +76,7 @@ func TestRegisterDuplicateIP(t *testing.T) {
|
||||
|
||||
func TestIsComplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(2, testCreds(), "", "")
|
||||
fs := NewFormationServer(2, testCreds(), "", "", nil)
|
||||
|
||||
assert.False(t, fs.IsComplete())
|
||||
require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
|
||||
@@ -87,7 +87,7 @@ func TestIsComplete(t *testing.T) {
|
||||
|
||||
func TestWaitForCompletion(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(2, testCreds(), "", "")
|
||||
fs := NewFormationServer(2, testCreds(), "", "", nil)
|
||||
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
@@ -102,7 +102,7 @@ func TestWaitForCompletion(t *testing.T) {
|
||||
|
||||
func TestWaitForCompletionTimeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
|
||||
require.NoError(t, fs.RegisterNode(testNode("node1", "10.0.0.1")))
|
||||
|
||||
@@ -122,7 +122,7 @@ func testServer(t *testing.T, fs *FormationServer) *httptest.Server {
|
||||
|
||||
func TestJoinEndpoint(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
@@ -142,7 +142,7 @@ func TestJoinEndpoint(t *testing.T) {
|
||||
|
||||
func TestJoinEndpointDuplicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "", "")
|
||||
fs := NewFormationServer(3, testCreds(), "", "", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
@@ -161,7 +161,7 @@ func TestJoinEndpointDuplicate(t *testing.T) {
|
||||
|
||||
func TestStatusEndpointIncomplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key")
|
||||
fs := NewFormationServer(3, testCreds(), "ca-cert", "ca-key", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
@@ -184,7 +184,7 @@ func TestStatusEndpointIncomplete(t *testing.T) {
|
||||
func TestStatusEndpointComplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
creds := testCreds()
|
||||
fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data")
|
||||
fs := NewFormationServer(2, creds, "ca-cert-data", "ca-key-data", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
@@ -209,7 +209,7 @@ func TestStatusEndpointComplete(t *testing.T) {
|
||||
|
||||
func TestStatusEndpointMasterKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key")
|
||||
fs := NewFormationServer(2, testCreds(), "ca-cert", "ca-key", nil)
|
||||
fs.SetMasterKey("dGVzdC1tYXN0ZXIta2V5LWJhc2U2NC1lbmNvZGVk")
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
@@ -237,7 +237,7 @@ func TestStatusEndpointMasterKey(t *testing.T) {
|
||||
|
||||
func TestHealthEndpoint(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs := NewFormationServer(1, testCreds(), "", "")
|
||||
fs := NewFormationServer(1, testCreds(), "", "", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
@@ -292,7 +292,7 @@ func TestBuildPredastoreNodes(t *testing.T) {
|
||||
func TestFullFormationFlow(t *testing.T) {
|
||||
t.Parallel()
|
||||
creds := testCreds()
|
||||
fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----")
|
||||
fs := NewFormationServer(3, creds, "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----", "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", nil)
|
||||
ts := testServer(t, fs)
|
||||
defer ts.Close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user