feat: auto-detect network topology + DHCP gateway in admin init

spx admin init now works with zero external networking flags: - Auto-detects WAN interface, gateway, prefix from ip route - Auto-detects single-NIC vs multi-NIC mode - Defaults to nat mode with DHCP for gateway IP - Prints detected topology with interface roles setup-ovn.sh gains --dhcp flag: obtains a DHCP lease on the macvlan/bridge interface for the OVN gateway SNAT address, then writes gateway_ip into spinifex.toml. Tested on VM: macvlan gets separate IP from router DHCP, host keeps its original IP. Three tiers of external networking: - Zero flags: auto nat+DHCP (VMs get outbound internet) - --external-pool=start-end: pool mode (VMs get public IPs) - --no-external: overlay only (no internet for VMs)
2026-10-15 17:11:30 +00:00 · 2026-03-20 05:45:30 +00:00
parent 1f38213d4b
commit 2af3bf5287
6 changed files with 367 additions and 23 deletions
@@ -216,12 +216,13 @@ func init() {
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")

 	// External networking flags
-	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (public IPs), 'nat' (outbound-only), or '' (disabled)")
-	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (e.g., eth0, eth1)")
-	adminInitCmd.Flags().Bool("single-nic", false, "Use macvlan for external bridge (single-NIC hosts, SSH-safe)")
+	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (public IPs), 'nat' (outbound-only, default), or '' (disabled)")
+	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
+	adminInitCmd.Flags().Bool("single-nic", false, "Use macvlan for external bridge (auto-detected if only one NIC)")
 	adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)")
-	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (e.g., 192.168.1.1)")
-	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length")
+	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
+	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
+	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")

 	// Flags for admin join
 	adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
@@ -543,24 +544,78 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	externalPool, _ := cmd.Flags().GetString("external-pool")
 	externalGateway, _ := cmd.Flags().GetString("external-gateway")
 	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
+	noExternal, _ := cmd.Flags().GetBool("no-external")
+
+	// Auto-detect network topology
+	var poolStart, poolEnd string
+	var detectedNet *admin.DetectedNetwork
+	if !noExternal {
+		detected, err := admin.DetectNetwork()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
+			fmt.Fprintf(os.Stderr, "   Use --no-external to skip, or specify flags manually.\n")
+		} else {
+			detectedNet = detected
+
+			// Print detected topology
+			fmt.Println("\n🔍 Detected network topology:\n")
+			fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", "Interface", "IP", "Subnet", "Gateway", "Role")
+			for _, iface := range detected.Interfaces {
+				gw := "—"
+				if iface.Gateway != "" {
+					gw = iface.Gateway
+				}
+				fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role))
+			}
+			if detected.SingleNIC {
+				fmt.Println("\n  Mode: single-NIC (will use macvlan for external bridge, SSH-safe)")
+			} else {
+				fmt.Printf("\n  Mode: multi-NIC (%d LAN + 1 WAN)\n", detected.LANCount)
+			}
+
+			// Apply auto-detected values when flags not explicitly set
+			if detected.WAN != nil {
+				if externalIface == "" {
+					externalIface = detected.WAN.Name
+				}
+				if externalGateway == "" {
+					externalGateway = detected.WAN.Gateway
+				}
+				if !cmd.Flags().Changed("external-prefix-len") {
+					externalPrefixLen = detected.WAN.PrefixLen
+				}
+				if !cmd.Flags().Changed("single-nic") {
+					singleNIC = detected.SingleNIC
+				}
+
+				// Default mode: "nat" with DHCP if no pool specified
+				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+					if externalPool != "" {
+						externalMode = "pool"
+					} else {
+						externalMode = "nat"
+					}
+				}
+			}
+		}
+	}

 	// Validate external networking flags
 	if externalMode != "" && externalMode != "pool" && externalMode != "nat" {
 		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode)
 		os.Exit(1)
 	}
-	var poolStart, poolEnd string
 	if externalMode == "pool" {
 		if externalPool == "" {
 			fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n")
+			if detectedNet != nil && detectedNet.WAN != nil {
+				sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
+				fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
+			}
 			os.Exit(1)
 		}
 		if externalGateway == "" {
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool (e.g., 192.168.1.1)\n")
-			os.Exit(1)
-		}
-		if externalIface == "" {
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-iface is required when --external-mode=pool (e.g., eth0)\n")
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n")
 			os.Exit(1)
 		}
 		parts := strings.SplitN(externalPool, "-", 2)
@@ -569,11 +624,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 		poolStart, poolEnd = parts[0], parts[1]
-		if net.ParseIP(externalGateway) == nil {
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
-			os.Exit(1)
-		}
 	}
+	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
+		os.Exit(1)
+	}
+
+	// For nat mode, we need the gateway IP. The DHCP flag tells setup-ovn.sh
+	// to obtain one via DHCP on the macvlan/bridge interface.
+	useExternalDHCP := externalMode == "nat" && poolStart == ""

 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
@@ -768,6 +827,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ExternalMode:  externalMode,
 		ExternalIface: externalIface,
 		SingleNIC:     singleNIC,
+		ExternalDHCP:  useExternalDHCP,
 		PoolName:      "wan",
 		PoolStart:     poolStart,
 		PoolEnd:       poolEnd,
@@ -775,6 +835,24 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		PoolPrefixLen: externalPrefixLen,
 	}

+	// Print external networking summary
+	if externalMode != "" {
+		fmt.Printf("\n📡 External networking: %s\n", externalMode)
+		fmt.Printf("  WAN interface: %s\n", externalIface)
+		if singleNIC {
+			fmt.Println("  Bridge mode:   macvlan (single-NIC, SSH-safe)")
+		} else {
+			fmt.Println("  Bridge mode:   dedicated NIC")
+		}
+		if externalMode == "pool" {
+			fmt.Printf("  IP pool:       %s - %s\n", poolStart, poolEnd)
+			fmt.Printf("  ⚠️  Ensure %s-%s is excluded from your router's DHCP range.\n", poolStart, poolEnd)
+		} else if useExternalDHCP {
+			fmt.Println("  Gateway IP:    DHCP (obtained during OVN setup)")
+			fmt.Println("  VMs get:       outbound internet via SNAT")
+		}
+	}
+
 	// Generate config files
 	configs := []admin.ConfigFile{
 		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
@@ -822,6 +900,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		if singleNIC {
 			ovnFlags += " --single-nic"
 		}
+		if useExternalDHCP {
+			ovnFlags += " --dhcp"
+		}
 		fmt.Printf("   %d. Setup OVN with external bridge:\n", step)
 		fmt.Printf("      ./scripts/setup-ovn.sh %s\n", ovnFlags)
 		step++
@@ -6,12 +6,20 @@ node = "{{.Node}}"

 [network]
 external_mode = "{{.ExternalMode}}"
+{{- if .ExternalDHCP}}
+external_dhcp = true
+{{- end}}

 [[network.external_pools]]
 name        = "{{.PoolName}}"
+{{- if .PoolStart}}
 range_start = "{{.PoolStart}}"
 range_end   = "{{.PoolEnd}}"
+{{- end}}
 gateway     = "{{.PoolGateway}}"
+{{- if .PoolGatewayIP}}
+gateway_ip  = "{{.PoolGatewayIP}}"
+{{- end}}
 prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
 {{- end}}

@@ -17,6 +17,7 @@
 #   --external-bridge  Create br-external for public subnet WAN uplink
 #   --external-iface   WAN NIC to add to br-external (default: eth1)
 #   --single-nic       Use macvlan instead of adding NIC directly (for single-NIC hosts)
+#   --dhcp             Obtain gateway IP via DHCP on the external bridge interface
 #   --ovn-remote       OVN SB DB address (default: tcp:127.0.0.1:6642)
 #   --encap-ip         Geneve tunnel endpoint IP (default: auto-detect)
 #   --chassis-id       OVN chassis identifier (default: hostname)
@@ -45,6 +46,7 @@ set -e
 MANAGEMENT=false
 EXTERNAL_BRIDGE=false
 SINGLE_NIC=false
+EXTERNAL_DHCP=false
 EXTERNAL_IFACE="eth1"
 OVN_REMOTE="tcp:127.0.0.1:6642"
 ENCAP_IP=""
@@ -56,6 +58,7 @@ for arg in "$@"; do
        --management)       MANAGEMENT=true ;;
        --external-bridge)  EXTERNAL_BRIDGE=true ;;
        --single-nic)       SINGLE_NIC=true ;;
+        --dhcp)             EXTERNAL_DHCP=true ;;
        --external-iface=*) EXTERNAL_IFACE="${arg#*=}" ;;
        --ovn-remote=*)     OVN_REMOTE="${arg#*=}" ;;
        --encap-ip=*)       ENCAP_IP="${arg#*=}" ;;
@@ -212,6 +215,54 @@ if [ "$EXTERNAL_BRIDGE" = true ]; then

        echo "  br-external: created with port $EXTERNAL_IFACE"
    fi
+
+    # --- DHCP: obtain gateway IP for OVN SNAT ---
+    if [ "$EXTERNAL_DHCP" = true ]; then
+        echo ""
+        echo "Step 3c: Obtaining external gateway IP via DHCP..."
+
+        # Determine which interface to DHCP on
+        if [ "$SINGLE_NIC" = true ]; then
+            DHCP_IFACE="spx-ext-${EXTERNAL_IFACE}"
+        else
+            DHCP_IFACE="br-external"
+        fi
+
+        # Run DHCP client to get a lease
+        if command -v dhcpcd >/dev/null 2>&1; then
+            sudo dhcpcd --waitip=4 --timeout 15 "$DHCP_IFACE" 2>/dev/null || true
+        elif command -v dhclient >/dev/null 2>&1; then
+            sudo dhclient -1 -timeout 15 "$DHCP_IFACE" 2>/dev/null || true
+        else
+            echo "  WARNING: no DHCP client found (dhcpcd or dhclient)"
+            echo "  Install dhcpcd-base or isc-dhcp-client, or set gateway_ip manually"
+        fi
+
+        # Read the obtained IP
+        DHCP_IP=$(ip -4 addr show dev "$DHCP_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1 | cut -d/ -f1)
+        if [ -n "$DHCP_IP" ]; then
+            echo "  DHCP obtained: $DHCP_IP on $DHCP_IFACE"
+
+            # Write the gateway IP to the spinifex config so vpcd can use it
+            CONFIG_DIR="${CONFIG_DIR:-$HOME/spinifex/config}"
+            CONFIG_FILE="$CONFIG_DIR/spinifex.toml"
+            if [ -f "$CONFIG_FILE" ]; then
+                # Update gateway_ip in the config
+                if grep -q "gateway_ip" "$CONFIG_FILE"; then
+                    sed -i "s/gateway_ip.*/gateway_ip = \"$DHCP_IP\"/" "$CONFIG_FILE"
+                else
+                    # Insert gateway_ip after the gateway line in the pool section
+                    sed -i "/^gateway *=.*/a gateway_ip  = \"$DHCP_IP\"" "$CONFIG_FILE"
+                fi
+                echo "  Updated $CONFIG_FILE with gateway_ip = $DHCP_IP"
+            else
+                echo "  WARNING: $CONFIG_FILE not found — set gateway_ip manually"
+            fi
+        else
+            echo "  WARNING: DHCP failed to obtain IP on $DHCP_IFACE"
+            echo "  VMs will not have external connectivity until gateway_ip is configured"
+        fi
+    fi
 fi

 # --- Step 4: Configure OVN external_ids ---
@@ -65,14 +65,16 @@ type ConfigSettings struct {
 	OVNSBAddr string

 	// External networking for public subnets
-	ExternalMode  string // "pool", "nat", or "" (disabled)
-	ExternalIface string // WAN NIC name (e.g., "eth0", "eth1")
-	SingleNIC     bool   // Use macvlan instead of adding NIC directly
-	PoolName      string // External pool name (e.g., "wan")
-	PoolStart     string // First IP in external pool range
-	PoolEnd       string // Last IP in external pool range
-	PoolGateway   string // WAN gateway IP
-	PoolPrefixLen int    // Subnet prefix length (default 24)
+	ExternalMode    string // "pool", "nat", or "" (disabled)
+	ExternalIface   string // WAN NIC name (e.g., "eth0", "eth1")
+	SingleNIC       bool   // Use macvlan instead of adding NIC directly
+	ExternalDHCP    bool   // Obtain gateway IP via DHCP on macvlan/bridge
+	PoolName        string // External pool name (e.g., "wan")
+	PoolStart       string // First IP in external pool range
+	PoolEnd         string // Last IP in external pool range
+	PoolGateway     string // WAN gateway IP
+	PoolGatewayIP   string // Explicit SNAT IP (for nat mode without DHCP)
+	PoolPrefixLen   int    // Subnet prefix length (default 24)

 	// Other nodes in the cluster (for config source of truth)
 	RemoteNodes []RemoteNode
@@ -0,0 +1,201 @@
+package admin
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+)
+
+// DetectedInterface represents a network interface discovered on the host.
+type DetectedInterface struct {
+	Name      string // e.g., "enp0s3"
+	IP        string // e.g., "192.168.1.21"
+	Subnet    string // e.g., "192.168.0.0/23"
+	PrefixLen int    // e.g., 23
+	Gateway   string // non-empty only for WAN (default route)
+	Role      string // "wan", "lan"
+}
+
+// DetectedNetwork holds the auto-detected network topology.
+type DetectedNetwork struct {
+	Interfaces []DetectedInterface
+	WAN        *DetectedInterface // The interface with the default route
+	LANCount   int                // Number of non-WAN interfaces
+	SingleNIC  bool               // Only one usable NIC
+}
+
+// DetectNetwork auto-detects the host's network topology from routing table.
+// It identifies the WAN interface (default route), LAN interfaces, and
+// determines single-NIC vs multi-NIC mode.
+func DetectNetwork() (*DetectedNetwork, error) {
+	// Get default route: "default via 192.168.1.1 dev enp0s3 ..."
+	out, err := exec.Command("ip", "-4", "route", "show", "default").Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get default route: %w", err)
+	}
+	defaultRoute := strings.TrimSpace(string(out))
+	if defaultRoute == "" {
+		return nil, fmt.Errorf("no default route found — host has no internet connectivity")
+	}
+
+	// Parse: "default via 192.168.1.1 dev enp0s3 ..."
+	var wanGateway, wanIface string
+	fields := strings.Fields(defaultRoute)
+	for i, f := range fields {
+		if f == "via" && i+1 < len(fields) {
+			wanGateway = fields[i+1]
+		}
+		if f == "dev" && i+1 < len(fields) {
+			wanIface = fields[i+1]
+		}
+	}
+	if wanIface == "" || wanGateway == "" {
+		return nil, fmt.Errorf("could not parse default route: %s", defaultRoute)
+	}
+
+	// Get all routes to find interfaces and their subnets
+	allRoutes, err := exec.Command("ip", "-4", "route", "show", "proto", "kernel", "scope", "link").Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get routes: %w", err)
+	}
+
+	// Parse routes: "192.168.0.0/23 dev enp0s3 ... src 192.168.1.21"
+	seen := make(map[string]bool)
+	var interfaces []DetectedInterface
+
+	for _, line := range strings.Split(strings.TrimSpace(string(allRoutes)), "\n") {
+		if line == "" {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			continue
+		}
+
+		subnet := fields[0]
+		var ifaceName, srcIP string
+		for i, f := range fields {
+			if f == "dev" && i+1 < len(fields) {
+				ifaceName = fields[i+1]
+			}
+			if f == "src" && i+1 < len(fields) {
+				srcIP = fields[i+1]
+			}
+		}
+		if ifaceName == "" || srcIP == "" {
+			continue
+		}
+
+		// Skip virtual/docker/ovs interfaces
+		if isVirtualInterface(ifaceName) {
+			continue
+		}
+
+		// Skip duplicates (same interface may have multiple routes)
+		if seen[ifaceName] {
+			continue
+		}
+		seen[ifaceName] = true
+
+		prefixLen := 24 // default
+		if _, ipNet, err := net.ParseCIDR(subnet); err == nil {
+			ones, _ := ipNet.Mask.Size()
+			prefixLen = ones
+		}
+
+		iface := DetectedInterface{
+			Name:      ifaceName,
+			IP:        srcIP,
+			Subnet:    subnet,
+			PrefixLen: prefixLen,
+		}
+
+		if ifaceName == wanIface {
+			iface.Gateway = wanGateway
+			iface.Role = "wan"
+		} else {
+			iface.Role = "lan"
+		}
+
+		interfaces = append(interfaces, iface)
+	}
+
+	if len(interfaces) == 0 {
+		return nil, fmt.Errorf("no usable network interfaces found")
+	}
+
+	result := &DetectedNetwork{
+		Interfaces: interfaces,
+	}
+
+	for i := range interfaces {
+		if interfaces[i].Role == "wan" {
+			result.WAN = &interfaces[i]
+		} else {
+			result.LANCount++
+		}
+	}
+
+	// If only the WAN interface exists (no LANs), it's single-NIC
+	result.SingleNIC = result.LANCount == 0
+
+	return result, nil
+}
+
+// SuggestPoolRange suggests a default external IP pool range based on the
+// WAN subnet. Places the pool at the high end of the subnet to avoid
+// collisions with typical DHCP ranges (which start low).
+func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
+	_, ipNet, err := net.ParseCIDR(wan.Subnet)
+	if err != nil {
+		return "", ""
+	}
+
+	// Calculate broadcast address
+	ip := ipNet.IP.To4()
+	mask := ipNet.Mask
+	broadcast := make(net.IP, 4)
+	for i := range ip {
+		broadcast[i] = ip[i] | ^mask[i]
+	}
+
+	// Pool: broadcast - 100 to broadcast - 5
+	// e.g., for 192.168.0.0/23 (broadcast 192.168.1.255):
+	//   start = 192.168.1.155, end = 192.168.1.250
+	endIP := make(net.IP, 4)
+	copy(endIP, broadcast)
+	endIP[3] -= 5 // Leave some room at top
+
+	startIP := make(net.IP, 4)
+	copy(startIP, endIP)
+	// Go back 100 IPs
+	carry := 100
+	for i := 3; i >= 0 && carry > 0; i-- {
+		val := int(startIP[i]) - carry
+		if val < 0 {
+			startIP[i] = byte(256 + val)
+			carry = 1
+		} else {
+			startIP[i] = byte(val)
+			carry = 0
+		}
+	}
+
+	return startIP.String(), endIP.String()
+}
+
+// isVirtualInterface returns true for interfaces that shouldn't be
+// considered as physical NICs.
+func isVirtualInterface(name string) bool {
+	prefixes := []string{
+		"docker", "br-", "veth", "ovs", "virbr",
+		"tun", "tap", "tailscale", "wg", "lo",
+	}
+	for _, p := range prefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}
@@ -32,6 +32,7 @@ type ExternalPool struct {
 // NetworkConfig holds cluster-wide external network settings.
 type NetworkConfig struct {
 	ExternalMode  string         `mapstructure:"external_mode"`  // "pool", "nat", or "" (disabled)
+	ExternalDHCP  bool           `mapstructure:"external_dhcp"`  // Gateway IP obtained via DHCP (nat mode)
 	ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools
 }