diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 2e27a9c50..1d585791b 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -216,12 +216,13 @@ func init() { adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui") // External networking flags - adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (public IPs), 'nat' (outbound-only), or '' (disabled)") - adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (e.g., eth0, eth1)") - adminInitCmd.Flags().Bool("single-nic", false, "Use macvlan for external bridge (single-NIC hosts, SSH-safe)") + adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (public IPs), 'nat' (outbound-only, default), or '' (disabled)") + adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)") + adminInitCmd.Flags().Bool("single-nic", false, "Use macvlan for external bridge (auto-detected if only one NIC)") adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)") - adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (e.g., 192.168.1.1)") - adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length") + adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)") + adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)") + adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)") // Flags for admin join adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node") @@ -543,24 +544,78 @@ func runAdminInit(cmd *cobra.Command, args []string) { externalPool, _ := cmd.Flags().GetString("external-pool") externalGateway, _ := cmd.Flags().GetString("external-gateway") externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len") + noExternal, _ := cmd.Flags().GetBool("no-external") + + // Auto-detect network topology + var poolStart, poolEnd string + var detectedNet *admin.DetectedNetwork + if !noExternal { + detected, err := admin.DetectNetwork() + if err != nil { + fmt.Fprintf(os.Stderr, "āš ļø Network auto-detection failed: %v\n", err) + fmt.Fprintf(os.Stderr, " Use --no-external to skip, or specify flags manually.\n") + } else { + detectedNet = detected + + // Print detected topology + fmt.Println("\nšŸ” Detected network topology:\n") + fmt.Printf(" %-14s %-18s %-20s %-16s %s\n", "Interface", "IP", "Subnet", "Gateway", "Role") + for _, iface := range detected.Interfaces { + gw := "ā€”" + if iface.Gateway != "" { + gw = iface.Gateway + } + fmt.Printf(" %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role)) + } + if detected.SingleNIC { + fmt.Println("\n Mode: single-NIC (will use macvlan for external bridge, SSH-safe)") + } else { + fmt.Printf("\n Mode: multi-NIC (%d LAN + 1 WAN)\n", detected.LANCount) + } + + // Apply auto-detected values when flags not explicitly set + if detected.WAN != nil { + if externalIface == "" { + externalIface = detected.WAN.Name + } + if externalGateway == "" { + externalGateway = detected.WAN.Gateway + } + if !cmd.Flags().Changed("external-prefix-len") { + externalPrefixLen = detected.WAN.PrefixLen + } + if !cmd.Flags().Changed("single-nic") { + singleNIC = detected.SingleNIC + } + + // Default mode: "nat" with DHCP if no pool specified + if externalMode == "" && !cmd.Flags().Changed("external-mode") { + if externalPool != "" { + externalMode = "pool" + } else { + externalMode = "nat" + } + } + } + } + } // Validate external networking flags if externalMode != "" && externalMode != "pool" && externalMode != "nat" { fmt.Fprintf(os.Stderr, "āŒ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode) os.Exit(1) } - var poolStart, poolEnd string if externalMode == "pool" { if externalPool == "" { fmt.Fprintf(os.Stderr, "āŒ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n") + if detectedNet != nil && detectedNet.WAN != nil { + sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN) + fmt.Fprintf(os.Stderr, " Suggested: --external-pool=%s-%s\n", sugStart, sugEnd) + } os.Exit(1) } if externalGateway == "" { - fmt.Fprintf(os.Stderr, "āŒ Error: --external-gateway is required when --external-mode=pool (e.g., 192.168.1.1)\n") - os.Exit(1) - } - if externalIface == "" { - fmt.Fprintf(os.Stderr, "āŒ Error: --external-iface is required when --external-mode=pool (e.g., eth0)\n") + fmt.Fprintf(os.Stderr, "āŒ Error: --external-gateway is required when --external-mode=pool\n") os.Exit(1) } parts := strings.SplitN(externalPool, "-", 2) @@ -569,11 +624,15 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } poolStart, poolEnd = parts[0], parts[1] - if net.ParseIP(externalGateway) == nil { - fmt.Fprintf(os.Stderr, "āŒ Error: --external-gateway is not a valid IP: %s\n", externalGateway) - os.Exit(1) - } } + if externalGateway != "" && net.ParseIP(externalGateway) == nil { + fmt.Fprintf(os.Stderr, "āŒ Error: --external-gateway is not a valid IP: %s\n", externalGateway) + os.Exit(1) + } + + // For nat mode, we need the gateway IP. The DHCP flag tells setup-ovn.sh + // to obtain one via DHCP on the macvlan/bridge interface. + useExternalDHCP := externalMode == "nat" && poolStart == "" // Validate IP address format if net.ParseIP(bindIP) == nil { @@ -768,6 +827,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { ExternalMode: externalMode, ExternalIface: externalIface, SingleNIC: singleNIC, + ExternalDHCP: useExternalDHCP, PoolName: "wan", PoolStart: poolStart, PoolEnd: poolEnd, @@ -775,6 +835,24 @@ func runAdminInit(cmd *cobra.Command, args []string) { PoolPrefixLen: externalPrefixLen, } + // Print external networking summary + if externalMode != "" { + fmt.Printf("\nšŸ“” External networking: %s\n", externalMode) + fmt.Printf(" WAN interface: %s\n", externalIface) + if singleNIC { + fmt.Println(" Bridge mode: macvlan (single-NIC, SSH-safe)") + } else { + fmt.Println(" Bridge mode: dedicated NIC") + } + if externalMode == "pool" { + fmt.Printf(" IP pool: %s - %s\n", poolStart, poolEnd) + fmt.Printf(" āš ļø Ensure %s-%s is excluded from your router's DHCP range.\n", poolStart, poolEnd) + } else if useExternalDHCP { + fmt.Println(" Gateway IP: DHCP (obtained during OVN setup)") + fmt.Println(" VMs get: outbound internet via SNAT") + } + } + // Generate config files configs := []admin.ConfigFile{ {Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate}, @@ -822,6 +900,9 @@ func runAdminInit(cmd *cobra.Command, args []string) { if singleNIC { ovnFlags += " --single-nic" } + if useExternalDHCP { + ovnFlags += " --dhcp" + } fmt.Printf(" %d. Setup OVN with external bridge:\n", step) fmt.Printf(" ./scripts/setup-ovn.sh %s\n", ovnFlags) step++ diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index e0fcef259..0d95ce7b4 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -6,12 +6,20 @@ node = "{{.Node}}" [network] external_mode = "{{.ExternalMode}}" +{{- if .ExternalDHCP}} +external_dhcp = true +{{- end}} [[network.external_pools]] name = "{{.PoolName}}" +{{- if .PoolStart}} range_start = "{{.PoolStart}}" range_end = "{{.PoolEnd}}" +{{- end}} gateway = "{{.PoolGateway}}" +{{- if .PoolGatewayIP}} +gateway_ip = "{{.PoolGatewayIP}}" +{{- end}} prefix_len = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}} {{- end}} diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index e853ab838..aa32b3262 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -17,6 +17,7 @@ # --external-bridge Create br-external for public subnet WAN uplink # --external-iface WAN NIC to add to br-external (default: eth1) # --single-nic Use macvlan instead of adding NIC directly (for single-NIC hosts) +# --dhcp Obtain gateway IP via DHCP on the external bridge interface # --ovn-remote OVN SB DB address (default: tcp:127.0.0.1:6642) # --encap-ip Geneve tunnel endpoint IP (default: auto-detect) # --chassis-id OVN chassis identifier (default: hostname) @@ -45,6 +46,7 @@ set -e MANAGEMENT=false EXTERNAL_BRIDGE=false SINGLE_NIC=false +EXTERNAL_DHCP=false EXTERNAL_IFACE="eth1" OVN_REMOTE="tcp:127.0.0.1:6642" ENCAP_IP="" @@ -56,6 +58,7 @@ for arg in "$@"; do --management) MANAGEMENT=true ;; --external-bridge) EXTERNAL_BRIDGE=true ;; --single-nic) SINGLE_NIC=true ;; + --dhcp) EXTERNAL_DHCP=true ;; --external-iface=*) EXTERNAL_IFACE="${arg#*=}" ;; --ovn-remote=*) OVN_REMOTE="${arg#*=}" ;; --encap-ip=*) ENCAP_IP="${arg#*=}" ;; @@ -212,6 +215,54 @@ if [ "$EXTERNAL_BRIDGE" = true ]; then echo " br-external: created with port $EXTERNAL_IFACE" fi + + # --- DHCP: obtain gateway IP for OVN SNAT --- + if [ "$EXTERNAL_DHCP" = true ]; then + echo "" + echo "Step 3c: Obtaining external gateway IP via DHCP..." + + # Determine which interface to DHCP on + if [ "$SINGLE_NIC" = true ]; then + DHCP_IFACE="spx-ext-${EXTERNAL_IFACE}" + else + DHCP_IFACE="br-external" + fi + + # Run DHCP client to get a lease + if command -v dhcpcd >/dev/null 2>&1; then + sudo dhcpcd --waitip=4 --timeout 15 "$DHCP_IFACE" 2>/dev/null || true + elif command -v dhclient >/dev/null 2>&1; then + sudo dhclient -1 -timeout 15 "$DHCP_IFACE" 2>/dev/null || true + else + echo " WARNING: no DHCP client found (dhcpcd or dhclient)" + echo " Install dhcpcd-base or isc-dhcp-client, or set gateway_ip manually" + fi + + # Read the obtained IP + DHCP_IP=$(ip -4 addr show dev "$DHCP_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1 | cut -d/ -f1) + if [ -n "$DHCP_IP" ]; then + echo " DHCP obtained: $DHCP_IP on $DHCP_IFACE" + + # Write the gateway IP to the spinifex config so vpcd can use it + CONFIG_DIR="${CONFIG_DIR:-$HOME/spinifex/config}" + CONFIG_FILE="$CONFIG_DIR/spinifex.toml" + if [ -f "$CONFIG_FILE" ]; then + # Update gateway_ip in the config + if grep -q "gateway_ip" "$CONFIG_FILE"; then + sed -i "s/gateway_ip.*/gateway_ip = \"$DHCP_IP\"/" "$CONFIG_FILE" + else + # Insert gateway_ip after the gateway line in the pool section + sed -i "/^gateway *=.*/a gateway_ip = \"$DHCP_IP\"" "$CONFIG_FILE" + fi + echo " Updated $CONFIG_FILE with gateway_ip = $DHCP_IP" + else + echo " WARNING: $CONFIG_FILE not found ā€” set gateway_ip manually" + fi + else + echo " WARNING: DHCP failed to obtain IP on $DHCP_IFACE" + echo " VMs will not have external connectivity until gateway_ip is configured" + fi + fi fi # --- Step 4: Configure OVN external_ids --- diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 602342e8c..56fd2921b 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -65,14 +65,16 @@ type ConfigSettings struct { OVNSBAddr string // External networking for public subnets - ExternalMode string // "pool", "nat", or "" (disabled) - ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") - SingleNIC bool // Use macvlan instead of adding NIC directly - PoolName string // External pool name (e.g., "wan") - PoolStart string // First IP in external pool range - PoolEnd string // Last IP in external pool range - PoolGateway string // WAN gateway IP - PoolPrefixLen int // Subnet prefix length (default 24) + ExternalMode string // "pool", "nat", or "" (disabled) + ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") + SingleNIC bool // Use macvlan instead of adding NIC directly + ExternalDHCP bool // Obtain gateway IP via DHCP on macvlan/bridge + PoolName string // External pool name (e.g., "wan") + PoolStart string // First IP in external pool range + PoolEnd string // Last IP in external pool range + PoolGateway string // WAN gateway IP + PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP) + PoolPrefixLen int // Subnet prefix length (default 24) // Other nodes in the cluster (for config source of truth) RemoteNodes []RemoteNode diff --git a/spinifex/admin/network_detect.go b/spinifex/admin/network_detect.go new file mode 100644 index 000000000..023b110a6 --- /dev/null +++ b/spinifex/admin/network_detect.go @@ -0,0 +1,201 @@ +package admin + +import ( + "fmt" + "net" + "os/exec" + "strings" +) + +// DetectedInterface represents a network interface discovered on the host. +type DetectedInterface struct { + Name string // e.g., "enp0s3" + IP string // e.g., "192.168.1.21" + Subnet string // e.g., "192.168.0.0/23" + PrefixLen int // e.g., 23 + Gateway string // non-empty only for WAN (default route) + Role string // "wan", "lan" +} + +// DetectedNetwork holds the auto-detected network topology. +type DetectedNetwork struct { + Interfaces []DetectedInterface + WAN *DetectedInterface // The interface with the default route + LANCount int // Number of non-WAN interfaces + SingleNIC bool // Only one usable NIC +} + +// DetectNetwork auto-detects the host's network topology from routing table. +// It identifies the WAN interface (default route), LAN interfaces, and +// determines single-NIC vs multi-NIC mode. +func DetectNetwork() (*DetectedNetwork, error) { + // Get default route: "default via 192.168.1.1 dev enp0s3 ..." + out, err := exec.Command("ip", "-4", "route", "show", "default").Output() + if err != nil { + return nil, fmt.Errorf("failed to get default route: %w", err) + } + defaultRoute := strings.TrimSpace(string(out)) + if defaultRoute == "" { + return nil, fmt.Errorf("no default route found ā€” host has no internet connectivity") + } + + // Parse: "default via 192.168.1.1 dev enp0s3 ..." + var wanGateway, wanIface string + fields := strings.Fields(defaultRoute) + for i, f := range fields { + if f == "via" && i+1 < len(fields) { + wanGateway = fields[i+1] + } + if f == "dev" && i+1 < len(fields) { + wanIface = fields[i+1] + } + } + if wanIface == "" || wanGateway == "" { + return nil, fmt.Errorf("could not parse default route: %s", defaultRoute) + } + + // Get all routes to find interfaces and their subnets + allRoutes, err := exec.Command("ip", "-4", "route", "show", "proto", "kernel", "scope", "link").Output() + if err != nil { + return nil, fmt.Errorf("failed to get routes: %w", err) + } + + // Parse routes: "192.168.0.0/23 dev enp0s3 ... src 192.168.1.21" + seen := make(map[string]bool) + var interfaces []DetectedInterface + + for _, line := range strings.Split(strings.TrimSpace(string(allRoutes)), "\n") { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + + subnet := fields[0] + var ifaceName, srcIP string + for i, f := range fields { + if f == "dev" && i+1 < len(fields) { + ifaceName = fields[i+1] + } + if f == "src" && i+1 < len(fields) { + srcIP = fields[i+1] + } + } + if ifaceName == "" || srcIP == "" { + continue + } + + // Skip virtual/docker/ovs interfaces + if isVirtualInterface(ifaceName) { + continue + } + + // Skip duplicates (same interface may have multiple routes) + if seen[ifaceName] { + continue + } + seen[ifaceName] = true + + prefixLen := 24 // default + if _, ipNet, err := net.ParseCIDR(subnet); err == nil { + ones, _ := ipNet.Mask.Size() + prefixLen = ones + } + + iface := DetectedInterface{ + Name: ifaceName, + IP: srcIP, + Subnet: subnet, + PrefixLen: prefixLen, + } + + if ifaceName == wanIface { + iface.Gateway = wanGateway + iface.Role = "wan" + } else { + iface.Role = "lan" + } + + interfaces = append(interfaces, iface) + } + + if len(interfaces) == 0 { + return nil, fmt.Errorf("no usable network interfaces found") + } + + result := &DetectedNetwork{ + Interfaces: interfaces, + } + + for i := range interfaces { + if interfaces[i].Role == "wan" { + result.WAN = &interfaces[i] + } else { + result.LANCount++ + } + } + + // If only the WAN interface exists (no LANs), it's single-NIC + result.SingleNIC = result.LANCount == 0 + + return result, nil +} + +// SuggestPoolRange suggests a default external IP pool range based on the +// WAN subnet. Places the pool at the high end of the subnet to avoid +// collisions with typical DHCP ranges (which start low). +func SuggestPoolRange(wan *DetectedInterface) (start, end string) { + _, ipNet, err := net.ParseCIDR(wan.Subnet) + if err != nil { + return "", "" + } + + // Calculate broadcast address + ip := ipNet.IP.To4() + mask := ipNet.Mask + broadcast := make(net.IP, 4) + for i := range ip { + broadcast[i] = ip[i] | ^mask[i] + } + + // Pool: broadcast - 100 to broadcast - 5 + // e.g., for 192.168.0.0/23 (broadcast 192.168.1.255): + // start = 192.168.1.155, end = 192.168.1.250 + endIP := make(net.IP, 4) + copy(endIP, broadcast) + endIP[3] -= 5 // Leave some room at top + + startIP := make(net.IP, 4) + copy(startIP, endIP) + // Go back 100 IPs + carry := 100 + for i := 3; i >= 0 && carry > 0; i-- { + val := int(startIP[i]) - carry + if val < 0 { + startIP[i] = byte(256 + val) + carry = 1 + } else { + startIP[i] = byte(val) + carry = 0 + } + } + + return startIP.String(), endIP.String() +} + +// isVirtualInterface returns true for interfaces that shouldn't be +// considered as physical NICs. +func isVirtualInterface(name string) bool { + prefixes := []string{ + "docker", "br-", "veth", "ovs", "virbr", + "tun", "tap", "tailscale", "wg", "lo", + } + for _, p := range prefixes { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 64c929ab3..0b554a56b 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -32,6 +32,7 @@ type ExternalPool struct { // NetworkConfig holds cluster-wide external network settings. type NetworkConfig struct { ExternalMode string `mapstructure:"external_mode"` // "pool", "nat", or "" (disabled) + ExternalDHCP bool `mapstructure:"external_dhcp"` // Gateway IP obtained via DHCP (nat mode) ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools }