From 8c6f66670aa0acd0db56b23b22e2bedeff7a014d Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:02:22 +1000 Subject: [PATCH] feat: add wan_bridge config field, update vpcd detection + network_detect - config.go: Add WanBridge field to VPCDConfig (default "br-wan") - spinifex.toml template: Add wan_bridge under [nodes.X.vpcd] - admin.go: Auto-detect WAN bridge name from default route interface, add detectedWanBridge helper - service.go: Wire WanBridge from cluster config to vpcd Config - vpcd.go: Add WanBridge to Config, log it on startup, update comments - network_detect.go: Fix isVirtualInterface to accept the default route device as a parameter instead of hardcoding br-external. Bridges like br-wan are now correctly treated as physical when they carry the default route. Part of mulga-741. --- cmd/spinifex/cmd/admin.go | 17 +++++++++++++++++ cmd/spinifex/cmd/service.go | 1 + cmd/spinifex/cmd/templates/spinifex.toml | 3 +++ spinifex/admin/admin.go | 1 + spinifex/admin/network_detect.go | 15 +++++++++------ spinifex/config/config.go | 3 ++- spinifex/services/vpcd/vpcd.go | 21 ++++++++++++++------- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 7128d78d0..849daba93 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -845,6 +845,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { ExternalMode: externalMode, ExternalIface: externalIface, + WanBridge: detectedWanBridge(detectedNet), ExternalDHCP: useExternalDHCP, PoolName: "wan", PoolStart: poolStart, @@ -1704,6 +1705,22 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey, } // detectDNSServers reads the DNS servers configured on the host for the given +// detectedWanBridge returns the WAN bridge name from the detected network +// topology. If the WAN interface is already a bridge, returns its name. +// Otherwise returns "br-wan" as the default. +func detectedWanBridge(detected *admin.DetectedNetwork) string { + if detected == nil || detected.WAN == nil { + return "" + } + // Check if the WAN interface name looks like a bridge (starts with "br-") + // If so, it's already set up — use it directly. + if strings.HasPrefix(detected.WAN.Name, "br-") { + return detected.WAN.Name + } + return "br-wan" +} + +// detectDNSServers auto-detects DNS servers from the host for the specified // interface. Uses resolvectl (systemd-resolved) first, then falls back to // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found. func detectDNSServers(iface string) []string { diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 168ace955..457c9d565 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -740,6 +740,7 @@ var vpcdStartCmd = &cobra.Command{ ChassisNames: chassisNames, Bootstrap: bootstrap, ExternalInterface: nodeConfig.VPCD.ExternalInterface, + WanBridge: nodeConfig.VPCD.WanBridge, BridgeMode: nodeConfig.VPCD.BridgeMode, }) if err != nil { diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index a89db6824..c9ac8f2a7 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -81,6 +81,9 @@ ovn_sb_addr = "{{.OVNSBAddr}}" {{- if .ExternalIface}} external_interface = "{{.ExternalIface}}" {{- end}} +{{- if .WanBridge}} +wan_bridge = "{{.WanBridge}}" +{{- end}} [nodes.{{.Node}}.predastore] host = "{{.BindIP}}:8443" diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index c51199d4f..10e0c8667 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -67,6 +67,7 @@ type ConfigSettings struct { // External networking for public subnets ExternalMode string // "pool", "nat", or "" (disabled) ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") + WanBridge string // OVS bridge for WAN traffic (default "br-wan") ExternalDHCP bool // Obtain gateway IP via DHCP on macvlan/bridge PoolName string // External pool name (e.g., "wan") PoolStart string // First IP in external pool range diff --git a/spinifex/admin/network_detect.go b/spinifex/admin/network_detect.go index 23d9f7cc7..df2604274 100644 --- a/spinifex/admin/network_detect.go +++ b/spinifex/admin/network_detect.go @@ -85,8 +85,8 @@ func DetectNetwork() (*DetectedNetwork, error) { continue } - // Skip virtual/docker/ovs interfaces - if isVirtualInterface(ifaceName) { + // Skip virtual/docker/ovs interfaces (but allow the default route device) + if isVirtualInterface(ifaceName, wanIface) { continue } @@ -181,10 +181,13 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) { } // isVirtualInterface returns true for interfaces that shouldn't be -// considered as physical NICs. -func isVirtualInterface(name string) bool { - // Allow br-external (OVS WAN bridge used for external networking) - if name == "br-external" { +// considered as physical NICs. The defaultRouteDev parameter is the +// interface carrying the default route — it's always considered physical +// even if its name matches a virtual prefix (e.g., br-wan). +func isVirtualInterface(name string, defaultRouteDev string) bool { + // The default route interface is always considered physical — it's the + // WAN uplink, even if it's a bridge (br-wan, br-external, etc.). + if defaultRouteDev != "" && name == defaultRouteDev { return false } prefixes := []string{ diff --git a/spinifex/config/config.go b/spinifex/config/config.go index a86d0c5c8..fc851cb6e 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -89,7 +89,8 @@ type ViperblockConfig struct { type VPCDConfig struct { OVNNBAddr string `mapstructure:"ovn_nb_addr"` // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641") OVNSBAddr string `mapstructure:"ovn_sb_addr"` // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642") - ExternalInterface string `mapstructure:"external_interface"` // WAN NIC name for br-external (per-node, e.g., "eth1", "enp0s3") + ExternalInterface string `mapstructure:"external_interface"` // WAN NIC name (e.g., "eth1", "enp0s3") — the physical NIC on the WAN bridge + WanBridge string `mapstructure:"wan_bridge"` // OVS bridge for WAN traffic (default "br-wan", maps to OVN "external" network) BridgeMode string `mapstructure:"bridge_mode"` // "direct" or "macvlan" (auto-detected if empty) } diff --git a/spinifex/services/vpcd/vpcd.go b/spinifex/services/vpcd/vpcd.go index 54b3af8ed..18b69817d 100644 --- a/spinifex/services/vpcd/vpcd.go +++ b/spinifex/services/vpcd/vpcd.go @@ -64,9 +64,12 @@ type Config struct { // ExternalInterface is the WAN NIC name (e.g., "enp0s3"). Used to align // the macvlan MAC with the OVN gateway MAC for inbound traffic. ExternalInterface string + // WanBridge is the OVS bridge name for WAN traffic (default "br-wan"). + // Maps to OVN logical network "external" via ovn-bridge-mappings. + WanBridge string // BridgeMode is "direct" or "macvlan". Direct bridge adds the WAN NIC - // directly to br-external; macvlan creates a sub-interface. When empty, - // auto-detected from the br-external port type at startup. + // directly to the WAN bridge; macvlan creates a sub-interface. When empty, + // auto-detected from the WAN bridge port type at startup. BridgeMode string } @@ -223,7 +226,7 @@ func launchService(cfg *Config) error { slog.Info("Connected to OVN NB DB", "endpoint", cfg.OVNNBAddr) // Detect bridge mode: if not explicitly configured, auto-detect by checking - // whether br-external has a macvlan port or a physical NIC. + // whether the WAN bridge has a macvlan port or a physical NIC. bridgeMode := cfg.BridgeMode if bridgeMode == "" && cfg.ExternalInterface != "" { bridgeMode = detectBridgeMode(cfg.ExternalInterface) @@ -231,7 +234,11 @@ func launchService(cfg *Config) error { if bridgeMode == "" { bridgeMode = BridgeModeMacvlan // default for backward compatibility } - slog.Info("External bridge mode", "mode", bridgeMode) + wanBridge := cfg.WanBridge + if wanBridge == "" { + wanBridge = "br-wan" + } + slog.Info("External bridge mode", "mode", bridgeMode, "wan_bridge", wanBridge) // Reconcile OVN topology from bootstrap config before subscribing. // This ensures the default VPC topology exists even if admin init ran @@ -292,14 +299,14 @@ func launchService(cfg *Config) error { return nil } -// detectBridgeMode checks how br-external is wired. If a macvlan interface +// detectBridgeMode checks how the WAN bridge is wired. If a macvlan interface // (spx-ext-{iface}) exists, we're in macvlan mode. Otherwise, the physical -// NIC is added directly to br-external (direct bridge mode). +// NIC is added directly to the WAN bridge (direct bridge mode). func detectBridgeMode(externalIface string) string { macvlanName := "spx-ext-" + externalIface out, err := exec.Command("ip", "-d", "link", "show", macvlanName).CombinedOutput() if err == nil && strings.Contains(string(out), "macvlan") { - slog.Debug("vpcd: detected macvlan interface on br-external", "iface", macvlanName) + slog.Debug("vpcd: detected macvlan interface on WAN bridge", "iface", macvlanName) return BridgeModeMacvlan } slog.Debug("vpcd: no macvlan found, assuming direct bridge mode", "checked", macvlanName)