diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index cb1fbbb4d..b12fe39ed 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -20,6 +20,10 @@ # --macvlan Create macvlan off --wan-iface instead of moving NIC directly. # SSH-safe for single-NIC hosts where WAN NIC carries SSH. # --dhcp Obtain gateway IP via DHCP on the WAN bridge interface +# --mgmt-bridge=NAME OVS bridge for system-instance control plane (default: br-mgmt) +# --mgmt-cidr=CIDR IPv4 CIDR to assign on the mgmt bridge (default: 10.15.8.1/24) +# --mgmt-iface=NAME Physical/virtual NIC to enslave to the mgmt bridge (multi-node only) +# --no-mgmt-bridge Skip mgmt bridge provisioning (for dev-networking hosts) # --ovn-remote=ADDR OVN SB DB address (default: tcp:127.0.0.1:6642) # --encap-ip=IP Geneve tunnel endpoint IP (default: auto-detect) # --chassis-id=ID OVN chassis identifier (default: chassis-$(hostname -s)) @@ -55,6 +59,10 @@ WAN_BRIDGE="" WAN_IFACE="" MACVLAN_MODE=false EXTERNAL_DHCP=false +MGMT_BRIDGE_ENABLED=true +MGMT_BRIDGE="br-mgmt" +MGMT_CIDR="10.15.8.1/24" +MGMT_IFACE="" OVN_REMOTE="tcp:127.0.0.1:6642" ENCAP_IP="" CHASSIS_ID="" @@ -67,11 +75,15 @@ for arg in "$@"; do --dhcp) EXTERNAL_DHCP=true ;; --wan-bridge=*) WAN_BRIDGE="${arg#*=}" ;; --wan-iface=*) WAN_IFACE="${arg#*=}" ;; + --mgmt-bridge=*) MGMT_BRIDGE="${arg#*=}" ;; + --mgmt-cidr=*) MGMT_CIDR="${arg#*=}" ;; + --mgmt-iface=*) MGMT_IFACE="${arg#*=}" ;; + --no-mgmt-bridge) MGMT_BRIDGE_ENABLED=false ;; --ovn-remote=*) OVN_REMOTE="${arg#*=}" ;; --encap-ip=*) ENCAP_IP="${arg#*=}" ;; --chassis-id=*) CHASSIS_ID="${arg#*=}" ;; --help|-h) - head -44 "$0" | tail -42 + head -50 "$0" | tail -48 exit 0 ;; *) @@ -451,6 +463,59 @@ if [ -n "$WAN_BRIDGE" ]; then fi fi +# --- Step 3d: Management bridge for system-instance control plane --- +# br-mgmt is an OVS bridge (not L2-learning Linux bridge, not part of OVN +# overlay). fail-mode=standalone makes it behave like a plain L2 switch +# without OVN flows; it is excluded from ovn-bridge-mappings so +# ovn-controller ignores it. System instance TAPs attach via ovs-vsctl +# add-port (see SetupMgmtTapDevice in daemon/network.go). +if [ "$MGMT_BRIDGE_ENABLED" = true ]; then + echo "" + echo "Step 3d: Configuring management bridge ($MGMT_BRIDGE)..." + + sudo ovs-vsctl --may-exist add-br "$MGMT_BRIDGE" + sudo ovs-vsctl set Bridge "$MGMT_BRIDGE" \ + fail-mode=standalone \ + other-config:disable-in-band=true + sudo ip link set "$MGMT_BRIDGE" up + echo " $MGMT_BRIDGE: OVS bridge, fail-mode=standalone, up" + + # Enslave physical/virtual mgmt NIC when provided (multi-node deployments + # where the node's mgmt subnet spans multiple hosts). + if [ -n "$MGMT_IFACE" ]; then + if ! ip link show "$MGMT_IFACE" >/dev/null 2>&1; then + echo " ERROR: mgmt interface $MGMT_IFACE does not exist" + ip -o link show | awk -F': ' '{print " " $2}' + exit 1 + fi + + # Drop any existing IP on the NIC — IP belongs on the bridge. + sudo ip addr flush dev "$MGMT_IFACE" || true + sudo ovs-vsctl --may-exist add-port "$MGMT_BRIDGE" "$MGMT_IFACE" + sudo ip link set "$MGMT_IFACE" up + echo " $MGMT_IFACE: port on $MGMT_BRIDGE" + fi + + if [ -n "$MGMT_CIDR" ]; then + sudo ip addr replace "$MGMT_CIDR" dev "$MGMT_BRIDGE" + echo " $MGMT_BRIDGE: address $MGMT_CIDR" + + # Persist the IP across reboots via a systemd-networkd drop-in. The + # OVS bridge definition itself survives in ovsdb; only the L3 IP + # needs re-applying on boot. + MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network" + sudo tee "$MGMT_NETD_UNIT" >/dev/null </dev/null | awk '{print $4}' | head -1) + echo " $MGMT_BRIDGE:$(printf '%*s' $((15 - ${#MGMT_BRIDGE})) '') OK (${MGMT_IP_ACTUAL:-no IP})" + else + echo " $MGMT_BRIDGE:$(printf '%*s' $((15 - ${#MGMT_BRIDGE})) '') FAILED" + OK=false + fi +fi + # Check ovn-controller if sudo ovs-appctl -t ovn-controller version >/dev/null 2>&1 || systemctl is-active --quiet ovn-controller 2>/dev/null; then echo " ovn-controller: OK" diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index ea7a79759..7cfcf5206 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -2806,7 +2806,10 @@ func (d *Daemon) wireLBAgentConfig() { } } else if d.config.Daemon.DevNetworking { gatewayHost = "10.0.2.2" - } else if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" { + } else if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && !net.ParseIP(awsgwBindIP).IsLoopback() { + // Without a mgmt bridge, we can only use the AWSGW bind IP if it's + // reachable from VMs. A loopback (127.x) is the VM's own loopback, + // not the host's — guard against silent mis-injection. gatewayHost = awsgwBindIP } diff --git a/spinifex/daemon/daemon_wire_lb_test.go b/spinifex/daemon/daemon_wire_lb_test.go index 52e8620de..de0dcc34d 100644 --- a/spinifex/daemon/daemon_wire_lb_test.go +++ b/spinifex/daemon/daemon_wire_lb_test.go @@ -111,6 +111,24 @@ func TestWireLBAgentConfig_GatewayURL_AWSGWBindOnly(t *testing.T) { d.wireLBAgentConfig() assert.Empty(t, d.mgmtRouteVia) + assert.Equal(t, "https://192.168.1.10:8443", d.elbv2Service.GatewayURL) +} + +func TestWireLBAgentConfig_GatewayURL_LoopbackBindNoMgmtBridge(t *testing.T) { + // Production single-node without br-mgmt (mulga-siv-6 regression guard): + // awsgw.host defaults to 127.0.0.1 and the host has no mgmt bridge. The + // fallback must NOT write 127.0.0.1 into LB VM cloud-init — the VM's + // loopback is its own, not the host's. GatewayURL stays empty and the + // daemon logs an error (handled downstream). + cfg := &config.Config{ + AWSGW: config.AWSGWConfig{Host: "127.0.0.1:9999"}, + } + d := newWireLBTestDaemon(t, cfg) + + d.wireLBAgentConfig() + + assert.Empty(t, d.mgmtRouteVia) + assert.Empty(t, d.elbv2Service.GatewayURL, "must not inject loopback as GatewayURL") } func TestWireLBAgentConfig_GatewayURL_NoHost(t *testing.T) { diff --git a/spinifex/daemon/network.go b/spinifex/daemon/network.go index 2aca7b2e1..57e76bc76 100644 --- a/spinifex/daemon/network.go +++ b/spinifex/daemon/network.go @@ -203,15 +203,17 @@ func GetBridgeIPv4(bridgeName string) (string, error) { return "", fmt.Errorf("no IPv4 address on %s", bridgeName) } -// SetupMgmtTapDevice creates a TAP device and adds it to the management Linux bridge. -// Unlike VPC TAPs (OVS/OVN), this is a plain Linux bridge TAP. +// SetupMgmtTapDevice creates a TAP device and attaches it to the management +// OVS bridge (provisioned by scripts/setup-ovn.sh). br-mgmt uses +// fail-mode=standalone and is excluded from ovn-bridge-mappings, so it +// behaves as a plain L2 switch outside the OVN overlay. func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) { tapName := MgmtTapName(instanceID) // Clean up stale TAP if present if _, err := os.Stat("/sys/class/net/" + tapName); err == nil { slog.Warn("Stale mgmt tap found, cleaning up", "tap", tapName) - if err := sudoCommand("ip", "link", "set", tapName, "nomaster").Run(); err != nil { + if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", bridge, tapName).Run(); err != nil { slog.Warn("Failed to remove stale mgmt tap from bridge", "tap", tapName, "err", err) } if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil { @@ -230,8 +232,8 @@ func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) { return "", fmt.Errorf("bring up mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err) } - // Add to Linux bridge - if out, err := sudoCommand("ip", "link", "set", tapName, "master", bridge).CombinedOutput(); err != nil { + // Attach to OVS mgmt bridge + if out, err := sudoCommand("ovs-vsctl", "--may-exist", "add-port", bridge, tapName).CombinedOutput(); err != nil { _ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run() return "", fmt.Errorf("add mgmt tap to %s: %s: %w", bridge, strings.TrimSpace(string(out)), err) } @@ -240,10 +242,11 @@ func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) { return tapName, nil } -// CleanupMgmtTapDevice removes a management TAP device from its bridge and deletes it. +// CleanupMgmtTapDevice removes a management TAP device from the mgmt OVS +// bridge and deletes it. func CleanupMgmtTapDevice(tapName string) error { - // Remove from bridge - if out, err := sudoCommand("ip", "link", "set", tapName, "nomaster").CombinedOutput(); err != nil { + // Detach from whichever OVS bridge holds this tap (usually br-mgmt) + if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", tapName).CombinedOutput(); err != nil { slog.Warn("Failed to remove mgmt tap from bridge", "tap", tapName, "err", err, "out", strings.TrimSpace(string(out))) }