gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: provision br-mgmt as OVS bridge + loopback guard for LB gateway

Browse Source 
Convert br-mgmt from an externally-provisioned Linux bridge to an OVS
bridge owned by scripts/setup-ovn.sh, giving every deployment type
(tofu-cluster, production single-node, production multi-node) a single
provisioning path.

setup-ovn.sh grows --mgmt-bridge / --mgmt-cidr / --mgmt-iface /
--no-mgmt-bridge flags. Default creates br-mgmt with fail-mode=standalone
(excluded from ovn-bridge-mappings so ovn-controller ignores it) and
assigns the mgmt IP via a systemd-networkd drop-in so it survives reboot.

SetupMgmtTapDevice / CleanupMgmtTapDevice switch from `ip link master`
to `ovs-vsctl add-port` / `del-port` to match the new bridge type.

Add a loopback guard to wireLBAgentConfig's fallback branch: on a
production single-node with awsgw bound to 127.0.0.1 and no mgmt bridge,
the fallback previously injected 127.0.0.1 into LB VM cloud-init —
silently unreachable because the VM's loopback is its own. The guard
leaves GatewayURL empty and the existing "LB agent gateway URL not
configured" error fires instead. Regression covered in
daemon_wire_lb_test.go.
This commit is contained in:
joshsiv-mulga
2026-04-21 20:55:35 +10:00
parent 0a78324405
commit 74b6e2dc2a
4 changed files with 110 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/setup-ovn.sh
+77 -1
View File
@@ -20,6 +20,10 @@
# --macvlan Create macvlan off --wan-iface instead of moving NIC directly.
# SSH-safe for single-NIC hosts where WAN NIC carries SSH.
# --dhcp Obtain gateway IP via DHCP on the WAN bridge interface
# --mgmt-bridge=NAME OVS bridge for system-instance control plane (default: br-mgmt)
# --mgmt-cidr=CIDR IPv4 CIDR to assign on the mgmt bridge (default: 10.15.8.1/24)
# --mgmt-iface=NAME Physical/virtual NIC to enslave to the mgmt bridge (multi-node only)
# --no-mgmt-bridge Skip mgmt bridge provisioning (for dev-networking hosts)
# --ovn-remote=ADDR OVN SB DB address (default: tcp:127.0.0.1:6642)
# --encap-ip=IP Geneve tunnel endpoint IP (default: auto-detect)
# --chassis-id=ID OVN chassis identifier (default: chassis-$(hostname -s))
@@ -55,6 +59,10 @@ WAN_BRIDGE=""
WAN_IFACE=""
MACVLAN_MODE=false
EXTERNAL_DHCP=false
MGMT_BRIDGE_ENABLED=true
MGMT_BRIDGE="br-mgmt"
MGMT_CIDR="10.15.8.1/24"
MGMT_IFACE=""
OVN_REMOTE="tcp:127.0.0.1:6642"
ENCAP_IP=""
CHASSIS_ID=""
@@ -67,11 +75,15 @@ for arg in "$@"; do
--dhcp) EXTERNAL_DHCP=true ;;
--wan-bridge=*) WAN_BRIDGE="${arg#*=}" ;;
--wan-iface=*) WAN_IFACE="${arg#*=}" ;;
--mgmt-bridge=*) MGMT_BRIDGE="${arg#*=}" ;;
--mgmt-cidr=*) MGMT_CIDR="${arg#*=}" ;;
--mgmt-iface=*) MGMT_IFACE="${arg#*=}" ;;
--no-mgmt-bridge) MGMT_BRIDGE_ENABLED=false ;;
--ovn-remote=*) OVN_REMOTE="${arg#*=}" ;;
--encap-ip=*) ENCAP_IP="${arg#*=}" ;;
--chassis-id=*) CHASSIS_ID="${arg#*=}" ;;
--help|-h)
head -44 "$0" | tail -42
head -50 "$0" | tail -48
exit 0
;;
*)
@@ -451,6 +463,59 @@ if [ -n "$WAN_BRIDGE" ]; then
fi
fi
# --- Step 3d: Management bridge for system-instance control plane ---
# br-mgmt is an OVS bridge (not L2-learning Linux bridge, not part of OVN
# overlay). fail-mode=standalone makes it behave like a plain L2 switch
# without OVN flows; it is excluded from ovn-bridge-mappings so
# ovn-controller ignores it. System instance TAPs attach via ovs-vsctl
# add-port (see SetupMgmtTapDevice in daemon/network.go).
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
echo ""
echo "Step 3d: Configuring management bridge ($MGMT_BRIDGE)..."
sudo ovs-vsctl --may-exist add-br "$MGMT_BRIDGE"
sudo ovs-vsctl set Bridge "$MGMT_BRIDGE" \
fail-mode=standalone \
other-config:disable-in-band=true
sudo ip link set "$MGMT_BRIDGE" up
echo " $MGMT_BRIDGE: OVS bridge, fail-mode=standalone, up"
# Enslave physical/virtual mgmt NIC when provided (multi-node deployments
# where the node's mgmt subnet spans multiple hosts).
if [ -n "$MGMT_IFACE" ]; then
if ! ip link show "$MGMT_IFACE" >/dev/null 2>&1; then
echo " ERROR: mgmt interface $MGMT_IFACE does not exist"
ip -o link show | awk -F': ' '{print " " $2}'
exit 1
fi
# Drop any existing IP on the NIC — IP belongs on the bridge.
sudo ip addr flush dev "$MGMT_IFACE" || true
sudo ovs-vsctl --may-exist add-port "$MGMT_BRIDGE" "$MGMT_IFACE"
sudo ip link set "$MGMT_IFACE" up
echo " $MGMT_IFACE: port on $MGMT_BRIDGE"
fi
if [ -n "$MGMT_CIDR" ]; then
sudo ip addr replace "$MGMT_CIDR" dev "$MGMT_BRIDGE"
echo " $MGMT_BRIDGE: address $MGMT_CIDR"
# Persist the IP across reboots via a systemd-networkd drop-in. The
# OVS bridge definition itself survives in ovsdb; only the L3 IP
# needs re-applying on boot.
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
[Match]
Name=$MGMT_BRIDGE
[Network]
Address=$MGMT_CIDR
ConfigureWithoutCarrier=yes
NETD
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
fi
fi
# --- Step 4: Configure OVN external_ids ---
echo ""
echo "Step 4: Setting OVS external_ids for OVN..."
@@ -669,6 +734,17 @@ if [ -n "$WAN_BRIDGE" ]; then
fi
fi
# Check mgmt bridge (only if enabled)
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
if sudo ovs-vsctl br-exists "$MGMT_BRIDGE"; then
MGMT_IP_ACTUAL=$(ip -4 -o addr show dev "$MGMT_BRIDGE" 2>/dev/null | awk '{print $4}' | head -1)
echo " $MGMT_BRIDGE:$(printf '%*s' $((15 - ${#MGMT_BRIDGE})) '') OK (${MGMT_IP_ACTUAL:-no IP})"
else
echo " $MGMT_BRIDGE:$(printf '%*s' $((15 - ${#MGMT_BRIDGE})) '') FAILED"
OK=false
fi
fi
# Check ovn-controller
if sudo ovs-appctl -t ovn-controller version >/dev/null 2>&1 || systemctl is-active --quiet ovn-controller 2>/dev/null; then
echo " ovn-controller: OK"
spinifex/daemon/daemon.go
+4 -1
View File
@@ -2806,7 +2806,10 @@ func (d *Daemon) wireLBAgentConfig() {
}
} else if d.config.Daemon.DevNetworking {
gatewayHost = "10.0.2.2"
} else if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" {
} else if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && !net.ParseIP(awsgwBindIP).IsLoopback() {
// Without a mgmt bridge, we can only use the AWSGW bind IP if it's
// reachable from VMs. A loopback (127.x) is the VM's own loopback,
// not the host's — guard against silent mis-injection.
gatewayHost = awsgwBindIP
}
spinifex/daemon/daemon_wire_lb_test.go
+18
View File
@@ -111,6 +111,24 @@ func TestWireLBAgentConfig_GatewayURL_AWSGWBindOnly(t *testing.T) {
d.wireLBAgentConfig()
assert.Empty(t, d.mgmtRouteVia)
assert.Equal(t, "https://192.168.1.10:8443", d.elbv2Service.GatewayURL)
}
func TestWireLBAgentConfig_GatewayURL_LoopbackBindNoMgmtBridge(t *testing.T) {
// Production single-node without br-mgmt (mulga-siv-6 regression guard):
// awsgw.host defaults to 127.0.0.1 and the host has no mgmt bridge. The
// fallback must NOT write 127.0.0.1 into LB VM cloud-init — the VM's
// loopback is its own, not the host's. GatewayURL stays empty and the
// daemon logs an error (handled downstream).
cfg := &config.Config{
AWSGW: config.AWSGWConfig{Host: "127.0.0.1:9999"},
}
d := newWireLBTestDaemon(t, cfg)
d.wireLBAgentConfig()
assert.Empty(t, d.mgmtRouteVia)
assert.Empty(t, d.elbv2Service.GatewayURL, "must not inject loopback as GatewayURL")
}
func TestWireLBAgentConfig_GatewayURL_NoHost(t *testing.T) {
spinifex/daemon/network.go
+11 -8
View File
@@ -203,15 +203,17 @@ func GetBridgeIPv4(bridgeName string) (string, error) {
return "", fmt.Errorf("no IPv4 address on %s", bridgeName)
}
// SetupMgmtTapDevice creates a TAP device and adds it to the management Linux bridge.
// Unlike VPC TAPs (OVS/OVN), this is a plain Linux bridge TAP.
// SetupMgmtTapDevice creates a TAP device and attaches it to the management
// OVS bridge (provisioned by scripts/setup-ovn.sh). br-mgmt uses
// fail-mode=standalone and is excluded from ovn-bridge-mappings, so it
// behaves as a plain L2 switch outside the OVN overlay.
func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) {
tapName := MgmtTapName(instanceID)
// Clean up stale TAP if present
if _, err := os.Stat("/sys/class/net/" + tapName); err == nil {
slog.Warn("Stale mgmt tap found, cleaning up", "tap", tapName)
if err := sudoCommand("ip", "link", "set", tapName, "nomaster").Run(); err != nil {
if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", bridge, tapName).Run(); err != nil {
slog.Warn("Failed to remove stale mgmt tap from bridge", "tap", tapName, "err", err)
}
if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil {
@@ -230,8 +232,8 @@ func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) {
return "", fmt.Errorf("bring up mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
}
// Add to Linux bridge
if out, err := sudoCommand("ip", "link", "set", tapName, "master", bridge).CombinedOutput(); err != nil {
// Attach to OVS mgmt bridge
if out, err := sudoCommand("ovs-vsctl", "--may-exist", "add-port", bridge, tapName).CombinedOutput(); err != nil {
_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
return "", fmt.Errorf("add mgmt tap to %s: %s: %w", bridge, strings.TrimSpace(string(out)), err)
}
@@ -240,10 +242,11 @@ func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) {
return tapName, nil
}
// CleanupMgmtTapDevice removes a management TAP device from its bridge and deletes it.
// CleanupMgmtTapDevice removes a management TAP device from the mgmt OVS
// bridge and deletes it.
func CleanupMgmtTapDevice(tapName string) error {
// Remove from bridge
if out, err := sudoCommand("ip", "link", "set", tapName, "nomaster").CombinedOutput(); err != nil {
// Detach from whichever OVS bridge holds this tap (usually br-mgmt)
if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", tapName).CombinedOutput(); err != nil {
slog.Warn("Failed to remove mgmt tap from bridge", "tap", tapName, "err", err, "out", strings.TrimSpace(string(out)))
}