mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
fix(ovn): bind the NB/SB client listeners to loopback and the lan plane (#765)
setup-ovn.sh set the OVN Northbound and Southbound client remotes with a bare set-connection ptcp:6641 / ptcp:6642. The address-free ptcp: form binds every interface, so on a multi-plane node the unauthenticated, unencrypted OVSDB control plane was reachable from the public interface. Write access to the Southbound database is control of the whole tenant network fabric. ovsdb-server takes remotes from two additive sources: the Connection table, which RAFT replicates cluster-wide, and local ovn-ctl command-line flags. That split is what the fix uses. Loopback goes in the replicated row, because 127.0.0.1 is correct on every node, so replication is harmless and local clients keep working. The lan address goes on the per-node command line via --db-nb-addr and --db-nb-create-insecure-remote, which never replicates, so each node binds its own address without pushing it to its peers. A new --lan-addr flag carries that address, falling back to --db-cluster-local-addr so clustered nodes stay correct without caller changes. The standalone branch now writes /etc/default/ovn-central too, since a two-node install still has a remote ovn-controller that needs the lan listener. The guard on --db-nb-addr is load-bearing: it defaults to 0.0.0.0, so emitting create-insecure-remote without a resolved address would recreate the bug. install-node.sh passes the lan address on both branches and warns when the lan plane has collapsed onto the public address, since the listener then lands on the public interface and a firewall is the only remaining control.
This commit is contained in:
Josh Siviour
@@ -88,8 +88,8 @@ The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) sa
|
||||
| 8443 | spinifex-predastore (gate) | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
|
||||
| 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
|
||||
| 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
|
||||
| 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
|
||||
| 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
|
||||
| 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd. Binds `127.0.0.1` plus the node's lan-plane address (`--lan-addr`), never the wildcard address. On a node with no separate lan plane that address is the public one, and a host firewall is the only remaining control. | Cluster network only; TLS planned |
|
||||
| 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state. Binds `127.0.0.1` plus the node's lan-plane address (`--lan-addr`), never the wildcard address. On a node with no separate lan plane that address is the public one, and a host firewall is the only remaining control. | Cluster network only; TLS planned |
|
||||
| 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
|
||||
| 6644 | OVN Southbound DB (RAFT) | OVSDB/TCP | Cluster | SB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
|
||||
| 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only |
|
||||
|
||||
@@ -376,6 +376,16 @@ fi
|
||||
|
||||
DB_NODES=$((N >= 3 ? 3 : 1))
|
||||
|
||||
# A collapsed lan plane puts the OVN NB/SB listener on the public address,
|
||||
# because it is the only one the node has. Nothing can be done about that
|
||||
# here, but the operator has to know a host firewall is the only control.
|
||||
for i in $(seq 0 $((DB_NODES - 1))); do
|
||||
if [ "${LAN_IPS[$i]}" = "${WAN_IPS[$i]}" ]; then
|
||||
log "WARNING: ${HOSTS[$i]} has no separate lan plane — OVN NB/SB (6641/6642)"
|
||||
log " will listen on ${WAN_IPS[$i]}. Restrict those ports with a firewall."
|
||||
fi
|
||||
done
|
||||
|
||||
# --node-name pins the OVS system-id, which is the OVN chassis-id, which
|
||||
# ovs-monitor-ipsec uses as the IKEv2 `@<name>` peer identity. `spx admin init`
|
||||
# enables IPsec by default and bakes the cluster node name into each peer
|
||||
@@ -389,6 +399,7 @@ if [ "$DB_NODES" -eq 3 ]; then
|
||||
on "${HOSTS[0]}" "sudo $SETUP_OVN --management \
|
||||
--node-name=${NODE_NAMES[0]} \
|
||||
--db-cluster-local-addr=${LAN_IPS[0]} \
|
||||
--lan-addr=${LAN_IPS[0]} \
|
||||
--recreate-db \
|
||||
--encap-ip=${VPC_IPS[0]}" || fail "${HOSTS[0]}: could not create the OVN database cluster"
|
||||
log " ${HOSTS[0]} created the database cluster"
|
||||
@@ -398,6 +409,7 @@ if [ "$DB_NODES" -eq 3 ]; then
|
||||
--node-name=${NODE_NAMES[$i]} \
|
||||
--db-cluster-local-addr=${LAN_IPS[$i]} \
|
||||
--db-cluster-remote-addr=${LAN_IPS[0]} \
|
||||
--lan-addr=${LAN_IPS[$i]} \
|
||||
--recreate-db \
|
||||
--encap-ip=${VPC_IPS[$i]}" || fail "${HOSTS[$i]}: could not join the OVN database cluster"
|
||||
log " ${HOSTS[$i]} joined the database cluster"
|
||||
@@ -407,6 +419,7 @@ if [ "$DB_NODES" -eq 3 ]; then
|
||||
else
|
||||
on "${HOSTS[0]}" "sudo $SETUP_OVN --management \
|
||||
--node-name=${NODE_NAMES[0]} \
|
||||
--lan-addr=${LAN_IPS[0]} \
|
||||
--encap-ip=${VPC_IPS[0]}" || fail "${HOSTS[0]}: setup-ovn.sh failed"
|
||||
log " ${HOSTS[0]} is running a standalone database"
|
||||
|
||||
|
||||
@@ -296,6 +296,14 @@ else
|
||||
echo "==> WAN is physical: $WAN_BRIDGE (direct bridge br-wan)"
|
||||
fi
|
||||
|
||||
# Bind the NB/SB client listener to the lan plane only when lan is a
|
||||
# genuinely distinct plane from wan — when it folds onto wan (single-nic),
|
||||
# --lan-addr=$LAN_IP would put the listener on the public interface.
|
||||
if [ "$LAN_BRIDGE" != "$WAN_BRIDGE" ] && [ -n "$LAN_IP" ]; then
|
||||
SETUP_OVN_FLAGS="$SETUP_OVN_FLAGS --lan-addr=$LAN_IP"
|
||||
echo "==> LAN plane: $LAN_BRIDGE ($LAN_IP) — binding NB/SB client listener"
|
||||
fi
|
||||
|
||||
# Mirror the ISO installer: internal services on lan, public dial target on
|
||||
# wan. --advertise must be explicit, because resolveAdvertiseIP echoes a
|
||||
# non-wildcard --bind straight back and would publish the internal address.
|
||||
|
||||
+65
-9
@@ -31,6 +31,11 @@
|
||||
# SB endpoints for compute nodes pointing at a RAFT
|
||||
# cluster (default: tcp:127.0.0.1:6642)
|
||||
# --encap-ip=IP Geneve tunnel endpoint IP (default: auto-detect)
|
||||
# --lan-addr=IP LAN-plane IP to bind the OVN NB(6641)/SB(6642)
|
||||
# client listeners on, in addition to loopback
|
||||
# (requires --management). Falls back to
|
||||
# --db-cluster-local-addr when omitted, since every
|
||||
# clustered node already passes its own LAN IP.
|
||||
# --db-cluster-local-addr=IP This DB node's own IP for OVSDB RAFT clustering.
|
||||
# Enables clustered NB(6643)/SB(6644) DBs (requires
|
||||
# --management). Empty --db-cluster-remote-addr ⇒ create
|
||||
@@ -99,6 +104,9 @@
|
||||
#
|
||||
# # Non-bridgeable uplink (WiFi/cellular) — routed NAT, outbound-only VMs:
|
||||
# ./scripts/setup-ovn.sh --management --nat-uplink
|
||||
#
|
||||
# # Standalone management node, NB/SB client listeners also on the LAN plane:
|
||||
# ./scripts/setup-ovn.sh --management --lan-addr=10.0.0.1
|
||||
|
||||
set -e
|
||||
|
||||
@@ -114,6 +122,10 @@ MGMT_CIDR="10.15.8.1/24"
|
||||
MGMT_IFACE=""
|
||||
OVN_REMOTE="tcp:127.0.0.1:6642"
|
||||
ENCAP_IP=""
|
||||
# LAN-plane IP for the NB/SB client listeners (6641/6642), in addition to
|
||||
# loopback. Empty means loopback-only; resolved further down against
|
||||
# DB_CLUSTER_LOCAL_ADDR once arguments are parsed.
|
||||
LAN_ADDR=""
|
||||
# OVSDB RAFT clustering. Empty by default ⇒ single, non-clustered ovn-central
|
||||
# (dev box, existing single-node clusters). When LOCAL_ADDR is set, the NB/SB
|
||||
# DBs run clustered; REMOTE_ADDR empty ⇒ create the cluster, set ⇒ join it.
|
||||
@@ -145,6 +157,7 @@ for arg in "$@"; do
|
||||
--no-mgmt-bridge) MGMT_BRIDGE_ENABLED=false ;;
|
||||
--ovn-remote=*) OVN_REMOTE="${arg#*=}" ;;
|
||||
--encap-ip=*) ENCAP_IP="${arg#*=}" ;;
|
||||
--lan-addr=*) LAN_ADDR="${arg#*=}" ;;
|
||||
--db-cluster-local-addr=*) DB_CLUSTER_LOCAL_ADDR="${arg#*=}" ;;
|
||||
--db-cluster-remote-addr=*) DB_CLUSTER_REMOTE_ADDR="${arg#*=}" ;;
|
||||
--recreate-db) RECREATE_DB=true ;;
|
||||
@@ -177,6 +190,13 @@ if [ "$RECREATE_DB" = true ] && [ -z "$DB_CLUSTER_LOCAL_ADDR" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Fall back to the RAFT clustering address when no --lan-addr was given:
|
||||
# every clustered node already passes --db-cluster-local-addr with its LAN
|
||||
# IP, so clusters stay correct even from a caller that predates this flag.
|
||||
if [ -z "$LAN_ADDR" ] && [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
|
||||
LAN_ADDR="$DB_CLUSTER_LOCAL_ADDR"
|
||||
fi
|
||||
|
||||
# --- WAN bridge auto-detection ---
|
||||
# Determine the WAN bridge name and how to set it up.
|
||||
WAN_BRIDGE_MODE="" # "existing", "veth", "direct", or ""
|
||||
@@ -395,6 +415,14 @@ echo " openvswitch-switch: started"
|
||||
if [ "$MANAGEMENT" = true ]; then
|
||||
sudo systemctl enable ovn-central
|
||||
|
||||
# LAN-plane NB/SB client listener, added to the loopback one set below.
|
||||
# Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so
|
||||
# create-insecure-remote=yes with no LAN_ADDR recreates the wildcard bug.
|
||||
OVN_DB_LISTEN_OPTS=""
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
OVN_DB_LISTEN_OPTS="--db-nb-addr=$LAN_ADDR --db-nb-create-insecure-remote=yes --db-sb-addr=$LAN_ADDR --db-sb-create-insecure-remote=yes"
|
||||
fi
|
||||
|
||||
if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
|
||||
ensure_clustered_db_storage
|
||||
|
||||
@@ -424,6 +452,13 @@ if [ "$MANAGEMENT" = true ]; then
|
||||
echo " ovn-northd: NB=$OVN_REMOTE_NB SB=$OVN_REMOTE (RAFT member list)"
|
||||
fi
|
||||
|
||||
if [ -n "$OVN_DB_LISTEN_OPTS" ]; then
|
||||
OVN_CTL_OPTS="$OVN_CTL_OPTS $OVN_DB_LISTEN_OPTS"
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
else
|
||||
echo " NB/SB client listen: 127.0.0.1"
|
||||
fi
|
||||
|
||||
echo "OVN_CTL_OPTS=\"$OVN_CTL_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
|
||||
@@ -446,8 +481,24 @@ EOF
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd
|
||||
echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)"
|
||||
else
|
||||
sudo systemctl start ovn-central
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
if [ -n "$OVN_DB_LISTEN_OPTS" ]; then
|
||||
# A two-node standalone install still has a remote
|
||||
# ovn-controller needing the LAN listener — write the file
|
||||
# before starting, same as the clustered branch.
|
||||
echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
sudo systemctl start ovn-central
|
||||
|
||||
# ovn-central is ExecStart=/bin/true; restarting it does not
|
||||
# restart its children, so a re-run with a changed LAN_ADDR needs
|
||||
# the per-DB units restarted directly to pick up the new options.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
else
|
||||
sudo systemctl start ovn-central
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Wait for OVN NB DB socket to become available
|
||||
@@ -495,14 +546,19 @@ EOF
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Set the NB/SB client listen addresses. set-connection writes through RAFT
|
||||
# (replicated cluster-wide), so it only runs on the create node — a joining
|
||||
# node would redirect to the leader, which the init node already configured.
|
||||
# set-connection replicates through RAFT, so it only runs on the create
|
||||
# node. Scoped to loopback, not a wildcard ptcp:6641, because 127.0.0.1
|
||||
# is valid on every node; the differing LAN address is set above instead.
|
||||
if [ -z "$DB_CLUSTER_REMOTE_ADDR" ]; then
|
||||
sudo ovn-nbctl set-connection ptcp:6641
|
||||
sudo ovn-sbctl set-connection ptcp:6642
|
||||
echo " OVN NB DB listening on tcp:6641"
|
||||
echo " OVN SB DB listening on tcp:6642"
|
||||
sudo ovn-nbctl set-connection ptcp:6641:127.0.0.1
|
||||
sudo ovn-sbctl set-connection ptcp:6642:127.0.0.1
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
echo " OVN NB DB listening on tcp:127.0.0.1:6641, tcp:$LAN_ADDR:6641"
|
||||
echo " OVN SB DB listening on tcp:127.0.0.1:6642, tcp:$LAN_ADDR:6642"
|
||||
else
|
||||
echo " OVN NB DB listening on tcp:127.0.0.1:6641"
|
||||
echo " OVN SB DB listening on tcp:127.0.0.1:6642"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user