gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: OVN RAFT multi-node bring-up (northd member list, ExecStop, add-nat deadline) (#466)

Browse Source 
* fix: repoint DB-node ovn-northd at RAFT member list; extend add-nat deadline

Two related fixes for multi-node OVN clusters:

- setup-ovn.sh (mulga-siv-504): in the clustered --management branch, point
  ovn-northd's client NB/SB connections at the full RAFT member list derived
  from OVN_REMOTE, so the active northd follows the leader instead of dialing
  its local member only. Without this, northd stops advancing SB_Global.nb_cfg
  once leadership moves off the node, wedging the ovn-nbctl --wait=hv barrier.
  Guarded on a real member list so the standalone single-DB-node path is
  unchanged.

- utils/nats.go (mulga-js-206): raise the add-nat request deadline from 10s to
  45s so the client no longer rolls back before the server-side --wait=hv flows
  barrier (30s) completes.

* fix: stop ovn-northd restart from tearing down split NB/SB ovsdb servers

The packaged ovn-northd.service ExecStop runs 'ovn-ctl stop_northd' without
--ovn-manage-ovsdb=no, so restarting northd to apply OVN_CTL_OPTS also stops the
separately-managed NB/SB ovsdb-server units. On a clean clustered bring-up this
races the freshly-started DBs and kills them, so setup-ovn's wait for the NB
socket times out and the RAFT create/join aborts. Install an ovn-northd.service
drop-in overriding ExecStop with --ovn-manage-ovsdb=no so northd never touches
the DB lifecycle (mulga-siv-504).
This commit is contained in:
Josh Siviour
2026-07-02 14:04:12 +10:00
committed by Julian Sommer
parent 656e8c8e4b
commit da33e574d4
2 changed files with 37 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/setup-ovn.sh
+26
View File
@@ -305,9 +305,35 @@ if [ "$MANAGEMENT" = true ]; then
echo " ovn-central: creating RAFT cluster (local=$DB_CLUSTER_LOCAL_ADDR)"
fi
# Point ovn-northd's client NB/SB connections at the full RAFT member
# list so the active northd follows the leader. Without this it dials the
# local member only and stops advancing SB_Global.nb_cfg once leadership
# moves off this node, wedging the ovn-nbctl --wait=hv flows barrier.
# Derived from OVN_REMOTE (the SB list); the NB list is the same hosts on
# 6641. Skipped when OVN_REMOTE is the localhost default (caller passed no
# member list) so a lone standalone DB node is unaffected.
if [ "$OVN_REMOTE" != "tcp:127.0.0.1:6642" ]; then
OVN_REMOTE_NB="${OVN_REMOTE//:6642/:6641}"
OVN_CTL_OPTS="$OVN_CTL_OPTS --ovn-northd-nb-db=$OVN_REMOTE_NB --ovn-northd-sb-db=$OVN_REMOTE"
echo " ovn-northd: NB=$OVN_REMOTE_NB SB=$OVN_REMOTE (RAFT member list)"
fi
echo "OVN_CTL_OPTS=\"$OVN_CTL_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
echo " wrote /etc/default/ovn-central"
# The packaged ovn-northd.service ExecStop runs `ovn-ctl stop_northd`
# without --ovn-manage-ovsdb=no, so restarting northd also tears down the
# NB/SB ovsdb-server units. With the split clustered units those DBs are
# owned by their own units, so override ExecStop to leave them alone —
# otherwise the restart below races and kills the freshly-started DBs.
sudo mkdir -p /etc/systemd/system/ovn-northd.service.d
sudo tee /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf >/dev/null <<'EOF'
[Service]
ExecStop=
ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ovsdb=no
EOF
sudo systemctl daemon-reload
# The ovn-central aggregator is ExecStart=/bin/true, so restarting it
# won't restart the children — restart the per-DB units directly to
# pick up the new OVN_CTL_OPTS.
spinifex/utils/nats.go
+11 -3
View File
@@ -419,13 +419,21 @@ type natEvent struct {
MAC string `json:"mac"`
}
// AddNAT requests vpcd commit the OVN dnat_and_snat rule via NATS request-reply (10 s timeout).
// addNATTimeout bounds the vpc.add-nat request-reply. vpcd's handler holds the
// reply until the flows barrier (ovn-nbctl --wait=hv sync, bounded 30 s) confirms
// every chassis realised the rule. On a cold multi-node cluster the gateway SB
// chassisredirect binding can take most of that budget to converge, so the caller
// deadline must exceed 30 s or it abandons a still-committing rule and rolls back
// the EIP mid-launch — surfacing as a spurious ServerInternal on RunInstances.
const addNATTimeout = 45 * time.Second
// AddNAT requests vpcd commit the OVN dnat_and_snat rule via NATS request-reply.
// A non-nil return means the rule may not be committed; callers must roll back and publish vpc.delete-nat.
func AddNAT(nc *nats.Conn, vpcID, externalIP, logicalIP, portName, mac string) error {
return RequestEvent(nc, "vpc.add-nat", natEvent{
VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP,
PortName: portName, MAC: mac,
}, 10*time.Second)
}, addNATTimeout)
}
// PublishNATEvent sends a NAT lifecycle event. vpc.add-nat uses request-reply (prevents ARP races);
@@ -437,7 +445,7 @@ func PublishNATEvent(nc *nats.Conn, topic, vpcID, externalIP, logicalIP, portNam
}
if topic == "vpc.add-nat" {
if err := RequestEvent(nc, topic, evt, 10*time.Second); err != nil {
if err := RequestEvent(nc, topic, evt, addNATTimeout); err != nil {
slog.Warn("PublishNATEvent: failed to add NAT rule — OVN dnat_and_snat rule not created; restart vpcd or re-associate EIP to recover",
"topic", topic, "externalIP", externalIP, "logicalIP", logicalIP, "err", err)
}