From da33e574d41b9f71eaea395402b4bd88eecef276 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Thu, 2 Jul 2026 14:04:12 +1000 Subject: [PATCH] fix: OVN RAFT multi-node bring-up (northd member list, ExecStop, add-nat deadline) (#466) * fix: repoint DB-node ovn-northd at RAFT member list; extend add-nat deadline Two related fixes for multi-node OVN clusters: - setup-ovn.sh (mulga-siv-504): in the clustered --management branch, point ovn-northd's client NB/SB connections at the full RAFT member list derived from OVN_REMOTE, so the active northd follows the leader instead of dialing its local member only. Without this, northd stops advancing SB_Global.nb_cfg once leadership moves off the node, wedging the ovn-nbctl --wait=hv barrier. Guarded on a real member list so the standalone single-DB-node path is unchanged. - utils/nats.go (mulga-js-206): raise the add-nat request deadline from 10s to 45s so the client no longer rolls back before the server-side --wait=hv flows barrier (30s) completes. * fix: stop ovn-northd restart from tearing down split NB/SB ovsdb servers The packaged ovn-northd.service ExecStop runs 'ovn-ctl stop_northd' without --ovn-manage-ovsdb=no, so restarting northd to apply OVN_CTL_OPTS also stops the separately-managed NB/SB ovsdb-server units. On a clean clustered bring-up this races the freshly-started DBs and kills them, so setup-ovn's wait for the NB socket times out and the RAFT create/join aborts. Install an ovn-northd.service drop-in overriding ExecStop with --ovn-manage-ovsdb=no so northd never touches the DB lifecycle (mulga-siv-504). --- scripts/setup-ovn.sh | 26 ++++++++++++++++++++++++++ spinifex/utils/nats.go | 14 +++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index 05818a496..3c84eba16 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -305,9 +305,35 @@ if [ "$MANAGEMENT" = true ]; then echo " ovn-central: creating RAFT cluster (local=$DB_CLUSTER_LOCAL_ADDR)" fi + # Point ovn-northd's client NB/SB connections at the full RAFT member + # list so the active northd follows the leader. Without this it dials the + # local member only and stops advancing SB_Global.nb_cfg once leadership + # moves off this node, wedging the ovn-nbctl --wait=hv flows barrier. + # Derived from OVN_REMOTE (the SB list); the NB list is the same hosts on + # 6641. Skipped when OVN_REMOTE is the localhost default (caller passed no + # member list) so a lone standalone DB node is unaffected. + if [ "$OVN_REMOTE" != "tcp:127.0.0.1:6642" ]; then + OVN_REMOTE_NB="${OVN_REMOTE//:6642/:6641}" + OVN_CTL_OPTS="$OVN_CTL_OPTS --ovn-northd-nb-db=$OVN_REMOTE_NB --ovn-northd-sb-db=$OVN_REMOTE" + echo " ovn-northd: NB=$OVN_REMOTE_NB SB=$OVN_REMOTE (RAFT member list)" + fi + echo "OVN_CTL_OPTS=\"$OVN_CTL_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null echo " wrote /etc/default/ovn-central" + # The packaged ovn-northd.service ExecStop runs `ovn-ctl stop_northd` + # without --ovn-manage-ovsdb=no, so restarting northd also tears down the + # NB/SB ovsdb-server units. With the split clustered units those DBs are + # owned by their own units, so override ExecStop to leave them alone — + # otherwise the restart below races and kills the freshly-started DBs. + sudo mkdir -p /etc/systemd/system/ovn-northd.service.d + sudo tee /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf >/dev/null <<'EOF' +[Service] +ExecStop= +ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ovsdb=no +EOF + sudo systemctl daemon-reload + # The ovn-central aggregator is ExecStart=/bin/true, so restarting it # won't restart the children — restart the per-DB units directly to # pick up the new OVN_CTL_OPTS. diff --git a/spinifex/utils/nats.go b/spinifex/utils/nats.go index a9117c9ca..5266192af 100644 --- a/spinifex/utils/nats.go +++ b/spinifex/utils/nats.go @@ -419,13 +419,21 @@ type natEvent struct { MAC string `json:"mac"` } -// AddNAT requests vpcd commit the OVN dnat_and_snat rule via NATS request-reply (10 s timeout). +// addNATTimeout bounds the vpc.add-nat request-reply. vpcd's handler holds the +// reply until the flows barrier (ovn-nbctl --wait=hv sync, bounded 30 s) confirms +// every chassis realised the rule. On a cold multi-node cluster the gateway SB +// chassisredirect binding can take most of that budget to converge, so the caller +// deadline must exceed 30 s or it abandons a still-committing rule and rolls back +// the EIP mid-launch — surfacing as a spurious ServerInternal on RunInstances. +const addNATTimeout = 45 * time.Second + +// AddNAT requests vpcd commit the OVN dnat_and_snat rule via NATS request-reply. // A non-nil return means the rule may not be committed; callers must roll back and publish vpc.delete-nat. func AddNAT(nc *nats.Conn, vpcID, externalIP, logicalIP, portName, mac string) error { return RequestEvent(nc, "vpc.add-nat", natEvent{ VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac, - }, 10*time.Second) + }, addNATTimeout) } // PublishNATEvent sends a NAT lifecycle event. vpc.add-nat uses request-reply (prevents ARP races); @@ -437,7 +445,7 @@ func PublishNATEvent(nc *nats.Conn, topic, vpcID, externalIP, logicalIP, portNam } if topic == "vpc.add-nat" { - if err := RequestEvent(nc, topic, evt, 10*time.Second); err != nil { + if err := RequestEvent(nc, topic, evt, addNATTimeout); err != nil { slog.Warn("PublishNATEvent: failed to add NAT rule — OVN dnat_and_snat rule not created; restart vpcd or re-associate EIP to recover", "topic", topic, "externalIP", externalIP, "logicalIP", logicalIP, "err", err) }