fix: let a role-assigning caller demote an OVN management node

setup-ovn.sh refuses to demote a node a previous run made a management
node, because demoting one stops and disables OVN central and takes the
cluster's database away. The guard is right to exist: the symptom lands
three layers off, with nothing on 6641 and vpcd crash-looping on
"connection refused", and it reached 15,061 restarts on one host before
anyone traced it back here.

What it cannot do is tell an operator who re-ran and forgot
--management from a tool that is assigning roles. Formation is the
second case. provision-iso-cluster.sh promotes every host during
--reset-to-iso because that is what firstboot leaves behind, and
install-node.sh then builds the database across a quorum of three and
attaches the rest as compute nodes, so on a cluster of more than three
it demotes by design. The guard refused the fourth host and the run
stopped with every node a standalone single-node cluster.

Add --demote, which says the demotion is deliberate and clears the role
marker. Clearing it is the demotion rather than a side effect: left
behind it names a role the node no longer has, and the next run without
the flag would refuse all over again. install-node.sh passes it for
nodes past the database quorum and nowhere else, so an operator running
setup-ovn.sh by hand is refused exactly as before.

The guard moves into assert_demotion_allowed so it can be tested. It
had no coverage at all, which is how it came to break formation. Four
cases cover both marker files, the refusal with its markers left intact,
and a node that was never a management node.

scripts/install-node.sh

@@ -622,9 +622,15 @@ else
 fi
 
 # Compute nodes: no database of their own, pointed at every database node.
+#
+# --demote because these hosts may already carry a management role marker:
+# provision-iso-cluster.sh promotes every host during --reset-to-iso, since that
+# is what firstboot leaves behind, and formation is what decides which of them
+# hold the database. Without it setup-ovn.sh refuses the fourth host and the run
+# stops with every node left a standalone single-node cluster.
 if [ "$N" -gt "$DB_NODES" ]; then
     for i in $(seq "$DB_NODES" $((N - 1))); do
-        on "${HOSTS[$i]}" "sudo $SETUP_OVN \
+        on "${HOSTS[$i]}" "sudo $SETUP_OVN --demote \
             --node-name=${NODE_NAMES[$i]} \
             --ovn-remote=$OVN_REMOTE $OVN_UPLINK_ARGS \
             --encap-ip=${VPC_IPS[$i]}" || fail "${HOSTS[$i]}: setup-ovn.sh failed"

scripts/setup-ovn.sh

@@ -23,6 +23,12 @@
 #
 # Options:
 #   --management         Also start OVN central services (NB DB, SB DB, ovn-northd)
+#   --demote             Allow demoting a node a previous run made a management
+#                        node, clearing its role marker. Without this a run that
+#                        omits --management on such a node is refused, because
+#                        demoting one takes the cluster's database down. Pass it
+#                        only from a caller that assigns roles, as
+#                        install-node.sh does for nodes past the DB quorum.
 #   --wan-bridge=NAME    OVS bridge for WAN traffic (default: auto-detect from default route)
 #   --wan-iface=NAME     Physical NIC to add to the WAN bridge (use with --wan-bridge)
 #   --dhcp               Obtain gateway IP via DHCP on the WAN bridge interface
@@ -147,6 +153,7 @@ fi
 
 # Defaults
 MANAGEMENT=false
+DEMOTE=false
 WAN_BRIDGE=""
 WAN_IFACE=""
 EXTERNAL_DHCP=false
@@ -193,6 +200,7 @@ DRY_RUN=false
 for arg in "$@"; do
     case "$arg" in
         --management)       MANAGEMENT=true ;;
+        --demote)           DEMOTE=true ;;
         --dhcp)             EXTERNAL_DHCP=true ;;
         --nat-uplink)       NAT_UPLINK=true ;;
         --wan-bridge=*)     WAN_BRIDGE="${arg#*=}" ;;
@@ -852,6 +860,43 @@ management_role_markers() {
     return $found
 }
 
+# Refuse to demote a node a previous run set up as a management node, because
+# demoting one stops and disables OVN central and takes the cluster's database
+# away. The symptom lands three layers off: nothing listens on 6641, and vpcd
+# crash-loops on "connection refused" with no hint that this script caused it.
+# Seen on mulga-poc at 15,061 restarts.
+#
+# --demote is how a caller that assigns roles says it meant it. install-node.sh
+# passes it for every node past the database quorum, and clearing the marker is
+# the demotion: left behind it names a role the node no longer has, and the next
+# run without --demote would refuse all over again.
+#
+# The signal is a role marker rather than the presence of a database. The OVN
+# central package creates databases on install, so every compute node has them
+# and testing for those would refuse every legitimate compute node.
+assert_demotion_allowed() {
+    local markers
+    markers=$(management_role_markers) || return 0
+    markers=${markers//$'\n'/ }
+
+    if [ "$DEMOTE" != true ]; then
+        echo "ERROR: this node was configured as a management node by a previous run"
+        echo "  (recorded in $markers), but neither --management nor --demote was given."
+        echo "  Re-running without either stops and disables OVN central and takes"
+        echo "  the control plane down."
+        echo ""
+        echo "  If this is a management node, re-run with --management."
+        echo "  If it is genuinely being demoted to a compute node, re-run with"
+        echo "  --demote, or clear the marker by hand:"
+        echo "     sudo rm -f $markers"
+        exit 1
+    fi
+
+    echo "  --demote: clearing the management role marker ($markers)"
+    # shellcheck disable=SC2086  # deliberately word-split: one or both markers.
+    sudo rm -f $markers
+}
+
 echo ""
 echo "Step 2: Enabling services..."
 
@@ -1107,29 +1152,7 @@ else
     # wrong database — a `--wait=hv` there can never be acknowledged and burns
     # its whole timeout instead.
     #
-    # Refuse to demote a node this script previously set up as a management
-    # node. --management is the only record of the role and is re-asserted on
-    # every run, so a re-run that forgets it silently takes the cluster's
-    # database away. The symptom lands three layers off: nothing listens on
-    # 6641, and vpcd crash-loops on "connection refused" with no hint that this
-    # script caused it. Seen on mulga-poc at 15,061 restarts.
-    #
-    # The signal is a role marker, not the presence of a database: the OVN
-    # central *package* creates DBs on install, so every compute node has them
-    # and testing for them would refuse every legitimate compute node. Only the
-    # --management path above writes a marker (management_role_markers).
-    if ROLE_MARKERS=$(management_role_markers); then
-        ROLE_MARKERS=${ROLE_MARKERS//$'\n'/ }
-        echo "ERROR: this node was configured as a management node by a previous run"
-        echo "  (recorded in $ROLE_MARKERS), but --management was not given."
-        echo "  Re-running without it stops and disables OVN central and takes the"
-        echo "  control plane down."
-        echo ""
-        echo "  If this is a management node, re-run with --management."
-        echo "  If it is genuinely being demoted to a compute node:"
-        echo "     sudo rm -f $ROLE_MARKERS"
-        exit 1
-    fi
+    assert_demotion_allowed
 
     if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 ||
         systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then

scripts/setup-ovn_test.sh

@@ -478,6 +478,70 @@ else
 fi
 printf '#!/bin/sh\nexit 1\n' >"$TMP/bin/pgrep"
 
+# --------------------------------------------------------------------------
+# Demoting a management node to a compute node.
+# --------------------------------------------------------------------------
+# install-node.sh assigns roles: past a three-node quorum the rest are compute
+# nodes, and provision-iso-cluster.sh has already promoted every host because
+# that is what firstboot leaves behind. Formation has to be able to say a
+# demotion is deliberate, while an operator who re-runs and forgets
+# --management is still refused.
+for fn in management_role_markers assert_demotion_allowed; do
+    awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/demote_fn.sh"
+done
+
+# demote_fixture <role|legacy...> — writes the named role markers and no others.
+demote_fixture() {
+    rm -f "$TMP/ovn-role" "$TMP/ovn-central"
+    for m in "$@"; do
+        case "$m" in
+            role) echo management >"$TMP/ovn-role" ;;
+            legacy) echo 'OVN_CTL_OPTS=--db-nb-addr=10.0.0.1' >"$TMP/ovn-central" ;;
+        esac
+    done
+}
+
+# demote <demote:true|false> — runs assert_demotion_allowed; sets RC, OUT.
+demote() {
+    RC=0
+    OUT=$(PATH="$TMP/bin:$PATH" bash -c '
+        OVN_ROLE_FILE="$1"; LEGACY_OVN_ROLE_FILE="$2"; DEMOTE="$3"
+        . "$4"; assert_demotion_allowed' _ \
+        "$TMP/ovn-role" "$TMP/ovn-central" "$1" "$TMP/demote_fn.sh" 2>&1) || RC=$?
+}
+
+demote_fixture role legacy
+demote false
+if [ "$RC" != 0 ] && [ -e "$TMP/ovn-role" ] && [ -e "$TMP/ovn-central" ]; then
+    pass "demote: a management node is refused without --demote, and its markers are left alone"
+else
+    fail "demote: no --demote gave rc=$RC, markers remaining [$(ls -A "$TMP" | grep -c 'ovn-role\|ovn-central')]"
+fi
+
+demote_fixture role legacy
+demote true
+if [ "$RC" = 0 ] && [ ! -e "$TMP/ovn-role" ] && [ ! -e "$TMP/ovn-central" ]; then
+    pass "demote: --demote allows the demotion and clears both role markers"
+else
+    fail "demote: --demote gave rc=$RC, out=[$OUT], remaining [$(ls -A "$TMP" | grep 'ovn-role\|ovn-central' | tr '\n' ' ')]"
+fi
+
+demote_fixture legacy
+demote true
+if [ "$RC" = 0 ] && [ ! -e "$TMP/ovn-central" ]; then
+    pass "demote: --demote clears a legacy-only marker, which is what an older node carries"
+else
+    fail "demote: legacy-only marker gave rc=$RC, remaining [$(ls -A "$TMP" | grep 'ovn-central' | tr '\n' ' ')]"
+fi
+
+demote_fixture
+demote false
+if [ "$RC" = 0 ]; then
+    pass "demote: a node that was never a management node needs no flag"
+else
+    fail "demote: unmarked node without --demote gave rc=$RC, out=[$OUT]"
+fi
+
 if [ "$fails" -ne 0 ]; then
     echo "  setup-ovn.sh: $fails failure(s)"
     exit 1