feat(firewall): scope ssh to an operator allowlist in custom.nft (#881)

The generated policy accepted ssh from any source and setup.sh rewrote
spinifex.nft on every run, so there was nowhere to record an allowlist that
survived an upgrade.

custom.nft is created once, never rewritten, and kept across a node reset.
It holds a define rather than rules, so an operator supplies a value and
cannot write a rule that fails the ruleset. The default accepts every source,
which is what the policy already did.

scripts/node-reset.sh

@@ -70,9 +70,14 @@ $KEEP_DATA || WIPE_DIRS+=("$DATA_DIR")
 # keep_args adds the regular files worth keeping: the service helper scripts
 # and the env files the units load, at the top level of $1 only. A *.sh deeper
 # in the tree is cluster state, not an installed helper.
+#
+# firewall/custom.nft is kept too, one level down: it is an operator's SSH
+# allowlist, and losing it silently returns the node to accepting ssh from
+# every source. Parenthesised so the -o groups against the whole top-level arm.
 KEEP_TOP=()
 keep_args() {
-    KEEP_TOP=(-path "$1/*" ! -path "$1/*/*" '(' -name '*.sh' -o -name '*.env' ')')
+    KEEP_TOP=('(' -path "$1/*" ! -path "$1/*/*" '(' -name '*.sh' -o -name '*.env' ')' ')' \
+              -o -path "$1/firewall/custom.nft")
 }
 
 # Report what is at stake in figures rather than adjectives. An operator who
@@ -150,6 +155,12 @@ if [ -r "$ETC_DIR/firewall/mode" ]; then
     FIREWALL_MODE=$(sudo cat "$ETC_DIR/firewall/mode" 2>/dev/null | tr -d '[:space:]')
 fi
 
+# Named in the transcript because the risk runs the other way too: an allowlist
+# naming the old cluster's management network locks an operator out of the new one.
+if [ -e "$ETC_DIR/firewall/custom.nft" ]; then
+    log "keeping the operator ssh allowlist in $ETC_DIR/firewall/custom.nft"
+fi
+
 log "removing OVN databases"
 run sudo systemctl stop ovn-central 2>/dev/null || true
 run sudo systemctl stop ovn-controller 2>/dev/null || true

scripts/setup.sh

@@ -36,6 +36,7 @@ FIREWALL_RULES="${FIREWALL_DIR}/spinifex.nft"
 FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
 FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
 FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
+FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
 FIREWALL_MODE_FILE="${FIREWALL_DIR}/mode"
 FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
 
@@ -349,6 +350,7 @@ install_firewall() {
 include "/etc/spinifex/firewall/local.nft"
 include "/etc/spinifex/firewall/open-ports.nft"
 include "/etc/spinifex/firewall/peers.nft"
+include "/etc/spinifex/firewall/custom.nft"
 
 # Create-then-delete so the replace is atomic and the delete cannot fail on a
 # first run. nft applies the whole file as one transaction or none of it.
@@ -379,12 +381,16 @@ table inet spinifex_filter {
         # conntrack entry, so the uplink and external-pool leases need this.
         udp sport 67 udp dport 68 accept
 
+        # Scoped by $trusted_ssh_peers from custom.nft, which the installer
+        # creates once and never rewrites. That default accepts every source,
+        # so a node is no less reachable than before until an operator narrows it.
+        ip saddr $trusted_ssh_peers tcp dport $spinifex_ssh_ports accept
+
         # Public plane. 9999 is also how every guest agent (lb, eks, ecs, rds)
         # reaches its control plane over br-mgmt: they speak SigV4 to the AWS
         # gateway, never NATS, so no cluster port needs to face the guests.
         # 443 has no listener yet and is held for the nginx TLS edge, so the
         # rollout does not need a firewall change on every node.
-        tcp dport $spinifex_ssh_ports accept
         tcp dport { 443, 3000, 8443, 9999 } accept
 
         # Transiently open to any source, for cluster formation. A node dialling
@@ -446,6 +452,31 @@ OPENPORTS
     $SUDO chmod 0644 "$FIREWALL_OPEN"
     info "  $FIREWALL_OPEN (closed)"
 
+    # Operator-owned, so it is created once and never rewritten: an upgrade, a
+    # re-install and a node reset all leave an edited file alone. Never write it
+    # empty — an undefined variable fails the whole ruleset, not just its rule.
+    if [ ! -e "$FIREWALL_CUSTOM" ]; then
+        $SUDO tee "$FIREWALL_CUSTOM" > /dev/null << 'CUSTOM'
+# Operator-owned host firewall settings. setup.sh creates this file once and
+# never overwrites it, so edits here survive upgrades, re-installs and resets.
+#
+# Check a change without applying it, then apply it:
+#   sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
+#   sudo nft -f  /etc/spinifex/firewall/spinifex.nft
+#
+# Check spinifex.nft, not this file: on its own this parses as a bare define
+# and proves nothing, because the rule that uses the variable lives there.
+#
+# Sources allowed to reach SSH. 0.0.0.0/0 accepts from anywhere; narrow it to
+# your management networks. IPv4 only — sshd reached over IPv6 is not matched.
+define trusted_ssh_peers = { 0.0.0.0/0 }
+CUSTOM
+        $SUDO chmod 0644 "$FIREWALL_CUSTOM"
+        info "  $FIREWALL_CUSTOM (created — ssh open to every source)"
+    else
+        info "  $FIREWALL_CUSTOM (already present, left unchanged)"
+    fi
+
     # Armed or merely installed. The daemon reads this when the config carries no
     # explicit firewall_enabled, which is the normal case on both install paths.
     printf '%s\n' "$INSTALL_SPINIFEX_FIREWALL" | $SUDO tee "$FIREWALL_MODE_FILE" > /dev/null
@@ -583,6 +614,14 @@ APPLY
     $SUDO chmod 0755 "$FIREWALL_APPLY"
     info "  $FIREWALL_APPLY"
 
+    # Say which state ssh is in either way. A node whose allowlist was never
+    # narrowed reads as configured otherwise, and that is the whole risk here.
+    _ssh_srcs="$(grep -o '{[^}]*}' "$FIREWALL_CUSTOM" 2>/dev/null | tr -d '{}' | tr -s ' ')"
+    case "$_ssh_srcs" in
+        *0.0.0.0/0*) info "  ssh (${_ssh_ports}) accepted from ANY source — narrow \$trusted_ssh_peers in $FIREWALL_CUSTOM" ;;
+        ?*) info "  ssh (${_ssh_ports}) accepted from:${_ssh_srcs}" ;;
+    esac
+
     if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
         info "Host firewall installed and armed (applied by spinifex-daemon once peers resolve)"
     else