mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
feat(security): add an nftables host firewall scoping cluster ports to peers (#769)
* feat(security): add an nftables host firewall scoping cluster ports to peers Nodes shipped no host firewall, so every cluster listener — OVN NB/SB, the formation port, NATS, predastore — was reachable from anything that could route to the node. On a single-NIC node the planes collapse onto the public address, which made that the default rather than the exception. setup.sh installs a policy and an apply helper but does not enable them: the peer set is unknown until the cluster's planes resolve, and a drop policy with an empty set would cut the first node that applied it out of its own formation. install-node.sh writes the peers and enables the unit after formation. The policy owns only table inet spinifex_filter and replaces it in one transaction. vpcd writes MASQUERADE and per-EIP FORWARD rules into the ip nat/filter tables and reinstalls them only on start, so a whole-ruleset flush — what Debian's nftables.service does on both start and stop — would break elastic IPs silently. INPUT only, for the same reason: a forward-hook drop here could not be rescued by an accept in the other table. Guest traffic keeps working: IMDS and VPC DNS terminate on ime-* ports in the host netns, and every guest agent reaches its control plane over the AWS gateway rather than NATS, so no cluster port needs to face the guests. * feat: reconcile the host firewall peer set from the daemon The peer set was written by install-node.sh and by cluster-bootstrap.yml. That missed two of the four formation paths (bm-bootstrap.sh and the ISO firstboot), duplicated the same logic in each, and went stale as soon as a node was added or replaced. host.ReconcileFirewall now owns it: every formation path starts the daemon, and it re-runs on each startup. A node's own config names its peers by their wan address only, so a set built from config alone drops every peer's NATS, OVSDB and formation traffic. The lan plane comes from the generated NATS cluster routes and the OVN remote lists, and the vpc plane from the Southbound Encap table; an empty encap answer is an error rather than a narrower set. ProtectSystem=full applies to the sudo child too, so the root helper could not write /etc/spinifex/firewall either — it is now in the daemon's ReadWritePaths, optional so a node installed before the firewall stage still starts. The apply helper gains set-peers and disable verbs, re-validates every address as a dotted quad with in-range octets, and rolls the peer file back if the ruleset will not load. * fix: retry the host firewall reconcile until every chassis registers The peer set is only safe to write once OVN reports an encap address for every node. On a fresh bootstrap the daemon routinely wins the race against ovn-controller, so a single startup attempt saw a partial chassis list and either wrote a set that dropped Geneve from the nodes missing from it, or failed outright and left the node with no policy until a restart. Gate on the full chassis count and schedule the reconcile instead of running it once: back off from 15s to 5m while it fails, then re-run every 5m so a node added or replaced after formation is picked up without a daemon restart. An unchanged peer set short-circuits before the helper runs. * test: cover the firewall chassis quorum gate in the multinode e2e suite A partial chassis list is the normal state early in a bootstrap, and writing a peer set from one produces a firewall that drops Geneve from every node missing from it. The gate that refuses that write had no live coverage: a 5-node bootstrap converges too fast for any daemon to observe a partial list, so the verification run only ever exercised the OVN-unreachable path. Reach the branch by inflating the expected count instead of breaking OVN: a phantom node in one node's config makes the cluster one chassis short, with nothing in the Southbound DB disturbed. Asserts the gate logs, that no peer file is written while it holds, that the loaded ruleset survives, and that the node recovers to a byte-identical peer set once the phantom is removed.
This commit is contained in:
Josh Siviour
+13
-2
@@ -101,8 +101,8 @@ while [[ $# -gt 0 ]]; do
|
||||
done <"$2"
|
||||
shift 2
|
||||
;;
|
||||
--smoke) RUN_SMOKE=true; shift ;;
|
||||
--wipe) WIPE=true; shift ;;
|
||||
--smoke) RUN_SMOKE=true; shift ;;
|
||||
--wipe) WIPE=true; shift ;;
|
||||
--yes|-y) ASSUME_YES=true; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) sed -n '2,49p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
@@ -596,6 +596,17 @@ for name in "${NODE_NAMES[@]}"; do
|
||||
Check 'sudo ovs-vsctl get Open_vSwitch . external_ids:system-id' on that host."
|
||||
done
|
||||
|
||||
# --- Host firewall ---------------------------------------------------------
|
||||
#
|
||||
# Not applied here. spinifex-daemon reconciles the peer sets from cluster
|
||||
# membership on every startup, so it also picks up nodes added or replaced after
|
||||
# formation, which a one-shot write from this script would not.
|
||||
|
||||
echo ""
|
||||
log "host firewall: reconciled by spinifex-daemon from cluster membership."
|
||||
log " Check with 'sudo nft list table inet spinifex_filter' on a node,"
|
||||
log " or disable it cluster-wide with network.firewall_enabled = false."
|
||||
|
||||
# The pool is the one thing the operator supplied by hand and the one thing
|
||||
# nothing else validates, so print what actually landed. dns_servers in
|
||||
# particular is detected rather than passed, and is wrong often enough to be
|
||||
|
||||
+187
-4
@@ -14,9 +14,9 @@
|
||||
# skip systemctl daemon-reload + enable, short-circuit setup_sudo.
|
||||
# VERBOSE Set to 1 to echo "[setup] <stage>" before each top-level step.
|
||||
# SETUP_STAGES Comma-separated subset of stages to run:
|
||||
# deps, aws, users, sudoers, files, directories,
|
||||
# env, systemd, logrotate, udev, fixown,
|
||||
# migrations
|
||||
# deps, aws, users, sudoers, firewall, files,
|
||||
# directories, env, systemd, logrotate, udev,
|
||||
# fixown, migrations
|
||||
# Unset = run every stage appropriate for the current mode.
|
||||
|
||||
set -e
|
||||
@@ -28,6 +28,14 @@ INSTALL_BASE_URL="${INSTALL_BASE_URL:-https://install.mulgadc.com}"
|
||||
ENDPOINT_SYSCTL_HELPER="/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl"
|
||||
IPSEC_STATE_HELPER="/usr/local/lib/spinifex/spinifex-set-ipsec-state"
|
||||
|
||||
# The firewall policy and the peer addresses it scopes cluster ports to. The
|
||||
# policy ships with the node; the peers are written by install-node.sh, which is
|
||||
# the only component that knows every node's resolved planes.
|
||||
FIREWALL_DIR="/etc/spinifex/firewall"
|
||||
FIREWALL_RULES="${FIREWALL_DIR}/spinifex.nft"
|
||||
FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
|
||||
FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
|
||||
|
||||
# --- Colors ---
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@@ -279,6 +287,175 @@ HELPER
|
||||
$SUDO chmod 0755 "${IPSEC_STATE_HELPER}"
|
||||
}
|
||||
|
||||
# The policy is written here rather than shipped in the tarball so the ISO chroot
|
||||
# and the ansible path get an identical file without a packaging step. Installed
|
||||
# but NOT enabled: the peer set is empty until install-node.sh resolves the
|
||||
# cluster's planes, and a drop policy with no peers would break formation.
|
||||
install_firewall() {
|
||||
stage "installing host firewall policy"
|
||||
info "Installing host firewall policy..."
|
||||
|
||||
$SUDO install -d -m 0755 "$FIREWALL_DIR" /usr/local/lib/spinifex
|
||||
$SUDO tee "$FIREWALL_RULES" > /dev/null << 'RULES'
|
||||
#!/usr/sbin/nft -f
|
||||
# Spinifex host firewall. Managed by scripts/setup.sh — edits are overwritten.
|
||||
#
|
||||
# Only `table inet spinifex_filter` is created, deleted and replaced. vpcd writes
|
||||
# MASQUERADE and per-EIP FORWARD ACCEPT rules into the `ip nat` and `ip filter`
|
||||
# tables and reinstalls them only when the service starts, so anything that
|
||||
# flushes the whole ruleset breaks elastic IPs silently until the next restart.
|
||||
#
|
||||
# INPUT only. nftables runs every table registered on a hook and any drop is
|
||||
# final, so a forward-hook policy here could not be rescued by vpcd's ACCEPTs in
|
||||
# the other table. OUTPUT is untouched: the IMDS reply path egresses under
|
||||
# per-tap policy routing.
|
||||
|
||||
include "/etc/spinifex/firewall/peers.nft"
|
||||
|
||||
# Create-then-delete so the replace is atomic and the delete cannot fail on a
|
||||
# first run. nft applies the whole file as one transaction or none of it.
|
||||
table inet spinifex_filter
|
||||
delete table inet spinifex_filter
|
||||
|
||||
table inet spinifex_filter {
|
||||
chain input {
|
||||
type filter hook input priority filter; policy drop;
|
||||
|
||||
# First, or every reply to a connection this node opened is dropped.
|
||||
ct state established,related accept
|
||||
ct state invalid drop
|
||||
iif lo accept
|
||||
|
||||
# Guest metadata and VPC DNS terminate on per-ENI OVS internal ports in
|
||||
# the host netns, so guest traffic to 169.254.169.254 and .253 arrives
|
||||
# here. Without this, cloud-init, instance-role credentials and all
|
||||
# guest DNS break. The ports exist only while instances run, so this
|
||||
# matches the prefix rather than an enumerated list.
|
||||
iifname "ime-*" accept
|
||||
|
||||
# PMTUD is not optional under a Geneve overlay.
|
||||
icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
|
||||
icmpv6 type { echo-request, destination-unreachable, packet-too-big, time-exceeded, parameter-problem, nd-neighbor-solicit, nd-neighbor-advert, nd-router-solicit, nd-router-advert } accept
|
||||
|
||||
# A broadcast DHCP reply does not reliably match an established
|
||||
# conntrack entry, so the uplink and external-pool leases need this.
|
||||
udp sport 67 udp dport 68 accept
|
||||
|
||||
# Public plane. 9999 is also how every guest agent (lb, eks, ecs, rds)
|
||||
# reaches its control plane over br-mgmt: they speak SigV4 to the AWS
|
||||
# gateway, never NATS, so no cluster port needs to face the guests.
|
||||
tcp dport { 22, 3000, 8443, 9999 } accept
|
||||
tcp dport 53 accept
|
||||
udp dport 53 accept
|
||||
|
||||
# Cluster plane, peer-scoped. On a single-NIC node the planes collapse
|
||||
# onto the public address, which is why these are peer addresses rather
|
||||
# than an interface or a CIDR.
|
||||
ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644 } accept
|
||||
ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
|
||||
|
||||
# Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
|
||||
# kernel UDP-tunnel socket, so encapsulated packets are delivered
|
||||
# locally and pass this hook before the OVS vport sees them.
|
||||
ip saddr $spinifex_encap_peers udp dport { 6081, 500, 4500 } accept
|
||||
ip saddr $spinifex_encap_peers meta l4proto esp accept
|
||||
|
||||
# Rate-limited so a scan cannot fill the journal. This is the only way
|
||||
# to tell a policy gap from an application fault after the fact.
|
||||
limit rate 5/minute burst 10 packets log prefix "spinifex-fw drop: " level info
|
||||
}
|
||||
}
|
||||
RULES
|
||||
$SUDO chmod 0644 "$FIREWALL_RULES"
|
||||
info " $FIREWALL_RULES"
|
||||
|
||||
$SUDO tee "$FIREWALL_APPLY" > /dev/null << 'APPLY'
|
||||
#!/bin/sh
|
||||
# Applies the spinifex host firewall. Fails without changing the ruleset when
|
||||
# the peer file is missing, so a node whose daemon has not yet resolved cluster
|
||||
# membership is left reachable rather than cut off from a cluster it cannot name.
|
||||
set -eu
|
||||
|
||||
RULES="/etc/spinifex/firewall/spinifex.nft"
|
||||
PEERS="/etc/spinifex/firewall/peers.nft"
|
||||
|
||||
[ -r "$RULES" ] || { echo "no firewall policy at $RULES" >&2; exit 1; }
|
||||
|
||||
# set-peers reads two comma-separated address lists on stdin (cluster peers,
|
||||
# then encap peers) and rewrites the peer file before applying. Every address is
|
||||
# re-validated here as a bare dotted quad: this runs as root under a NOPASSWD
|
||||
# grant, so a caller must not be able to inject nft syntax through it.
|
||||
# disable removes spinifex's own table and the peer file, leaving every other
|
||||
# table alone. Idempotent, so it is safe on a node that never had the policy.
|
||||
if [ "${1:-}" = "disable" ]; then
|
||||
nft delete table inet spinifex_filter 2>/dev/null || true
|
||||
rm -f "$PEERS"
|
||||
systemctl disable --now spinifex-firewall.service >/dev/null 2>&1 || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${1:-}" = "set-peers" ]; then
|
||||
read -r peers_in || peers_in=""
|
||||
read -r encap_in || encap_in=""
|
||||
|
||||
# Octet ranges are checked, not just the shape: nft rejects 10.0.0.999 and
|
||||
# the whole transaction with it, which would leave a peer file that fails on
|
||||
# every boot. A loop, not a pipeline — a rejection in a `while read` subshell
|
||||
# cannot fail the function.
|
||||
OCTET='(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
|
||||
validate() {
|
||||
_out=""
|
||||
for a in $(echo "$1" | tr ',' ' '); do
|
||||
echo "$a" | grep -Eq "^${OCTET}(\.${OCTET}){3}$" || {
|
||||
echo "address not permitted: $a" >&2
|
||||
return 2
|
||||
}
|
||||
if [ -z "$_out" ]; then _out="$a"; else _out="$_out, $a"; fi
|
||||
done
|
||||
printf '%s' "$_out"
|
||||
}
|
||||
|
||||
peers=$(validate "$peers_in") || exit 2
|
||||
encap=$(validate "$encap_in") || exit 2
|
||||
[ -n "$peers" ] && [ -n "$encap" ] || {
|
||||
echo "set-peers needs a non-empty peer and encap list" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
tmp=$(mktemp)
|
||||
bak=$(mktemp)
|
||||
printf '%s\n' \
|
||||
'# Managed by spinifex-daemon. Regenerated from cluster membership.' \
|
||||
"define spinifex_peers = { $peers }" \
|
||||
"define spinifex_encap_peers = { $encap }" > "$tmp"
|
||||
if [ -r "$PEERS" ]; then cp "$PEERS" "$bak"; fi
|
||||
install -m 0644 "$tmp" "$PEERS"
|
||||
|
||||
# Roll the peer file back if the ruleset will not load, so the boot-time
|
||||
# unit never inherits a file already known to fail.
|
||||
if ! nft -f "$RULES"; then
|
||||
if [ -s "$bak" ]; then install -m 0644 "$bak" "$PEERS"; else rm -f "$PEERS"; fi
|
||||
rm -f "$tmp" "$bak"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$tmp" "$bak"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -r "$PEERS" ]; then
|
||||
echo "no peer file at $PEERS: the daemon writes it once cluster membership" >&2
|
||||
echo "resolves. Ruleset left unchanged." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec nft -f "$RULES"
|
||||
APPLY
|
||||
$SUDO chown root:root "$FIREWALL_APPLY"
|
||||
$SUDO chmod 0755 "$FIREWALL_APPLY"
|
||||
info " $FIREWALL_APPLY"
|
||||
info "Host firewall installed (applied by spinifex-daemon once peers resolve)"
|
||||
}
|
||||
|
||||
install_sudoers() {
|
||||
stage "installing scoped sudoers rules"
|
||||
install_endpoint_sysctl_helper
|
||||
@@ -329,6 +506,7 @@ SUDOERS
|
||||
$SUDO tee -a /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
|
||||
spinifex-daemon ALL=(root) NOPASSWD: ${ENDPOINT_SYSCTL_HELPER}
|
||||
spinifex-daemon ALL=(root) NOPASSWD: ${IPSEC_STATE_HELPER}
|
||||
spinifex-daemon ALL=(root) NOPASSWD: ${FIREWALL_APPLY}
|
||||
SUDOERS
|
||||
$SUDO chmod 0440 /etc/sudoers.d/spinifex-network
|
||||
$SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
|
||||
@@ -350,7 +528,7 @@ libvirt-daemon-system libvirt-clients
|
||||
pciutils
|
||||
jq curl iproute2 netcat-openbsd wget unzip xz-utils file
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
|
||||
chrony"
|
||||
chrony nftables"
|
||||
|
||||
install_apt_deps() {
|
||||
stage "installing apt dependencies"
|
||||
@@ -810,6 +988,10 @@ EOF
|
||||
# takes effect once enabled) — without this the JetStream ENOSPC-latch
|
||||
# watchdog is inert and a full disk requires a manual restart forever.
|
||||
$SUDO systemctl enable --now spinifex-nats-watchdog.timer
|
||||
# Enabled, not started: its ConditionPathExists holds it inert until the
|
||||
# daemon writes a peer file, so this only decides that a node which has one
|
||||
# gets the policy back at boot, before any service opens a socket.
|
||||
$SUDO systemctl enable spinifex-firewall.service
|
||||
info "Systemd units installed and enabled (per-service users)"
|
||||
}
|
||||
|
||||
@@ -977,6 +1159,7 @@ main() {
|
||||
stage_enabled aws && install_aws_cli
|
||||
stage_enabled users && create_service_users
|
||||
stage_enabled sudoers && install_sudoers
|
||||
stage_enabled firewall && install_firewall
|
||||
stage_enabled files && install_files
|
||||
stage_enabled directories && create_directories
|
||||
stage_enabled env && install_systemd_env
|
||||
|
||||
Reference in New Issue
Block a user