diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 0049532da..dc3eb95ca 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -1,10 +1,10 @@ -# spinifex-unit-version: 3 +# spinifex-unit-version: 4 [Unit] Description=Spinifex Daemon (VM Orchestration) PartOf=spinifex.target -After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service -# Wants (not Requires): stay bootable when openvswitch-ipsec is absent -Wants=openvswitch-ipsec.service +After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service +# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet +Wants=openvswitch-ipsec.service spinifex-hostnet.service [Service] EnvironmentFile=-/etc/spinifex/telemetry.env diff --git a/build/systemd/spinifex-vpcd.service b/build/systemd/spinifex-vpcd.service index 3b5cfe510..5318024ff 100644 --- a/build/systemd/spinifex-vpcd.service +++ b/build/systemd/spinifex-vpcd.service @@ -1,9 +1,9 @@ -# spinifex-unit-version: 1 +# spinifex-unit-version: 2 [Unit] Description=Spinifex VPC Daemon (OVN Networking) PartOf=spinifex.target -After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service -Wants=spinifex-nats.service ovn-controller.service +After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service +Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service [Service] EnvironmentFile=-/etc/spinifex/telemetry.env diff --git a/build/systemd/spinifex.target b/build/systemd/spinifex.target index 7f1427296..90faac54e 100644 --- a/build/systemd/spinifex.target +++ b/build/systemd/spinifex.target @@ -1,9 +1,9 @@ -# spinifex-unit-version: 1 +# spinifex-unit-version: 2 [Unit] Description=Spinifex Infrastructure Platform After=network-online.target Wants=network-online.target -Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service +Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service [Install] WantedBy=multi-user.target diff --git a/scripts/README.md b/scripts/README.md index 4b5ccbb3d..85ae2a87d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,7 +7,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati | Script | What it does | |---|---| | `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. | -| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. | +| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services. | | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. | | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. | | `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. | @@ -25,6 +25,9 @@ Each probe prints its answer, or names every candidate it tried on stderr and re | `distro_ca_trust` | the trust-store update command and its anchor directory | | `distro_ovn_logrotate` | `/etc/logrotate.d/ovn-common` or `/etc/logrotate.d/ovn` | +`distro_missing_ovs_deps ...` is a check rather than a probe: it prints each OVS or OVN unit the given unit files depend on or order against that is not installed, and fails only when a file cannot be read. +`setup-ovn.sh` runs it against the installed Spinifex units and stops when one is missing, because systemd silently drops ordering against a unit that does not exist; a missing `openvswitch-ipsec.service` is only a warning. + `setup-ovn.sh`, `uninstall-spx.sh` and `dev-install.sh` source it from `lib/` beside themselves, then from `/usr/local/share/spinifex/lib/distro.sh`, where `setup.sh` installs it, and stop naming both paths when neither exists. `setup.sh` is piped from the install endpoint before the tarball is downloaded, and `install-node.sh --wipe` copies `node-reset.sh` alone to each host, so those two carry inline copies of the probes they need between `# --- BEGIN distro.sh probes ---` markers. `lib/distro_test.sh` fails when an inline copy differs from the library. diff --git a/scripts/lib/distro.sh b/scripts/lib/distro.sh index 398b4b12c..96a432a39 100644 --- a/scripts/lib/distro.sh +++ b/scripts/lib/distro.sh @@ -103,3 +103,33 @@ distro_ovn_logrotate() { echo "distro.sh: OVN logrotate: found neither /etc/logrotate.d/ovn-common nor /etc/logrotate.d/ovn; is OVN installed?" >&2 return 1 } + +# The OVS and OVN units the given unit files depend on or order against that +# are not installed, one per line; prints nothing when all are. Fails only +# when a unit file cannot be read. +distro_missing_ovs_deps() { + local file line dep missing="" + for file in "$@"; do + if [ ! -r "${DISTRO_ROOT:-}$file" ]; then + echo "distro.sh: OVS dependencies: cannot read $file" >&2 + return 1 + fi + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + After=* | Before=* | Wants=* | Requires=* | Requisite=* | BindsTo=* | PartOf=* | Upholds=*) ;; + *) continue ;; + esac + for dep in ${line#*=}; do + case "$dep" in + ovs* | ovn* | openvswitch*) ;; + *) continue ;; + esac + case " $missing " in + *" $dep "*) continue ;; + esac + distro_unit_exists "$dep" || missing="${missing:+$missing }$dep" + done + done <"${DISTRO_ROOT:-}$file" + done + for dep in $missing; do echo "$dep"; done +} diff --git a/scripts/lib/distro_test.sh b/scripts/lib/distro_test.sh index bf14ff620..fcbdb5004 100644 --- a/scripts/lib/distro_test.sh +++ b/scripts/lib/distro_test.sh @@ -27,11 +27,11 @@ unit() { mkdir -p "$ROOT/usr/lib/systemd/system" && : > "$ROOT/usr/lib/systemd/s file() { mkdir -p "$ROOT$(dirname "$1")" && : > "$ROOT$1"; } cmd() { printf '#!/bin/sh\nexit 0\n' > "$BIN/$1" && chmod +x "$BIN/$1"; } -# probe — runs one probe in bash against the current fixture, with only -# the stub dir on PATH. Sets OUT, ERR and RC. +# probe [arg...] — runs one probe in bash against the current fixture, +# with only the stub dir on PATH. Sets OUT, ERR and RC. probe() { RC=0 - OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; "$2"' _ "$DISTRO" "$1" 2>"$TMP/err") || RC=$? + OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; shift; "$@"' _ "$DISTRO" "$@" 2>"$TMP/err") || RC=$? ERR=$(cat "$TMP/err") } @@ -126,6 +126,38 @@ expect_fail "OVN logrotate: both fails naming both" /etc/logrotate.d/ovn-common fixture lr-none; probe distro_ovn_logrotate expect_fail "OVN logrotate: neither fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn +# -------------------------------------------------------------------------- +# OVS and OVN units the shipped Spinifex units name. Run against the real unit +# files, so a unit that names a Debian-only OVS unit fails on the EL fixture. +# -------------------------------------------------------------------------- +UNITS_SRC="$(dirname -- "$SCRIPTS_DIR")/build/systemd" +spinifex_units() { + mkdir -p "$ROOT/etc/systemd/system" + cp "$UNITS_SRC"/spinifex.target "$UNITS_SRC"/spinifex-*.service "$ROOT/etc/systemd/system/" +} +deps_probe() { + set -- + for f in "$ROOT"/etc/systemd/system/spinifex*; do set -- "$@" "${f#"$ROOT"}"; done + probe distro_missing_ovs_deps "$@" +} +el_ovs_units() { + for u in openvswitch.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service ovn-northd.service openvswitch-ipsec.service; do unit "$u"; done +} + +fixture deps-el; spinifex_units; el_ovs_units; deps_probe +expect_ok "OVS dependencies: every one the shipped units name is installed on EL" "" +fixture deps-no-ipsec; spinifex_units; el_ovs_units; rm "$ROOT/usr/lib/systemd/system/openvswitch-ipsec.service"; deps_probe +expect_ok "OVS dependencies: a missing unit is printed once" openvswitch-ipsec.service +fixture deps-debian-only; el_ovs_units +mkdir -p "$ROOT/etc/systemd/system" +printf '[Unit]\n# After=ovs-old.service\nAfter=openvswitch-switch.service ovn-controller.service\nWants=openvswitch-switch.service\n' >"$ROOT/etc/systemd/system/spinifex-x.service" +printf '[Unit]\nRequires=ovn-central.service' >"$ROOT/etc/systemd/system/spinifex-y.service" +deps_probe +expect_ok "OVS dependencies: Debian-only names are missing on EL, comments are not read" "openvswitch-switch.service +ovn-central.service" +fixture deps-unreadable; probe distro_missing_ovs_deps /etc/systemd/system/spinifex.target +expect_fail "OVS dependencies: an unreadable unit file fails naming it" "cannot read /etc/systemd/system/spinifex.target" + # -------------------------------------------------------------------------- # Inline copies. setup.sh and node-reset.sh cannot source the library, so they # carry copies between markers; each copied function must match the library's diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index f65784bc0..a1539b689 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -4,7 +4,7 @@ # # This script bootstraps a compute node for OVN-based VPC networking: # 1. Installs OVN/OVS packages (if not present) -# 2. Enables required services (openvswitch-switch, ovn-controller) +# 2. Enables required services (Open vSwitch, ovn-controller) # 3. Creates br-int with secure fail-mode # 4. Configures WAN bridge for public subnet uplink (auto-detected or manual) # 5. Configures OVS external_ids for OVN chassis identity @@ -588,19 +588,70 @@ ensure_clustered_db_storage() { # fresh SB and re-registers under whatever system-id it started with — which # on a renamed node is the old one, and that row then holds this node's encap # IP so the correctly-named chassis can never commit. Step 5 restarts it. - sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true + sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" ovn-controller 2>/dev/null || true sudo rm -f "${standalone[@]}" } +# set_root_service_user — sets ="root:root" in when +# the file exists. Returns 0 only when it changed the file. +set_root_service_user() { + local file=$1 var=$2 + [ -f "$file" ] || return 1 + grep -qx "$var=\"root:root\"" "$file" && return 1 + if grep -q "^$var=" "$file"; then + sudo sed -i "s|^$var=.*|$var=\"root:root\"|" "$file" + else + echo "$var=\"root:root\"" | sudo tee -a "$file" >/dev/null + fi + echo " $file: $var=\"root:root\"" +} + echo "" echo "Step 2: Enabling services..." -sudo systemctl enable openvswitch-switch -sudo systemctl start openvswitch-switch -echo " openvswitch-switch: started" +# Unit names differ by distro: the OVS umbrella, and whether the OVN databases +# have units of their own (Debian) or are started by ovn-northd.service (EL). +OVS_UNIT=$(distro_ovs_unit) || exit 1 +OVN_CENTRAL_LAYOUT="" +if [ "$MANAGEMENT" = true ] || distro_unit_exists ovn-central.service || distro_unit_exists ovn-northd.service; then + OVN_CENTRAL_LAYOUT=$(distro_ovn_central_layout) || exit 1 +fi +case "$OVN_CENTRAL_LAYOUT" in + split) + OVN_CENTRAL_UNIT=ovn-central.service + OVN_DB_UNITS=(ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service) + OVN_SERVER_UNITS=("${OVN_DB_UNITS[@]}" ovn-northd.service) + OVN_CENTRAL_UNITS=(ovn-central.service "${OVN_SERVER_UNITS[@]}") + ;; + lone-northd) + OVN_CENTRAL_UNIT=ovn-northd.service + OVN_DB_UNITS=(ovn-northd.service) + OVN_SERVER_UNITS=(ovn-northd.service) + OVN_CENTRAL_UNITS=(ovn-northd.service) + ;; +esac + +# OVS and OVN run as root, as on Debian: the service users reach the sockets +# through the spinifex group, a model tested only against root-owned daemons. +# EL ships non-root users and re-chowns the run dirs to them on every start. +OVS_USER_CHANGED=false +if set_root_service_user /etc/sysconfig/openvswitch OVS_USER_ID; then + OVS_USER_CHANGED=true +fi +# Takes effect below: ovn-northd and ovn-controller are restarted either way. +set_root_service_user /etc/sysconfig/ovn OVN_USER_ID || true + +sudo systemctl enable "$OVS_UNIT" +if [ "$OVS_USER_CHANGED" = true ]; then + # start is a no-op on a running OVS, which would keep its old user. + sudo systemctl restart "$OVS_UNIT" +else + sudo systemctl start "$OVS_UNIT" +fi +echo " ${OVS_UNIT%.service}: started" if [ "$MANAGEMENT" = true ]; then - sudo systemctl enable ovn-central + sudo systemctl enable "$OVN_CENTRAL_UNIT" # LAN-plane NB/SB client listener, added to the loopback one set below. # Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so @@ -664,10 +715,10 @@ ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ov EOF sudo systemctl daemon-reload - # The ovn-central aggregator is ExecStart=/bin/true, so restarting it - # won't restart the children — restart the per-DB units directly to - # pick up the new OVN_CTL_OPTS. - sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd + # Debian's ovn-central aggregator is ExecStart=/bin/true, so restarting + # it won't restart the children — restart the DB and northd units + # directly to pick up the new OVN_CTL_OPTS. + sudo systemctl restart "${OVN_SERVER_UNITS[@]}" echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)" else # Always write the file, even when there is nothing to put in it. This @@ -677,12 +728,12 @@ EOF # peer that no longer exists, which looks like a hung leader election. echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null echo " wrote /etc/default/ovn-central" - sudo systemctl start ovn-central + sudo systemctl start "$OVN_CENTRAL_UNIT" - # ovn-central is ExecStart=/bin/true; restarting it does not restart - # its children, so a re-run with changed options needs the per-DB units - # restarted directly to pick them up. - sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb + # Debian's ovn-central is ExecStart=/bin/true; restarting it does not + # restart its children, so a re-run with changed options needs the DB + # units restarted directly to pick them up. + sudo systemctl restart "${OVN_DB_UNITS[@]}" echo " ovn-central: started (NB DB + SB DB + ovn-northd)" if [ -n "$LAN_ADDR" ]; then echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR" @@ -825,10 +876,10 @@ else exit 1 fi - if systemctl is-enabled ovn-central >/dev/null 2>&1 || - systemctl is-active ovn-central >/dev/null 2>&1; then - sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true - sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true + if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 || + systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then + sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true + sudo systemctl disable "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true echo " ovn-central: stopped and disabled (compute node runs no database)" fi for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do @@ -1522,17 +1573,24 @@ echo " OVS/OVN sockets: 0660 root:spinifex (group-scoped, no world access)" # file is the old 0666 override, and skipping would leave that exposure in place. # openvswitch-ipsec is included because Step 10 restarts it after this sweep, # so its ctl socket would otherwise be the one file left at the shipped mode. -for unit in openvswitch-switch:/var/run/openvswitch/db.sock \ - "ovn-controller:/var/run/ovn/*.ctl" \ - "openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl"; do +# ovs-vswitchd starts after ovsdb-server and restarts with it (Requires=), so +# its hook sees both daemons' sockets and fires on a restart of either. +PERMS_UNITS=("ovs-vswitchd:/var/run/openvswitch/ovs-vswitchd.*.ctl" + "ovn-controller:/var/run/ovn/*.ctl" + "openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl") +# EL's ovn-northd.service starts the NB and SB databases as well as northd. +if [ "$OVN_CENTRAL_LAYOUT" = lone-northd ]; then + PERMS_UNITS+=("ovn-northd:/var/run/ovn/ovn-northd.*.ctl") +fi +for unit in "${PERMS_UNITS[@]}"; do UNIT="${unit%%:*}" WAIT_GLOB="${unit#*:}" OVERRIDE_DIR="/etc/systemd/system/${UNIT}.service.d" sudo mkdir -p "$OVERRIDE_DIR" # `ExecStartPost=-` so a failure here cannot stop the unit. This drop-in goes # into another package's service, and tightening a socket mode is hardening, - # not a liveness requirement — it must never be the reason openvswitch-switch - # or ovn-controller will not start. Without the dash, removing the helper + # not a liveness requirement — it must never be the reason OVS or + # ovn-controller will not start. Without the dash, removing the helper # (an uninstall, a partial re-run that has not reached this step yet) takes # OVS down across the whole host with `Failed at step EXEC`, which reads as # an OVS fault and is nothing of the sort. @@ -1583,7 +1641,7 @@ fi # to prevent log spam during those retries. echo "" echo "Step 10: Enabling OVN auto-start on boot..." -sudo systemctl enable openvswitch-switch 2>/dev/null || true +sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true sudo systemctl enable ovn-controller 2>/dev/null || true # ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's # enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops @@ -1596,9 +1654,30 @@ sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true # now that strongswan-starter is present. sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true -echo " openvswitch-switch: enabled on boot" -echo " ovn-controller: enabled on boot" -echo " openvswitch-ipsec: enabled on boot" +printf ' %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller: openvswitch-ipsec: + +# systemd silently drops ordering against a unit that is not installed, so a +# missing OVS or OVN unit would leave Spinifex starting unordered against OVS. +SPINIFEX_UNITS=() +for f in /etc/systemd/system/spinifex.target /etc/systemd/system/spinifex-*.service; do + [ -f "$f" ] && SPINIFEX_UNITS+=("$f") +done +if [ ${#SPINIFEX_UNITS[@]} -eq 0 ]; then + echo " Spinifex units: not installed yet — run setup.sh, then re-run this script" +else + MISSING_DEPS=$(distro_missing_ovs_deps "${SPINIFEX_UNITS[@]}") || exit 1 + DEPS_OK=true + for dep in $MISSING_DEPS; do + if [ "$dep" = openvswitch-ipsec.service ]; then + echo " WARNING: $dep is not installed; IPsec cannot be turned on" + else + echo "ERROR: the Spinifex units order against $dep, which is not installed" >&2 + DEPS_OK=false + fi + done + [ "$DEPS_OK" = true ] || exit 1 + echo " OVS/OVN units the Spinifex units name: installed" +fi # --- Step 11: Health check --- echo "" diff --git a/scripts/setup-ovn_test.sh b/scripts/setup-ovn_test.sh index 6dabebc24..712b8a49e 100755 --- a/scripts/setup-ovn_test.sh +++ b/scripts/setup-ovn_test.sh @@ -139,6 +139,41 @@ else fail "transit MAC disagrees: script=$script_mac go=$go_mac" fi +# -------------------------------------------------------------------------- +# OVS and OVN service users. +# -------------------------------------------------------------------------- +# EL runs OVS and OVN as non-root users and re-chowns their sockets on every +# start. The function is run on its own against scratch files, through a sudo +# that only runs its arguments. +awk '$0 == "set_root_service_user() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/user_fn.sh" +mkdir -p "$TMP/bin" +printf '#!/bin/sh\nexec "$@"\n' >"$TMP/bin/sudo" +chmod +x "$TMP/bin/sudo" + +# set_user — runs the function on for OVS_USER_ID; sets RC. +set_user() { + RC=0 + PATH="$TMP/bin:$PATH" bash -c '. "$1"; set_root_service_user "$2" OVS_USER_ID' _ "$TMP/user_fn.sh" "$1" >/dev/null || RC=$? +} + +set_user "$TMP/absent" +if [ "$RC" != 0 ] && [ ! -e "$TMP/absent" ]; then pass "service user: no file, no change"; else fail "service user: absent file gave rc=$RC or was created"; fi + +printf '# OVS_USER_ID="openvswitch:openvswitch"\nOPTIONS=""\nOVS_USER_ID="openvswitch:hugetlbfs"\n' >"$TMP/el" +set_user "$TMP/el" +want='# OVS_USER_ID="openvswitch:openvswitch" +OPTIONS="" +OVS_USER_ID="root:root"' +if [ "$RC" = 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: EL default replaced in place, comments and other lines kept"; else fail "service user: EL file rc=$RC: $(cat "$TMP/el")"; fi + +set_user "$TMP/el" +if [ "$RC" != 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: already root reports no change"; else fail "service user: second run rc=$RC: $(cat "$TMP/el")"; fi + +printf 'OPTIONS=""\n' >"$TMP/unset" +set_user "$TMP/unset" +if [ "$RC" = 0 ] && [ "$(cat "$TMP/unset")" = 'OPTIONS="" +OVS_USER_ID="root:root"' ]; then pass "service user: appended when the file does not set it"; else fail "service user: unset file rc=$RC: $(cat "$TMP/unset")"; fi + if [ "$fails" -ne 0 ]; then echo " setup-ovn.sh: $fails failure(s)" exit 1 diff --git a/scripts/uninstall-spx.sh b/scripts/uninstall-spx.sh index c62362084..8e55eb81c 100755 --- a/scripts/uninstall-spx.sh +++ b/scripts/uninstall-spx.sh @@ -307,12 +307,14 @@ rm_path /etc/systemd/system/spinifex-*.service \ # exactly that way on the first dev-oci round trip. log "removing drop-ins written into other units" rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \ + /etc/systemd/system/ovs-vswitchd.service.d/spinifex-perms.conf \ /etc/systemd/system/ovn-controller.service.d/spinifex-perms.conf \ /etc/systemd/system/ovn-controller.service.d/log-level.conf \ /etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \ + /etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \ /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf # Only if we emptied them — another package may own the same override dir. -for d in openvswitch-switch ovn-controller openvswitch-ipsec ovn-northd; do +for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do dir="/etc/systemd/system/${d}.service.d" if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then rm_path "$dir" diff --git a/spinifex/operator/host/systemd/units_gen.go b/spinifex/operator/host/systemd/units_gen.go index 914acc1aa..bf1570795 100644 --- a/spinifex/operator/host/systemd/units_gen.go +++ b/spinifex/operator/host/systemd/units_gen.go @@ -8,7 +8,7 @@ package systemd var Units = map[string]string{ "regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n", "spinifex-awsgw.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", - "spinifex-daemon.service": "# spinifex-unit-version: 3\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service\n# Wants (not Requires): stay bootable when openvswitch-ipsec is absent\nWants=openvswitch-ipsec.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-daemon.service": "# spinifex-unit-version: 4\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-firewall.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\nConditionPathExists=/etc/spinifex/firewall/peers.nft\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n", "spinifex-guests.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n", "spinifex-nats-watchdog.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n# Same 34s worst case bounds the start side: without this the unit inherits\n# oneshot's infinite default, so a genuinely hung run would sit in\n# activating(start) forever and could not be retriggered by its timer.\nTimeoutStartSec=120\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n", @@ -21,7 +21,7 @@ var Units = map[string]string{ "spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n", "spinifex-ui.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\nEnvironment=SPINIFEX_UI_OCHRE_ENABLED=false\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex-viperblock.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's\n# QEMU /proc entries (different uid) orphan nbdkit reaping must see.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", - "spinifex-vpcd.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", + "spinifex-vpcd.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service\nWants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n", "spinifex.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n", - "spinifex.target": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n", + "spinifex.target": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n", } diff --git a/spinifex/operator/host/systemd/units_invariants_test.go b/spinifex/operator/host/systemd/units_invariants_test.go index 65b9560cb..1e94e515b 100644 --- a/spinifex/operator/host/systemd/units_invariants_test.go +++ b/spinifex/operator/host/systemd/units_invariants_test.go @@ -423,6 +423,7 @@ var unitBodyHashes = map[string]map[int]string{ 1: "04b0f23b9cda322e4012d589acc196abe466b619dec5ac0d832897b9af2f926c", 2: "97a2d4c72967a9bdf9545ff74593de532e8469568bb717c9958d33b77e62d0ed", 3: "8059fe13a678b8cb2a383ae75c069560cf39adece9a9394e17781998c711cd57", + 4: "f66e1240cc5655f6a13f3b56ca282c73e2fe289f2e51d0ce624d14b90f12f838", }, "spinifex-firewall.service": {1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392"}, "spinifex-guests.slice": {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"}, @@ -449,9 +450,15 @@ var unitBodyHashes = map[string]map[int]string{ 1: "5c8cdd2004abf8e5725cbce0200565b9b29be7ab3210a4e0a2cdfe37ec5facb9", 2: "5d691bf5ce4a5a636d5dcc07f7bfa36d0da9eb0d7fb26079f89358ca0e5440cd", }, - "spinifex-vpcd.service": {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"}, - "spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"}, - "spinifex.target": {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"}, + "spinifex-vpcd.service": { + 1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965", + 2: "6ca01c8104ce0fc96ee4fdd8c0f2cd2ae907211b58b6f868fdc6a26f55a579f8", + }, + "spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"}, + "spinifex.target": { + 1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078", + 2: "0dc31d0e415277d10e4d90959f40d574f5c2319560299448002f701453737d5b", + }, } // TestUnitBodyPinnedToVersion asserts each unit's body hash matches the @@ -493,3 +500,58 @@ func stripVersionMarkerLine(content string) string { } return content } + +// crossDistroOVSUnits are the OVS and OVN units Debian and EL both ship under +// the same name. The umbrella differs (openvswitch-switch against openvswitch), +// and systemd silently drops ordering against a unit that does not exist. +var crossDistroOVSUnits = []string{ + "ovsdb-server.service", + "ovs-vswitchd.service", + "ovn-controller.service", + "ovn-northd.service", + "openvswitch-ipsec.service", +} + +// TestOVSDependenciesExistOnEveryDistro asserts every OVS or OVN unit a +// Spinifex unit depends on or orders against is one both distros ship. +func TestOVSDependenciesExistOnEveryDistro(t *testing.T) { + dir := unitsDir(t) + keys := []string{"After", "Before", "Wants", "Requires", "Requisite", "BindsTo", "PartOf", "Upholds"} + for _, name := range allUnitFiles(t, dir) { + u := readUnit(t, dir, name) + for line := range strings.SplitSeq(u, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok || !slices.Contains(keys, key) { + continue + } + for dep := range strings.FieldsSeq(value) { + if !strings.HasPrefix(dep, "ovs") && !strings.HasPrefix(dep, "ovn") && !strings.HasPrefix(dep, "openvswitch") { + continue + } + if !slices.Contains(crossDistroOVSUnits, dep) { + t.Errorf("%s: %s=%s is not shipped under that name on every distro; use one of %v", name, key, dep, crossDistroOVSUnits) + } + } + } + } +} + +// TestHostnetIsWantedNotRequired asserts the daemon and vpcd start after +// spinifex-hostnet without Requires=: hostnet is PartOf ovs-vswitchd, so a +// Requires= would restart both on every OVS restart or package upgrade. +func TestHostnetIsWantedNotRequired(t *testing.T) { + dir := unitsDir(t) + for _, name := range []string{"spinifex-daemon.service", "spinifex-vpcd.service"} { + u := readUnit(t, dir, name) + for _, key := range []string{"Wants", "After"} { + if !directiveContains(u, key, "spinifex-hostnet.service") { + t.Errorf("%s: %s= must include spinifex-hostnet.service", name, key) + } + } + for _, key := range []string{"Requires", "Requisite", "BindsTo", "PartOf"} { + if directiveContains(u, key, "spinifex-hostnet.service") { + t.Errorf("%s: %s=spinifex-hostnet.service would restart it with OVS; use Wants=", name, key) + } + } + } +}