mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-23 20:57:02 +00:00
feat: order Spinifex units against OVS names every distro ships, and probe OVS/OVN units in setup-ovn
spinifex.target, the daemon and vpcd now order against ovsdb-server, ovs-vswitchd and ovn-controller instead of Debian's openvswitch-switch, which EL does not have, and want spinifex-hostnet without requiring it. A unit test fails when a unit names an OVS or OVN unit outside that set. setup-ovn.sh takes the OVS umbrella and the OVN central units from the distro.sh probes, runs OVS and OVN as root where /etc/sysconfig sets their users, and hooks the socket-permission helper onto ovs-vswitchd (and EL's ovn-northd), so a standalone restart of either OVS daemon keeps the sockets usable by the service users. It stops when an OVS or OVN unit the installed Spinifex units name is not installed.
This commit is contained in:
Julian Sommer
@@ -1,10 +1,10 @@
|
||||
# spinifex-unit-version: 3
|
||||
# spinifex-unit-version: 4
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
|
||||
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
|
||||
Wants=openvswitch-ipsec.service
|
||||
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
|
||||
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
|
||||
Wants=openvswitch-ipsec.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex VPC Daemon (OVN Networking)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service ovn-controller.service
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
|
||||
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex Infrastructure Platform
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
+4
-1
@@ -7,7 +7,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
|
||||
| `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
|
||||
@@ -25,6 +25,9 @@ Each probe prints its answer, or names every candidate it tried on stderr and re
|
||||
| `distro_ca_trust` | the trust-store update command and its anchor directory |
|
||||
| `distro_ovn_logrotate` | `/etc/logrotate.d/ovn-common` or `/etc/logrotate.d/ovn` |
|
||||
|
||||
`distro_missing_ovs_deps <unit-file>...` is a check rather than a probe: it prints each OVS or OVN unit the given unit files depend on or order against that is not installed, and fails only when a file cannot be read.
|
||||
`setup-ovn.sh` runs it against the installed Spinifex units and stops when one is missing, because systemd silently drops ordering against a unit that does not exist; a missing `openvswitch-ipsec.service` is only a warning.
|
||||
|
||||
`setup-ovn.sh`, `uninstall-spx.sh` and `dev-install.sh` source it from `lib/` beside themselves, then from `/usr/local/share/spinifex/lib/distro.sh`, where `setup.sh` installs it, and stop naming both paths when neither exists.
|
||||
`setup.sh` is piped from the install endpoint before the tarball is downloaded, and `install-node.sh --wipe` copies `node-reset.sh` alone to each host, so those two carry inline copies of the probes they need between `# --- BEGIN distro.sh probes ---` markers.
|
||||
`lib/distro_test.sh` fails when an inline copy differs from the library.
|
||||
|
||||
@@ -103,3 +103,33 @@ distro_ovn_logrotate() {
|
||||
echo "distro.sh: OVN logrotate: found neither /etc/logrotate.d/ovn-common nor /etc/logrotate.d/ovn; is OVN installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS and OVN units the given unit files depend on or order against that
|
||||
# are not installed, one per line; prints nothing when all are. Fails only
|
||||
# when a unit file cannot be read.
|
||||
distro_missing_ovs_deps() {
|
||||
local file line dep missing=""
|
||||
for file in "$@"; do
|
||||
if [ ! -r "${DISTRO_ROOT:-}$file" ]; then
|
||||
echo "distro.sh: OVS dependencies: cannot read $file" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
After=* | Before=* | Wants=* | Requires=* | Requisite=* | BindsTo=* | PartOf=* | Upholds=*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
for dep in ${line#*=}; do
|
||||
case "$dep" in
|
||||
ovs* | ovn* | openvswitch*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
case " $missing " in
|
||||
*" $dep "*) continue ;;
|
||||
esac
|
||||
distro_unit_exists "$dep" || missing="${missing:+$missing }$dep"
|
||||
done
|
||||
done <"${DISTRO_ROOT:-}$file"
|
||||
done
|
||||
for dep in $missing; do echo "$dep"; done
|
||||
}
|
||||
|
||||
@@ -27,11 +27,11 @@ unit() { mkdir -p "$ROOT/usr/lib/systemd/system" && : > "$ROOT/usr/lib/systemd/s
|
||||
file() { mkdir -p "$ROOT$(dirname "$1")" && : > "$ROOT$1"; }
|
||||
cmd() { printf '#!/bin/sh\nexit 0\n' > "$BIN/$1" && chmod +x "$BIN/$1"; }
|
||||
|
||||
# probe <fn> — runs one probe in bash against the current fixture, with only
|
||||
# the stub dir on PATH. Sets OUT, ERR and RC.
|
||||
# probe <fn> [arg...] — runs one probe in bash against the current fixture,
|
||||
# with only the stub dir on PATH. Sets OUT, ERR and RC.
|
||||
probe() {
|
||||
RC=0
|
||||
OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; "$2"' _ "$DISTRO" "$1" 2>"$TMP/err") || RC=$?
|
||||
OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; shift; "$@"' _ "$DISTRO" "$@" 2>"$TMP/err") || RC=$?
|
||||
ERR=$(cat "$TMP/err")
|
||||
}
|
||||
|
||||
@@ -126,6 +126,38 @@ expect_fail "OVN logrotate: both fails naming both" /etc/logrotate.d/ovn-common
|
||||
fixture lr-none; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: neither fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN units the shipped Spinifex units name. Run against the real unit
|
||||
# files, so a unit that names a Debian-only OVS unit fails on the EL fixture.
|
||||
# --------------------------------------------------------------------------
|
||||
UNITS_SRC="$(dirname -- "$SCRIPTS_DIR")/build/systemd"
|
||||
spinifex_units() {
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
cp "$UNITS_SRC"/spinifex.target "$UNITS_SRC"/spinifex-*.service "$ROOT/etc/systemd/system/"
|
||||
}
|
||||
deps_probe() {
|
||||
set --
|
||||
for f in "$ROOT"/etc/systemd/system/spinifex*; do set -- "$@" "${f#"$ROOT"}"; done
|
||||
probe distro_missing_ovs_deps "$@"
|
||||
}
|
||||
el_ovs_units() {
|
||||
for u in openvswitch.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service ovn-northd.service openvswitch-ipsec.service; do unit "$u"; done
|
||||
}
|
||||
|
||||
fixture deps-el; spinifex_units; el_ovs_units; deps_probe
|
||||
expect_ok "OVS dependencies: every one the shipped units name is installed on EL" ""
|
||||
fixture deps-no-ipsec; spinifex_units; el_ovs_units; rm "$ROOT/usr/lib/systemd/system/openvswitch-ipsec.service"; deps_probe
|
||||
expect_ok "OVS dependencies: a missing unit is printed once" openvswitch-ipsec.service
|
||||
fixture deps-debian-only; el_ovs_units
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
printf '[Unit]\n# After=ovs-old.service\nAfter=openvswitch-switch.service ovn-controller.service\nWants=openvswitch-switch.service\n' >"$ROOT/etc/systemd/system/spinifex-x.service"
|
||||
printf '[Unit]\nRequires=ovn-central.service' >"$ROOT/etc/systemd/system/spinifex-y.service"
|
||||
deps_probe
|
||||
expect_ok "OVS dependencies: Debian-only names are missing on EL, comments are not read" "openvswitch-switch.service
|
||||
ovn-central.service"
|
||||
fixture deps-unreadable; probe distro_missing_ovs_deps /etc/systemd/system/spinifex.target
|
||||
expect_fail "OVS dependencies: an unreadable unit file fails naming it" "cannot read /etc/systemd/system/spinifex.target"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline copies. setup.sh and node-reset.sh cannot source the library, so they
|
||||
# carry copies between markers; each copied function must match the library's
|
||||
|
||||
+107
-28
@@ -4,7 +4,7 @@
|
||||
#
|
||||
# This script bootstraps a compute node for OVN-based VPC networking:
|
||||
# 1. Installs OVN/OVS packages (if not present)
|
||||
# 2. Enables required services (openvswitch-switch, ovn-controller)
|
||||
# 2. Enables required services (Open vSwitch, ovn-controller)
|
||||
# 3. Creates br-int with secure fail-mode
|
||||
# 4. Configures WAN bridge for public subnet uplink (auto-detected or manual)
|
||||
# 5. Configures OVS external_ids for OVN chassis identity
|
||||
@@ -588,19 +588,70 @@ ensure_clustered_db_storage() {
|
||||
# fresh SB and re-registers under whatever system-id it started with — which
|
||||
# on a renamed node is the old one, and that row then holds this node's encap
|
||||
# IP so the correctly-named chassis can never commit. Step 5 restarts it.
|
||||
sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" ovn-controller 2>/dev/null || true
|
||||
sudo rm -f "${standalone[@]}"
|
||||
}
|
||||
|
||||
# set_root_service_user <file> <var> — sets <var>="root:root" in <file> when
|
||||
# the file exists. Returns 0 only when it changed the file.
|
||||
set_root_service_user() {
|
||||
local file=$1 var=$2
|
||||
[ -f "$file" ] || return 1
|
||||
grep -qx "$var=\"root:root\"" "$file" && return 1
|
||||
if grep -q "^$var=" "$file"; then
|
||||
sudo sed -i "s|^$var=.*|$var=\"root:root\"|" "$file"
|
||||
else
|
||||
echo "$var=\"root:root\"" | sudo tee -a "$file" >/dev/null
|
||||
fi
|
||||
echo " $file: $var=\"root:root\""
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Enabling services..."
|
||||
|
||||
sudo systemctl enable openvswitch-switch
|
||||
sudo systemctl start openvswitch-switch
|
||||
echo " openvswitch-switch: started"
|
||||
# Unit names differ by distro: the OVS umbrella, and whether the OVN databases
|
||||
# have units of their own (Debian) or are started by ovn-northd.service (EL).
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
OVN_CENTRAL_LAYOUT=""
|
||||
if [ "$MANAGEMENT" = true ] || distro_unit_exists ovn-central.service || distro_unit_exists ovn-northd.service; then
|
||||
OVN_CENTRAL_LAYOUT=$(distro_ovn_central_layout) || exit 1
|
||||
fi
|
||||
case "$OVN_CENTRAL_LAYOUT" in
|
||||
split)
|
||||
OVN_CENTRAL_UNIT=ovn-central.service
|
||||
OVN_DB_UNITS=(ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service)
|
||||
OVN_SERVER_UNITS=("${OVN_DB_UNITS[@]}" ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-central.service "${OVN_SERVER_UNITS[@]}")
|
||||
;;
|
||||
lone-northd)
|
||||
OVN_CENTRAL_UNIT=ovn-northd.service
|
||||
OVN_DB_UNITS=(ovn-northd.service)
|
||||
OVN_SERVER_UNITS=(ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-northd.service)
|
||||
;;
|
||||
esac
|
||||
|
||||
# OVS and OVN run as root, as on Debian: the service users reach the sockets
|
||||
# through the spinifex group, a model tested only against root-owned daemons.
|
||||
# EL ships non-root users and re-chowns the run dirs to them on every start.
|
||||
OVS_USER_CHANGED=false
|
||||
if set_root_service_user /etc/sysconfig/openvswitch OVS_USER_ID; then
|
||||
OVS_USER_CHANGED=true
|
||||
fi
|
||||
# Takes effect below: ovn-northd and ovn-controller are restarted either way.
|
||||
set_root_service_user /etc/sysconfig/ovn OVN_USER_ID || true
|
||||
|
||||
sudo systemctl enable "$OVS_UNIT"
|
||||
if [ "$OVS_USER_CHANGED" = true ]; then
|
||||
# start is a no-op on a running OVS, which would keep its old user.
|
||||
sudo systemctl restart "$OVS_UNIT"
|
||||
else
|
||||
sudo systemctl start "$OVS_UNIT"
|
||||
fi
|
||||
echo " ${OVS_UNIT%.service}: started"
|
||||
|
||||
if [ "$MANAGEMENT" = true ]; then
|
||||
sudo systemctl enable ovn-central
|
||||
sudo systemctl enable "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# LAN-plane NB/SB client listener, added to the loopback one set below.
|
||||
# Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so
|
||||
@@ -664,10 +715,10 @@ ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ov
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
# The ovn-central aggregator is ExecStart=/bin/true, so restarting it
|
||||
# won't restart the children — restart the per-DB units directly to
|
||||
# pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd
|
||||
# Debian's ovn-central aggregator is ExecStart=/bin/true, so restarting
|
||||
# it won't restart the children — restart the DB and northd units
|
||||
# directly to pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart "${OVN_SERVER_UNITS[@]}"
|
||||
echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)"
|
||||
else
|
||||
# Always write the file, even when there is nothing to put in it. This
|
||||
@@ -677,12 +728,12 @@ EOF
|
||||
# peer that no longer exists, which looks like a hung leader election.
|
||||
echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
sudo systemctl start ovn-central
|
||||
sudo systemctl start "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# ovn-central is ExecStart=/bin/true; restarting it does not restart
|
||||
# its children, so a re-run with changed options needs the per-DB units
|
||||
# restarted directly to pick them up.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb
|
||||
# Debian's ovn-central is ExecStart=/bin/true; restarting it does not
|
||||
# restart its children, so a re-run with changed options needs the DB
|
||||
# units restarted directly to pick them up.
|
||||
sudo systemctl restart "${OVN_DB_UNITS[@]}"
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
@@ -825,10 +876,10 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if systemctl is-enabled ovn-central >/dev/null 2>&1 ||
|
||||
systemctl is-active ovn-central >/dev/null 2>&1; then
|
||||
sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 ||
|
||||
systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
sudo systemctl disable "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
echo " ovn-central: stopped and disabled (compute node runs no database)"
|
||||
fi
|
||||
for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
|
||||
@@ -1522,17 +1573,24 @@ echo " OVS/OVN sockets: 0660 root:spinifex (group-scoped, no world access)"
|
||||
# file is the old 0666 override, and skipping would leave that exposure in place.
|
||||
# openvswitch-ipsec is included because Step 10 restarts it after this sweep,
|
||||
# so its ctl socket would otherwise be the one file left at the shipped mode.
|
||||
for unit in openvswitch-switch:/var/run/openvswitch/db.sock \
|
||||
"ovn-controller:/var/run/ovn/*.ctl" \
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl"; do
|
||||
# ovs-vswitchd starts after ovsdb-server and restarts with it (Requires=), so
|
||||
# its hook sees both daemons' sockets and fires on a restart of either.
|
||||
PERMS_UNITS=("ovs-vswitchd:/var/run/openvswitch/ovs-vswitchd.*.ctl"
|
||||
"ovn-controller:/var/run/ovn/*.ctl"
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl")
|
||||
# EL's ovn-northd.service starts the NB and SB databases as well as northd.
|
||||
if [ "$OVN_CENTRAL_LAYOUT" = lone-northd ]; then
|
||||
PERMS_UNITS+=("ovn-northd:/var/run/ovn/ovn-northd.*.ctl")
|
||||
fi
|
||||
for unit in "${PERMS_UNITS[@]}"; do
|
||||
UNIT="${unit%%:*}"
|
||||
WAIT_GLOB="${unit#*:}"
|
||||
OVERRIDE_DIR="/etc/systemd/system/${UNIT}.service.d"
|
||||
sudo mkdir -p "$OVERRIDE_DIR"
|
||||
# `ExecStartPost=-` so a failure here cannot stop the unit. This drop-in goes
|
||||
# into another package's service, and tightening a socket mode is hardening,
|
||||
# not a liveness requirement — it must never be the reason openvswitch-switch
|
||||
# or ovn-controller will not start. Without the dash, removing the helper
|
||||
# not a liveness requirement — it must never be the reason OVS or
|
||||
# ovn-controller will not start. Without the dash, removing the helper
|
||||
# (an uninstall, a partial re-run that has not reached this step yet) takes
|
||||
# OVS down across the whole host with `Failed at step EXEC`, which reads as
|
||||
# an OVS fault and is nothing of the sort.
|
||||
@@ -1583,7 +1641,7 @@ fi
|
||||
# to prevent log spam during those retries.
|
||||
echo ""
|
||||
echo "Step 10: Enabling OVN auto-start on boot..."
|
||||
sudo systemctl enable openvswitch-switch 2>/dev/null || true
|
||||
sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true
|
||||
sudo systemctl enable ovn-controller 2>/dev/null || true
|
||||
# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
|
||||
# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
|
||||
@@ -1596,9 +1654,30 @@ sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
|
||||
# now that strongswan-starter is present.
|
||||
sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
|
||||
sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
|
||||
echo " openvswitch-switch: enabled on boot"
|
||||
echo " ovn-controller: enabled on boot"
|
||||
echo " openvswitch-ipsec: enabled on boot"
|
||||
printf ' %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller: openvswitch-ipsec:
|
||||
|
||||
# systemd silently drops ordering against a unit that is not installed, so a
|
||||
# missing OVS or OVN unit would leave Spinifex starting unordered against OVS.
|
||||
SPINIFEX_UNITS=()
|
||||
for f in /etc/systemd/system/spinifex.target /etc/systemd/system/spinifex-*.service; do
|
||||
[ -f "$f" ] && SPINIFEX_UNITS+=("$f")
|
||||
done
|
||||
if [ ${#SPINIFEX_UNITS[@]} -eq 0 ]; then
|
||||
echo " Spinifex units: not installed yet — run setup.sh, then re-run this script"
|
||||
else
|
||||
MISSING_DEPS=$(distro_missing_ovs_deps "${SPINIFEX_UNITS[@]}") || exit 1
|
||||
DEPS_OK=true
|
||||
for dep in $MISSING_DEPS; do
|
||||
if [ "$dep" = openvswitch-ipsec.service ]; then
|
||||
echo " WARNING: $dep is not installed; IPsec cannot be turned on"
|
||||
else
|
||||
echo "ERROR: the Spinifex units order against $dep, which is not installed" >&2
|
||||
DEPS_OK=false
|
||||
fi
|
||||
done
|
||||
[ "$DEPS_OK" = true ] || exit 1
|
||||
echo " OVS/OVN units the Spinifex units name: installed"
|
||||
fi
|
||||
|
||||
# --- Step 11: Health check ---
|
||||
echo ""
|
||||
|
||||
@@ -139,6 +139,41 @@ else
|
||||
fail "transit MAC disagrees: script=$script_mac go=$go_mac"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN service users.
|
||||
# --------------------------------------------------------------------------
|
||||
# EL runs OVS and OVN as non-root users and re-chowns their sockets on every
|
||||
# start. The function is run on its own against scratch files, through a sudo
|
||||
# that only runs its arguments.
|
||||
awk '$0 == "set_root_service_user() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/user_fn.sh"
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\nexec "$@"\n' >"$TMP/bin/sudo"
|
||||
chmod +x "$TMP/bin/sudo"
|
||||
|
||||
# set_user <file> — runs the function on <file> for OVS_USER_ID; sets RC.
|
||||
set_user() {
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" bash -c '. "$1"; set_root_service_user "$2" OVS_USER_ID' _ "$TMP/user_fn.sh" "$1" >/dev/null || RC=$?
|
||||
}
|
||||
|
||||
set_user "$TMP/absent"
|
||||
if [ "$RC" != 0 ] && [ ! -e "$TMP/absent" ]; then pass "service user: no file, no change"; else fail "service user: absent file gave rc=$RC or was created"; fi
|
||||
|
||||
printf '# OVS_USER_ID="openvswitch:openvswitch"\nOPTIONS=""\nOVS_USER_ID="openvswitch:hugetlbfs"\n' >"$TMP/el"
|
||||
set_user "$TMP/el"
|
||||
want='# OVS_USER_ID="openvswitch:openvswitch"
|
||||
OPTIONS=""
|
||||
OVS_USER_ID="root:root"'
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: EL default replaced in place, comments and other lines kept"; else fail "service user: EL file rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
set_user "$TMP/el"
|
||||
if [ "$RC" != 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: already root reports no change"; else fail "service user: second run rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
printf 'OPTIONS=""\n' >"$TMP/unset"
|
||||
set_user "$TMP/unset"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/unset")" = 'OPTIONS=""
|
||||
OVS_USER_ID="root:root"' ]; then pass "service user: appended when the file does not set it"; else fail "service user: unset file rc=$RC: $(cat "$TMP/unset")"; fi
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " setup-ovn.sh: $fails failure(s)"
|
||||
exit 1
|
||||
|
||||
@@ -307,12 +307,14 @@ rm_path /etc/systemd/system/spinifex-*.service \
|
||||
# exactly that way on the first dev-oci round trip.
|
||||
log "removing drop-ins written into other units"
|
||||
rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovs-vswitchd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf \
|
||||
/etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf
|
||||
# Only if we emptied them — another package may own the same override dir.
|
||||
for d in openvswitch-switch ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
dir="/etc/systemd/system/${d}.service.d"
|
||||
if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then
|
||||
rm_path "$dir"
|
||||
|
||||
@@ -8,7 +8,7 @@ package systemd
|
||||
var Units = map[string]string{
|
||||
"regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
"spinifex-awsgw.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-daemon.service": "# spinifex-unit-version: 3\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service\n# Wants (not Requires): stay bootable when openvswitch-ipsec is absent\nWants=openvswitch-ipsec.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-daemon.service": "# spinifex-unit-version: 4\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-firewall.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\nConditionPathExists=/etc/spinifex/firewall/peers.nft\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n",
|
||||
"spinifex-guests.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n",
|
||||
"spinifex-nats-watchdog.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n# Same 34s worst case bounds the start side: without this the unit inherits\n# oneshot's infinite default, so a genuinely hung run would sit in\n# activating(start) forever and could not be retriggered by its timer.\nTimeoutStartSec=120\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n",
|
||||
@@ -21,7 +21,7 @@ var Units = map[string]string{
|
||||
"spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n",
|
||||
"spinifex-ui.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\nEnvironment=SPINIFEX_UI_OCHRE_ENABLED=false\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-viperblock.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's\n# QEMU /proc entries (different uid) orphan nbdkit reaping must see.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-vpcd.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-vpcd.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service\nWants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n",
|
||||
"spinifex.target": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
"spinifex.target": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
}
|
||||
|
||||
@@ -423,6 +423,7 @@ var unitBodyHashes = map[string]map[int]string{
|
||||
1: "04b0f23b9cda322e4012d589acc196abe466b619dec5ac0d832897b9af2f926c",
|
||||
2: "97a2d4c72967a9bdf9545ff74593de532e8469568bb717c9958d33b77e62d0ed",
|
||||
3: "8059fe13a678b8cb2a383ae75c069560cf39adece9a9394e17781998c711cd57",
|
||||
4: "f66e1240cc5655f6a13f3b56ca282c73e2fe289f2e51d0ce624d14b90f12f838",
|
||||
},
|
||||
"spinifex-firewall.service": {1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392"},
|
||||
"spinifex-guests.slice": {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"},
|
||||
@@ -449,9 +450,15 @@ var unitBodyHashes = map[string]map[int]string{
|
||||
1: "5c8cdd2004abf8e5725cbce0200565b9b29be7ab3210a4e0a2cdfe37ec5facb9",
|
||||
2: "5d691bf5ce4a5a636d5dcc07f7bfa36d0da9eb0d7fb26079f89358ca0e5440cd",
|
||||
},
|
||||
"spinifex-vpcd.service": {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"},
|
||||
"spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
|
||||
"spinifex.target": {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"},
|
||||
"spinifex-vpcd.service": {
|
||||
1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965",
|
||||
2: "6ca01c8104ce0fc96ee4fdd8c0f2cd2ae907211b58b6f868fdc6a26f55a579f8",
|
||||
},
|
||||
"spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
|
||||
"spinifex.target": {
|
||||
1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078",
|
||||
2: "0dc31d0e415277d10e4d90959f40d574f5c2319560299448002f701453737d5b",
|
||||
},
|
||||
}
|
||||
|
||||
// TestUnitBodyPinnedToVersion asserts each unit's body hash matches the
|
||||
@@ -493,3 +500,58 @@ func stripVersionMarkerLine(content string) string {
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// crossDistroOVSUnits are the OVS and OVN units Debian and EL both ship under
|
||||
// the same name. The umbrella differs (openvswitch-switch against openvswitch),
|
||||
// and systemd silently drops ordering against a unit that does not exist.
|
||||
var crossDistroOVSUnits = []string{
|
||||
"ovsdb-server.service",
|
||||
"ovs-vswitchd.service",
|
||||
"ovn-controller.service",
|
||||
"ovn-northd.service",
|
||||
"openvswitch-ipsec.service",
|
||||
}
|
||||
|
||||
// TestOVSDependenciesExistOnEveryDistro asserts every OVS or OVN unit a
|
||||
// Spinifex unit depends on or orders against is one both distros ship.
|
||||
func TestOVSDependenciesExistOnEveryDistro(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
keys := []string{"After", "Before", "Wants", "Requires", "Requisite", "BindsTo", "PartOf", "Upholds"}
|
||||
for _, name := range allUnitFiles(t, dir) {
|
||||
u := readUnit(t, dir, name)
|
||||
for line := range strings.SplitSeq(u, "\n") {
|
||||
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !ok || !slices.Contains(keys, key) {
|
||||
continue
|
||||
}
|
||||
for dep := range strings.FieldsSeq(value) {
|
||||
if !strings.HasPrefix(dep, "ovs") && !strings.HasPrefix(dep, "ovn") && !strings.HasPrefix(dep, "openvswitch") {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(crossDistroOVSUnits, dep) {
|
||||
t.Errorf("%s: %s=%s is not shipped under that name on every distro; use one of %v", name, key, dep, crossDistroOVSUnits)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostnetIsWantedNotRequired asserts the daemon and vpcd start after
|
||||
// spinifex-hostnet without Requires=: hostnet is PartOf ovs-vswitchd, so a
|
||||
// Requires= would restart both on every OVS restart or package upgrade.
|
||||
func TestHostnetIsWantedNotRequired(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
for _, name := range []string{"spinifex-daemon.service", "spinifex-vpcd.service"} {
|
||||
u := readUnit(t, dir, name)
|
||||
for _, key := range []string{"Wants", "After"} {
|
||||
if !directiveContains(u, key, "spinifex-hostnet.service") {
|
||||
t.Errorf("%s: %s= must include spinifex-hostnet.service", name, key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"Requires", "Requisite", "BindsTo", "PartOf"} {
|
||||
if directiveContains(u, key, "spinifex-hostnet.service") {
|
||||
t.Errorf("%s: %s=spinifex-hostnet.service would restart it with OVS; use Wants=", name, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user