spinifex/scripts/setup.sh

#!/bin/bash
# Spinifex binary installer
# Usage: curl -sfL https://install.mulgadc.com | bash
#        curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
#
# Environment variables:
#   INSTALL_SPINIFEX_CHANNEL   Release channel: latest (default), dev
#   INSTALL_SPINIFEX_VERSION   Pin to specific version (overrides channel)
#   INSTALL_SPINIFEX_TARBALL   Path to local tarball (skips download, for testing/air-gapped)
#   INSTALL_SPINIFEX_SKIP_APT  Set to 1 to skip the system package install (apt or dnf)
#   INSTALL_SPINIFEX_SKIP_AWS  Set to 1 to skip AWS CLI install
#   INSTALL_SPINIFEX_SKIP_NEWGRP  Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
#   ISO_BUILD                  Set to 1 when running inside a debootstrap chroot from the ISO
#                              builder: skip handle_upgrade/restart/migrations/newgrp/print_summary,
#                              skip systemctl daemon-reload + enable, short-circuit setup_sudo.
#   VERBOSE                    Set to 1 to echo "[setup] <stage>" before each top-level step.
#   SETUP_STAGES               Comma-separated subset of stages to run:
#                                deps, aws, dns, users, sudoers, firewall, timesync, files,
#                                directories, env, systemd, sysctl, needrestart, modules,
#                                logrotate, udev, fixown, migrations
#                              Unset = run every stage appropriate for the current mode.

set -e

INSTALL_SPINIFEX_CHANNEL="${INSTALL_SPINIFEX_CHANNEL:-latest}"
INSTALL_BASE_URL="${INSTALL_BASE_URL:-https://install.mulgadc.com}"

# Referenced by both the sudoers grant and the daemon, so the paths are fixed here.
ENDPOINT_SYSCTL_HELPER="/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl"
IPSEC_STATE_HELPER="/usr/local/lib/spinifex/spinifex-set-ipsec-state"

# The firewall policy and the peer addresses it scopes cluster ports to. The
# policy ships with the node; the peers are written by spinifex-daemon, which is
# the only component that knows every node's resolved planes.
FIREWALL_DIR="/etc/spinifex/firewall"
FIREWALL_RULES="${FIREWALL_DIR}/spinifex.nft"
# shellcheck disable=SC2034  # part of the firewall dir layout; written by install-node.sh
FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"

# The daemon runs with ProtectKernelModules=yes, so every module it needs must
# already be loaded when it starts.
SPINIFEX_KERNEL_MODULES="openvswitch geneve vhost_net tun"

# Whether the policy is armed, not merely installed. The ISO is an appliance on
# hardware we define, so a default-deny policy is the product; a curl-to-bash
# install lands on a machine that was already doing something, where arming a
# drop policy uninvited can cut off services we know nothing about.
INSTALL_SPINIFEX_FIREWALL="${INSTALL_SPINIFEX_FIREWALL:-}"

# --- Colors ---
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

info()  { echo -e "${GREEN}[INFO]${NC} $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC} $*"; }
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }

# --- BEGIN distro.sh probes ---
# Copied from scripts/lib/distro.sh, which this script installs but cannot source
# when piped from the install endpoint. distro_test.sh keeps the copies equal.
distro_pkg_manager() {
    local apt=false dnf=false
    command -v apt-get >/dev/null 2>&1 && apt=true
    command -v dnf >/dev/null 2>&1 && dnf=true
    if $apt && $dnf; then
        echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
        return 1
    fi
    if $apt; then echo apt; return 0; fi
    if $dnf; then echo dnf; return 0; fi
    echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
    return 1
}

distro_unit_exists() {
    local dir
    for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
        if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
            return 0
        fi
    done
    return 1
}

distro_chrony_conf() {
    local deb=false el=false
    [ -f "${DISTRO_ROOT:-}/etc/chrony/chrony.conf" ] && deb=true
    [ -f "${DISTRO_ROOT:-}/etc/chrony.conf" ] && el=true
    if $deb && $el; then
        echo "distro.sh: chrony config: found both /etc/chrony/chrony.conf and /etc/chrony.conf; cannot tell which one chronyd reads" >&2
        return 1
    fi
    if $deb; then echo /etc/chrony/chrony.conf; return 0; fi
    if $el; then echo /etc/chrony.conf; return 0; fi
    echo "distro.sh: chrony config: found neither /etc/chrony/chrony.conf nor /etc/chrony.conf; is chrony installed?" >&2
    return 1
}
# --- END distro.sh probes ---

stage() {
    [ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
    return 0
}

stage_enabled() {
    [ -z "${SETUP_STAGES:-}" ] && return 0
    case ",${SETUP_STAGES}," in
        *",$1,"*) return 0 ;;
        *) return 1 ;;
    esac
}

# --- Sudo setup ---
setup_sudo() {
    # Inside a debootstrap chroot we are already root and sudo may not be installed.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        SUDO=""
        return
    fi
    if [ "$(id -u)" -eq 0 ]; then
        SUDO=""
    elif command -v sudo >/dev/null 2>&1; then
        SUDO="sudo"
        if ! $SUDO -n true 2>/dev/null; then
            info "This installer requires sudo access for system-level operations"
            $SUDO true || fatal "Failed to obtain sudo access"
        fi
    else
        fatal "This script requires root or sudo access"
    fi
}

# --- OS detection ---
# os_release_field <key> <file> — the value of KEY=value, with one level of quotes removed.
os_release_field() {
    sed -n "s/^$1=//p" "$2" | head -n 1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
}

# The only place that keys on the OS ID: the gate and, later, the package table.
# Everything else probes for the capability (scripts/lib/distro.sh).
detect_os() {
    local os_release="${SPINIFEX_OS_RELEASE:-/etc/os-release}"
    if [ ! -f "$os_release" ]; then
        fatal "Cannot detect OS: $os_release not found"
    fi

    # Parsed, not sourced, so a malformed file is refused instead of executed.
    ID=$(os_release_field ID "$os_release")
    VERSION_ID=$(os_release_field VERSION_ID "$os_release")
    PRETTY_NAME=$(os_release_field PRETTY_NAME "$os_release")

    case "${ID:-}" in
        debian)
            if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
                fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
            fi
            ;;
        ubuntu)
            major="${VERSION_ID%%.*}"
            if [ "$major" -lt 24 ] 2>/dev/null; then
                fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
            fi
            ;;
        ol)
            check_oracle_linux
            ;;
        *)
            fatal "Unsupported OS: ${ID:-unknown} ${VERSION_ID:-}. Spinifex requires Debian 13+, Ubuntu 24.04+ or Oracle Linux 10.2+"
            ;;
    esac

    info "Detected OS: ${PRETTY_NAME:-$ID $VERSION_ID}"
}

# Oracle Linux 10.2+ on x86_64 only, running UEK R8: the KVM Utilities QEMU
# needs UEK, and Oracle ships kernel modules per kernel, so installing a newer
# kernel-uek alone leaves the running one without openvswitch.ko.
check_oracle_linux() {
    local unsupported="Oracle Linux ${VERSION_ID:-<no VERSION_ID>} is not supported. Spinifex requires Oracle Linux 10.2 or a later 10.x"
    local major="${VERSION_ID%%.*}" minor="${VERSION_ID#*.}" kernel
    minor="${minor%%.*}"
    case "${VERSION_ID:-}" in *.*) ;; *) fatal "$unsupported" ;; esac
    case "$major" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
    case "$minor" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
    if [ "$major" -ne 10 ] || [ "$minor" -lt 2 ]; then
        fatal "$unsupported"
    fi

    if [ "$(uname -m)" != "x86_64" ]; then
        fatal "Oracle Linux on $(uname -m) is not supported. Spinifex supports Oracle Linux on x86_64 only"
    fi

    kernel=$(uname -r)
    case "$kernel" in
        6.12.0-*uek*) ;;
        *) fatal "The running kernel $kernel is not UEK R8 (6.12.0-*uek). Install kernel-uek, reboot into it, then re-run this installer" ;;
    esac

    check_x86_64_v3
    check_selinux_not_enforcing
}

# Spinifex ships no SELinux policy yet. Oracle's templates boot enforcing, and
# setenforce alone reverts on reboot, so the config file is checked as well.
check_selinux_not_enforcing() {
    local config="${SPINIFEX_SELINUX_CONFIG:-/etc/selinux/config}" now="" boot=""
    if command -v getenforce >/dev/null 2>&1; then
        now=$(getenforce 2>/dev/null | tr '[:upper:]' '[:lower:]')
    fi
    if [ -f "$config" ]; then
        boot=$(os_release_field SELINUX "$config" | tr '[:upper:]' '[:lower:]')
    fi
    if [ "$now" = "enforcing" ]; then
        fatal "SELinux is enforcing, and Spinifex has no SELinux policy. Run 'sudo setenforce 0', set SELINUX=permissive in $config, then re-run this installer"
    fi
    if [ "$boot" = "enforcing" ]; then
        fatal "SELinux is set to enforcing at next boot (SELINUX=enforcing in $config). Set SELINUX=permissive there, then re-run this installer"
    fi
}

# The NFV SIG OVS/OVN and the KVM Utilities QEMU are built for x86-64-v3, so an
# older CPU installs cleanly and then dies on an illegal instruction.
check_x86_64_v3() {
    local cpuinfo="${SPINIFEX_CPUINFO:-/proc/cpuinfo}" flags flag missing=""
    flags=" $(grep -m1 '^flags' "$cpuinfo" 2>/dev/null | cut -d: -f2) "
    for flag in cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave; do
        case "$flags" in
            *" $flag "*) ;;
            *) missing="${missing:+$missing }$flag" ;;
        esac
    done
    if [ -n "$missing" ]; then
        fatal "This CPU is below x86-64-v3 (missing: $missing). Oracle Linux's OVS, OVN and QEMU builds require x86-64-v3"
    fi
}

# --- Architecture detection ---
detect_arch() {
    MACHINE=$(uname -m)
    case "$MACHINE" in
        x86_64)
            ARCH="amd64"
            QEMU_PACKAGES="qemu-system-x86"
            AWS_ARCH="x86_64"
            ;;
        aarch64|arm64)
            ARCH="arm64"
            QEMU_PACKAGES="qemu-system-arm"
            AWS_ARCH="aarch64"
            ;;
        *)
            fatal "Unsupported architecture: $MACHINE. Spinifex requires x86_64 or aarch64"
            ;;
    esac

    info "Detected architecture: $MACHINE ($ARCH)"
}

# --- Create per-service system users ---
create_service_users() {
    stage "creating spinifex group and per-service users"
    SPINIFEX_GROUP="spinifex"

    # Create shared group
    if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
        $SUDO groupadd --system "$SPINIFEX_GROUP"
    fi

    # Producer-typed access group for viperblock's runtime resources
    # (/run/spinifex/nbd socket dir). Viperblock writes as owner; daemon reads
    # via supplementary membership; every other spinifex-* service is excluded.
    if ! getent group spinifex-viperblock > /dev/null 2>&1; then
        $SUDO groupadd --system spinifex-viperblock
    fi

    # Create per-service users with correct home directories
    declare -A SERVICE_HOMES=(
        [nats]="/var/lib/spinifex/nats"
        [gw]="/var/lib/spinifex/awsgw"
        [daemon]="/var/lib/spinifex/spinifex"
        [storage]="/var/lib/spinifex/predastore"
        [northstar]="/var/lib/spinifex/northstar"
        [viperblock]="/var/lib/spinifex/viperblock"
        [vpcd]="/var/lib/spinifex"
        [ui]="/var/lib/spinifex"
    )
    for svc in nats gw daemon storage northstar viperblock vpcd ui; do
        local user="spinifex-${svc}"
        if ! id "$user" > /dev/null 2>&1; then
            $SUDO useradd --system --no-create-home \
                --home-dir "${SERVICE_HOMES[$svc]}" \
                --gid "$SPINIFEX_GROUP" \
                --shell /usr/sbin/nologin \
                "$user"
        fi
    done

    # Add invoking user to spinifex group for admin CLI access.
    # Skip under ISO_BUILD: the chroot has no invoking user (tf-user,
    # whoever ran `sudo make`, etc. don't exist in the rootfs). The ISO's
    # interactive 'spinifex' login account is created later in Phase 4 of
    # build-rootfs.sh with spinifex as its primary gid. In curl|bash mode
    # guard against a stale/missing SUDO_USER too.
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        ADMIN_USER="${SUDO_USER:-$(whoami)}"
        if [ "$ADMIN_USER" != "root" ] && id -u "$ADMIN_USER" > /dev/null 2>&1; then
            $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
        fi
    fi

    # KVM access for daemon
    if getent group kvm > /dev/null 2>&1; then
        $SUDO usermod -aG kvm spinifex-daemon
    fi

    # Daemon consumes viperblock's NBD socket — join the producer-typed group.
    $SUDO usermod -aG spinifex-viperblock spinifex-daemon

    info "Service users created (spinifex-{nats,gw,daemon,storage,northstar,viperblock,vpcd,ui})"
}

# --- Install scoped sudoers rules ---
# sudo-rs (Ubuntu's default sudo since 25.10) implements only a subset of
# Defaults and rejects an unknown one outright, so pam_session must be omitted
# there rather than emitted and ignored.
sudo_is_sudo_rs() {
    sudo --version 2>/dev/null | head -1 | grep -qi 'sudo-rs'
}

# The daemon only ever sets rp_filter and accept_local on an endpoint it just
# created. A fixed-verb helper keeps that grant expressible without a wildcard,
# which sudo-rs forbids inside command arguments.
install_endpoint_sysctl_helper() {
    $SUDO install -d -m 0755 /usr/local/lib/spinifex
    $SUDO tee "${ENDPOINT_SYSCTL_HELPER}" > /dev/null << 'HELPER'
#!/bin/sh
# Sets one of two per-endpoint sysctls for the asymmetric IMDS reply path.
# Interface, key and value are all constrained: this runs as root under a
# NOPASSWD grant, so it must never pass an arbitrary key through to sysctl.
set -eu

if [ "$#" -ne 3 ]; then
    echo "usage: $0 <interface> <rp_filter|accept_local> <0|1>" >&2
    exit 2
fi

iface=$1
key=$2
value=$3

# Reject anything that is not a plain interface name, so the key cannot be
# steered elsewhere in the sysctl tree with a "." or a "/".
case "${iface}" in
    ''|*[!A-Za-z0-9_-]*) echo "invalid interface: ${iface}" >&2; exit 2 ;;
esac

case "${key}" in
    rp_filter|accept_local) ;;
    *) echo "key not permitted: ${key}" >&2; exit 2 ;;
esac

case "${value}" in
    0|1) ;;
    *) echo "value not permitted: ${value}" >&2; exit 2 ;;
esac

exec sysctl -qw "net.ipv4.conf.${iface}.${key}=${value}"
HELPER
    $SUDO chown root:root "${ENDPOINT_SYSCTL_HELPER}"
    $SUDO chmod 0755 "${ENDPOINT_SYSCTL_HELPER}"
}

# The daemon turns OVN IPsec on and off as the cluster topology resolves, so it
# needs unit control. A NOPASSWD systemctl grant takes an arbitrary unit name and
# is root-equivalent, so the units are fixed here and only the state is an input.
install_ipsec_state_helper() {
    $SUDO install -d -m 0755 /usr/local/lib/spinifex
    $SUDO tee "${IPSEC_STATE_HELPER}" > /dev/null << 'HELPER'
#!/bin/sh
# Turns the host's OVN IPsec services on or off. Runs as root under a NOPASSWD
# grant, so it must never act on a unit the caller names.
set -eu

if [ "$#" -ne 1 ]; then
    echo "usage: $0 <on|off|guard|unguard>" >&2
    exit 2
fi

# libreswan drops a tunnel's xfrm policies while IKE fails, and pluto flushes
# the whole xfrm table when it starts or stops, so Geneve from the encap address
# is dropped in nft unless the route carries an SA.
GUARD_TABLE="inet spinifex_ipsec_guard"

unguard() {
    printf 'table %s\ndelete table %s\n' "$GUARD_TABLE" "$GUARD_TABLE" | nft -f -
}

want=""
want_src() {
    case "$1" in
        *.*.*.*) case "$1" in *[!0-9.]*) src=0.0.0.0/0 ;; *) src="$1/32" ;; esac ;;
        *) src=0.0.0.0/0 ;;
    esac
    case " $want " in *" $src "*) ;; *) want="$want $src" ;; esac
}

# One source per local address of a tunnel with a remote_name, which
# ovn-controller sets only while NB_Global.ipsec is on. A tunnel without a
# usable local_ip is guarded from every source rather than left open.
guard() {
    ifaces=$(ovs-vsctl --timeout=5 --bare --columns=name find Interface type=geneve)
    for iface in $ifaces; do
        peer=$(ovs-vsctl --timeout=5 --if-exists get Interface "$iface" options:remote_name)
        [ -n "$peer" ] || continue
        want_src "$(ovs-vsctl --timeout=5 --if-exists get Interface "$iface" options:local_ip | tr -d '"')"
    done
    # At boot pluto is up before ovn-controller builds the tunnels, and its
    # callers guard only while encryption is wanted, so guard the encap address
    # until the tunnels say otherwise.
    if [ -z "$ifaces" ]; then
        encap=$(ovs-vsctl --timeout=5 --if-exists get Open_vSwitch . external_ids:ovn-encap-ip | tr -d '"')
        for ip in $(echo "$encap" | tr ',' ' '); do want_src "$ip"; done
        [ -n "$want" ] || want_src ""
    fi
    if [ -z "$want" ]; then
        unguard
        return
    fi
    # One transaction, so the old set is never gone before the new one is in.
    nft -f - << NFT
table $GUARD_TABLE
delete table $GUARD_TABLE
table $GUARD_TABLE {
    set srcs {
        type ipv4_addr
        flags interval
        elements = { $(echo $want | tr ' ' ',') }
    }
    chain out {
        type filter hook output priority 0; policy accept;
        ip saddr @srcs udp dport 6081 rt ipsec missing drop
    }
}
NFT
}

# mask/unmask rather than disable/enable. The daemon's unit sets
# ProtectSystem=full, so /etc is read-only in the namespace its children inherit,
# and disabling a unit that also ships a SysV script shells out to update-rc.d
# there and fails. Masking is done by PID 1 over D-Bus and is unaffected.

# ovs-monitor-ipsec execs the strongSwan starter itself rather than going through
# this unit, so leaving it enabled only contends for UDP 500/4500. Off in both
# states. Masking a unit no package ships leaves a stray /dev/null link, so a
# host without strongSwan gets none, and loses one an older helper left.
if [ -e /usr/lib/systemd/system/strongswan-starter.service ] || [ -e /lib/systemd/system/strongswan-starter.service ]; then
    systemctl mask --now strongswan-starter.service >/dev/null 2>&1 || true
elif [ "$(systemctl is-enabled strongswan-starter.service 2>/dev/null)" = masked ]; then
    systemctl unmask strongswan-starter.service >/dev/null 2>&1 || true
fi

case "$1" in
    on)
        if [ -x /usr/libexec/ipsec/pluto ]; then
            # ovs-monitor-ipsec drives pluto over its control socket but does
            # not start it, so pluto comes up first under its own unit.
            systemctl unmask ipsec.service openvswitch-ipsec.service
            systemctl start ipsec.service
        else
            systemctl unmask openvswitch-ipsec.service
        fi
        exec systemctl start openvswitch-ipsec.service
        ;;
    off)
        systemctl mask --now openvswitch-ipsec.service
        # libreswan's pluto listens on UDP 500/4500 whenever ipsec.service runs.
        # On Debian that name is strongswan-starter's alias, masked above.
        if [ -x /usr/libexec/ipsec/pluto ]; then
            systemctl mask --now ipsec.service
            unguard
        fi
        ;;
    guard)
        guard
        ;;
    unguard)
        unguard
        ;;
    *)
        echo "state not permitted: $1" >&2
        exit 2
        ;;
esac
HELPER
    $SUDO chown root:root "${IPSEC_STATE_HELPER}"
    $SUDO chmod 0755 "${IPSEC_STATE_HELPER}"
}

# Every port sshd actually listens on, so a host hardened onto a non-standard
# port is not locked out by its own firewall. Reads the Include drop-ins too,
# since Debian's default config ends with one. Falls back to 22, which is what
# sshd itself does when no Port is set.
sshd_ports() {
    _cfg="/etc/ssh/sshd_config"
    _files="$_cfg"
    if [ -r "$_cfg" ]; then
        for _inc in $(awk '$1 == "Include" { $1 = ""; print }' "$_cfg" 2>/dev/null); do
            for _f in $_inc; do
                [ -r "$_f" ] && _files="$_files $_f"
            done
        done
    fi

    # shellcheck disable=SC2086
    _ports=$(awk '$1 == "Port" && $2 ~ /^[0-9]+$/ && $2+0 > 0 && $2+0 < 65536 { print $2 }' \
        $_files 2>/dev/null | sort -un | tr '\n' ' ')
    [ -n "$_ports" ] || _ports="22"

    _out=""
    for _p in $_ports; do
        if [ -z "$_out" ]; then _out="$_p"; else _out="$_out, $_p"; fi
    done
    printf '%s' "$_out"
}

# An unset --firewall keeps the mode a previous install recorded, so a rerun
# never disarms a node. The record is read as root: /etc/spinifex is closed to
# anyone outside the spinifex group, who would otherwise see no record at all.
resolve_firewall_mode() {
    local recorded
    case "${INSTALL_SPINIFEX_FIREWALL}" in
        on|off) return 0 ;;
        "") ;;
        *) fatal "--firewall must be 'on' or 'off', got: ${INSTALL_SPINIFEX_FIREWALL}" ;;
    esac
    if ${SUDO:-} test -e "$FIREWALL_MODE_FILE"; then
        recorded=$(${SUDO:-} cat "$FIREWALL_MODE_FILE") \
            || fatal "Cannot read the recorded firewall mode in $FIREWALL_MODE_FILE. Re-run with --firewall=on or --firewall=off"
        recorded=$(printf '%s' "$recorded" | tr -d '[:space:]')
        case "$recorded" in
            on|off) INSTALL_SPINIFEX_FIREWALL="$recorded"
                info "Host firewall: keeping the recorded mode '$recorded' ($FIREWALL_MODE_FILE)"
                return 0 ;;
            *) fatal "$FIREWALL_MODE_FILE holds '${recorded}', not 'on' or 'off'. Re-run with --firewall=on or --firewall=off" ;;
        esac
    fi
    if [ "${ISO_BUILD:-0}" != "1" ] && firewalld_active; then
        INSTALL_SPINIFEX_FIREWALL="on"
        info "Host firewall: firewalld is active, so the Spinifex policy will be armed in its place"
        return 0
    fi
    if [ "${ISO_BUILD:-0}" != "1" ] && ufw_active; then
        INSTALL_SPINIFEX_FIREWALL="on"
        info "Host firewall: ufw is active, so the Spinifex policy will be armed in its place"
        return 0
    fi
    # The ISO is an appliance and arms; any other host was already doing something.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        INSTALL_SPINIFEX_FIREWALL="on"
    else
        INSTALL_SPINIFEX_FIREWALL="off"
    fi
}

firewalld_active() {
    systemctl is-active --quiet firewalld.service 2>/dev/null
}

# Disable firewalld if present. The node firewall is spinifex-managed: with
# firewalld running, a packet must be accepted by both its table and ours, so
# routed NAT, elastic IPs and IMDS on ime-* ports break with nothing logged.
# Masked, not just disabled, so a package update or D-Bus activation cannot
# bring it back. Its rules are not saved or migrated.
mask_firewalld() {
    local state
    [ "${ISO_BUILD:-0}" = "1" ] && return 0
    state=$(systemctl is-enabled firewalld.service 2>/dev/null) || true
    case "$state" in
        "" | not-found) return 0 ;;
        masked) if ! firewalld_active; then
                info "  firewalld already masked"
                return 0
            fi ;;
    esac
    $SUDO systemctl mask --now firewalld.service > /dev/null \
        || fatal "Cannot stop and mask firewalld. The Spinifex nftables policy cannot run beside it"
    warn "firewalld is now stopped and masked: the Spinifex nftables policy is the only supported node firewall."
    warn "  firewalld's rules were not migrated. The Spinifex policy filters input only, accepts ssh"
    warn "  from every IPv4 source until \$trusted_ssh_peers in $FIREWALL_CUSTOM narrows it, and opens"
    warn "  53, 443, 3000, 8443 and 9999 to any source. Restate any other restriction in that file."
    if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
        warn "  Until the daemon first loads the policy, this host accepts every connection."
    else
        warn "  The Spinifex policy is not armed (--firewall=off), so this host now has no firewall."
    fi
}

ufw_active() {
    command -v ufw > /dev/null 2>&1 || return 1
    ${SUDO:-} ufw status 2>/dev/null | grep -q '^Status: active'
}

# Disable ufw if it is enforcing, as some cloud images ship it (Vultr's Ubuntu
# does, with INPUT DROP). Its rules would have to accept everything the Spinifex
# policy does, so guest IMDS, routed NAT and the API break with nothing logged.
# `ufw disable` is persistent and reversible; the rules are kept, not migrated.
disable_ufw() {
    [ "${ISO_BUILD:-0}" = "1" ] && return 0
    ufw_active || return 0
    $SUDO ufw disable > /dev/null \
        || fatal "Cannot disable ufw. The Spinifex nftables policy cannot run beside it"
    warn "ufw is now disabled: the Spinifex nftables policy is the only supported node firewall."
    warn "  ufw's rules were not migrated. Restate any restriction you relied on in $FIREWALL_CUSTOM."
    if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
        warn "  Until the daemon first loads the policy, this host accepts every connection."
    else
        warn "  The Spinifex policy is not armed (--firewall=off), so this host now has no firewall."
    fi
}

# The policy is written here rather than shipped in the tarball so the ISO chroot
# and the ansible path get an identical file without a packaging step. Installed
# but NOT enabled: the peer set is empty until spinifex-daemon resolves the
# cluster's planes, and a drop policy with no peers would break formation.
install_firewall() {
    stage "installing host firewall policy"
    info "Installing host firewall policy..."

    $SUDO install -d -m 0755 "$FIREWALL_DIR" /usr/local/lib/spinifex
    $SUDO tee "$FIREWALL_RULES" > /dev/null << 'RULES'
#!/usr/sbin/nft -f
# Spinifex host firewall. Managed by scripts/setup.sh — edits are overwritten.
#
# Only `table inet spinifex_filter` is created, deleted and replaced. vpcd writes
# MASQUERADE and per-EIP FORWARD ACCEPT rules into the `ip nat` and `ip filter`
# tables and reinstalls them only when the service starts, so anything that
# flushes the whole ruleset breaks elastic IPs silently until the next restart.
#
# INPUT only. nftables runs every table registered on a hook and any drop is
# final, so a forward-hook policy here could not be rescued by vpcd's ACCEPTs in
# the other table. OUTPUT is untouched: the IMDS reply path egresses under
# per-tap policy routing.

include "/etc/spinifex/firewall/local.nft"
include "/etc/spinifex/firewall/open-ports.nft"
include "/etc/spinifex/firewall/peers.nft"
include "/etc/spinifex/firewall/custom.nft"

# Create-then-delete so the replace is atomic and the delete cannot fail on a
# first run. nft applies the whole file as one transaction or none of it.
table inet spinifex_filter
delete table inet spinifex_filter

table inet spinifex_filter {
    chain input {
        type filter hook input priority filter; policy drop;

        # First, or every reply to a connection this node opened is dropped.
        ct state established,related accept
        ct state invalid drop
        iif lo accept

        # Guest metadata and VPC DNS terminate on per-ENI OVS internal ports in
        # the host netns, so guest traffic to 169.254.169.254 and .253 arrives
        # here. Without this, cloud-init, instance-role credentials and all
        # guest DNS break. The ports exist only while instances run, so this
        # matches the prefix rather than an enumerated list.
        #
        # Scoped to the two link-local endpoints and their ports, not a blanket
        # accept: the OVS demux delivers any guest packet to 169.254.169.254/.253
        # onto an ime-* port regardless of L4 port, so `iifname "ime-*" accept`
        # would expose every wildcard-bound host service (sshd, the AWS gateway,
        # northstar, the UI) to any tenant. IMDS is HTTP on .254:80; VPC DNS is
        # 53 on .253. Everything else on these ports falls through to policy drop.
        #
        # Matched on the pre-DNAT destination: with imds_host_meta_ip set, the
        # remap rewrites the address in prerouting, before this hook sees it.
        iifname "ime-*" ct original ip daddr 169.254.169.254 tcp dport 80 accept
        iifname "ime-*" ct original ip daddr 169.254.169.253 udp dport 53 accept
        iifname "ime-*" ct original ip daddr 169.254.169.253 tcp dport 53 accept

        # PMTUD is not optional under a Geneve overlay.
        icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
        icmpv6 type { echo-request, destination-unreachable, packet-too-big, time-exceeded, parameter-problem, nd-neighbor-solicit, nd-neighbor-advert, nd-router-solicit, nd-router-advert } accept

        # A broadcast DHCP reply does not reliably match an established
        # conntrack entry, so the uplink and external-pool leases need this.
        udp sport 67 udp dport 68 accept

        # Scoped by $trusted_ssh_peers from custom.nft, which the installer
        # creates once and never rewrites. That default accepts every source,
        # so a node is no less reachable than before until an operator narrows it.
        ip saddr $trusted_ssh_peers tcp dport $spinifex_ssh_ports accept

        # Public plane. 9999 is also how every guest agent (lb, eks, ecs, rds)
        # reaches its control plane over br-mgmt: they speak SigV4 to the AWS
        # gateway, never NATS, so no cluster port needs to face the guests.
        # 443 has no listener yet and is held for the nginx TLS edge, so the
        # rollout does not need a firewall change on every node.
        # nfproto ipv4: these accepts carry no source scope, so without a family
        # qualifier they would also serve on any global IPv6 a node autoconfigures
        # from an upstream RA. The scoped rules are v4-only via `ip saddr` already.
        meta nfproto ipv4 tcp dport { 443, 3000, 8443, 9999 } accept

        # Transiently open to any source, for cluster formation. A node dialling
        # in to join is not a peer yet, so the peer-scoped rule below cannot let
        # it in; `spx admin init` opens the formation port here for the length of
        # the formation window and closes it again afterwards. Normally holds
        # only the sentinel, which no packet can match.
        meta nfproto ipv4 tcp dport $spinifex_open_ports accept

        # 53 is open on every node, deliberately: northstar serves public
        # authoritative DNS and binds its advertise address. A node running no
        # public zone answers here too, which is the accepted cost of not
        # templating the policy per node.
        meta nfproto ipv4 tcp dport 53 accept
        meta nfproto ipv4 udp dport 53 accept

        # Cluster plane, peer-scoped. On a single-NIC node the planes collapse
        # onto the public address, which is why these are peer addresses rather
        # than an interface or a CIDR. 4432 stays here rather than joining the
        # public rule above: outside formation it is the daemon cluster manager,
        # whose /health and /local/* routes report node topology, service
        # inventory and running instances to anyone who asks. 8660 is
        # predastore's unauthenticated /healthz and /readyz, which name the
        # peers a gate cannot reach, so it belongs here for the same reason.
        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
        ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept

        # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
        # kernel UDP-tunnel socket, so encapsulated packets are delivered
        # locally and pass this hook before the OVS vport sees them.
        ip saddr $spinifex_encap_peers udp dport { 6081, 500, 4500 } accept
        ip saddr $spinifex_encap_peers meta l4proto esp accept

        # Rate-limited so a scan cannot fill the journal. This is the only way
        # to tell a policy gap from an application fault after the fact.
        limit rate 5/minute burst 10 packets log prefix "spinifex-fw drop: " level info
    }
}
RULES
    $SUDO chmod 0644 "$FIREWALL_RULES"
    info "  $FIREWALL_RULES"

    # Host-local facts the policy needs that the daemon has no view of. Separate
    # from peers.nft because the daemon rewrites that file and would drop this.
    # Refreshed on every setup.sh run, so an sshd port change is picked up by an
    # upgrade rather than needing the file edited by hand.
    _ssh_ports="$(sshd_ports)"
    $SUDO tee "$FIREWALL_LOCAL" > /dev/null << LOCAL
# Managed by scripts/setup.sh — edits are overwritten on upgrade.
define spinifex_ssh_ports = { ${_ssh_ports} }
LOCAL
    $SUDO chmod 0644 "$FIREWALL_LOCAL"
    info "  $FIREWALL_LOCAL (sshd ports: ${_ssh_ports})"

    # Reset closed on every run, so an install or upgrade that interrupts a
    # formation window cannot leave the port open indefinitely.
    $SUDO tee "$FIREWALL_OPEN" > /dev/null << 'OPENPORTS'
# Managed by spinifex-firewall-apply. Rewritten for the formation window only.
define spinifex_open_ports = { 0 }
OPENPORTS
    $SUDO chmod 0644 "$FIREWALL_OPEN"
    info "  $FIREWALL_OPEN (closed)"

    # Operator-owned, so it is created once and never rewritten: an upgrade, a
    # re-install and a node reset all leave an edited file alone. Never write it
    # empty — an undefined variable fails the whole ruleset, not just its rule.
    if [ ! -e "$FIREWALL_CUSTOM" ]; then
        $SUDO tee "$FIREWALL_CUSTOM" > /dev/null << 'CUSTOM'
# Operator-owned host firewall settings. setup.sh creates this file once and
# never overwrites it, so edits here survive upgrades, re-installs and resets.
#
# Check a change without applying it, then apply it:
#   sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
#   sudo nft -f  /etc/spinifex/firewall/spinifex.nft
#
# Check spinifex.nft, not this file: on its own this parses as a bare define
# and proves nothing, because the rule that uses the variable lives there.
#
# Sources allowed to reach SSH. 0.0.0.0/0 accepts from anywhere; narrow it to
# your management networks. IPv4 only — sshd reached over IPv6 is not matched.
define trusted_ssh_peers = { 0.0.0.0/0 }
CUSTOM
        $SUDO chmod 0644 "$FIREWALL_CUSTOM"
        info "  $FIREWALL_CUSTOM (created — ssh open to every source)"
    else
        info "  $FIREWALL_CUSTOM (already present, left unchanged)"
    fi

    # Armed or merely installed. The daemon reads this when the config carries no
    # explicit firewall_enabled, which is the normal case on both install paths.
    printf '%s\n' "$INSTALL_SPINIFEX_FIREWALL" | $SUDO tee "$FIREWALL_MODE_FILE" > /dev/null
    $SUDO chmod 0644 "$FIREWALL_MODE_FILE"
    info "  $FIREWALL_MODE_FILE ($INSTALL_SPINIFEX_FIREWALL)"

    # After the mode file, so a retry after an interrupted install still finds
    # 'on' once firewalld or ufw is no longer active.
    mask_firewalld
    disable_ufw

    $SUDO tee "$FIREWALL_APPLY" > /dev/null << 'APPLY'
#!/bin/sh
# Applies the spinifex host firewall. Fails without changing the ruleset when
# the peer file is missing, so a node whose daemon has not yet resolved cluster
# membership is left reachable rather than cut off from a cluster it cannot name.
set -eu

RULES="/etc/spinifex/firewall/spinifex.nft"
PEERS="/etc/spinifex/firewall/peers.nft"
OPEN="/etc/spinifex/firewall/open-ports.nft"

# 0 is a sentinel that keeps the set non-empty, because nft rejects an empty
# one. No packet can match it: the kernel has no destination port 0.
write_open_ports() {
    _tmp=$(mktemp)
    printf '%s\n' \
        '# Managed by spinifex-firewall-apply. Rewritten for the formation window only.' \
        "define spinifex_open_ports = { $1 }" > "$_tmp"
    install -m 0644 "$_tmp" "$OPEN"
    rm -f "$_tmp"
}

# set-peers reads two comma-separated address lists on stdin (cluster peers,
# then encap peers) and rewrites the peer file before applying. Every address is
# re-validated here as a bare dotted quad: this runs as root under a NOPASSWD
# grant, so a caller must not be able to inject nft syntax through it.
# disable removes spinifex's own table and the peer file, leaving every other
# table alone. Idempotent, so it is safe on a node that never had the policy.
# Deliberately ahead of the policy-file check below: turning the firewall off is
# the recovery path, and refusing to run it because the policy is missing leaves
# an operator with a node they can neither arm nor disarm.
if [ "${1:-}" = "disable" ]; then
    nft delete table inet spinifex_filter 2>/dev/null || true
    rm -f "$PEERS"
    systemctl disable --now spinifex-firewall.service >/dev/null 2>&1 || true
    exit 0
fi

# Every remaining verb writes the policy, so it has to exist. Failing here
# changes nothing, which leaves a node whose daemon has not yet resolved cluster
# membership reachable rather than cut off from a cluster it cannot name.
[ -r "$RULES" ] || { echo "no firewall policy at $RULES" >&2; exit 1; }

# open-port and close-port hold a single port open to any source while a cluster
# forms. One slot, not a list: the only caller is `spx admin init` opening its
# formation port, and a set that can only ever hold one entry cannot be grown
# into a general-purpose hole in the policy.
if [ "${1:-}" = "open-port" ] || [ "${1:-}" = "close-port" ]; then
    port="${2:-}"
    case "$port" in
        '' | *[!0-9]*) echo "port must be a number: '$port'" >&2; exit 2 ;;
    esac
    [ "$port" -gt 0 ] && [ "$port" -lt 65536 ] || {
        echo "port out of range: $port" >&2
        exit 2
    }

    if [ "$1" = "open-port" ]; then write_open_ports "0, $port"; else write_open_ports "0"; fi

    # Nothing to reload on a node that is installed but not armed: the table is
    # not loaded, so every port is already open and the file is enough.
    [ -r "$PEERS" ] || exit 0
    exec nft -f "$RULES"
fi

if [ "${1:-}" = "set-peers" ]; then
    read -r peers_in || peers_in=""
    read -r encap_in || encap_in=""

    # Octet ranges are checked, not just the shape: nft rejects 10.0.0.999 and
    # the whole transaction with it, which would leave a peer file that fails on
    # every boot. A loop, not a pipeline — a rejection in a `while read` subshell
    # cannot fail the function.
    OCTET='(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
    validate() {
        _out=""
        for a in $(echo "$1" | tr ',' ' '); do
            echo "$a" | grep -Eq "^${OCTET}(\.${OCTET}){3}$" || {
                echo "address not permitted: $a" >&2
                return 2
            }
            if [ -z "$_out" ]; then _out="$a"; else _out="$_out, $a"; fi
        done
        printf '%s' "$_out"
    }

    peers=$(validate "$peers_in") || exit 2
    encap=$(validate "$encap_in") || exit 2
    [ -n "$peers" ] && [ -n "$encap" ] || {
        echo "set-peers needs a non-empty peer and encap list" >&2
        exit 2
    }

    tmp=$(mktemp)
    bak=$(mktemp)
    printf '%s\n' \
        '# Managed by spinifex-daemon. Regenerated from cluster membership.' \
        "define spinifex_peers = { $peers }" \
        "define spinifex_encap_peers = { $encap }" > "$tmp"
    if [ -r "$PEERS" ]; then cp "$PEERS" "$bak"; fi
    install -m 0644 "$tmp" "$PEERS"

    # Roll the peer file back if the ruleset will not load, so the boot-time
    # unit never inherits a file already known to fail.
    if ! nft -f "$RULES"; then
        if [ -s "$bak" ]; then install -m 0644 "$bak" "$PEERS"; else rm -f "$PEERS"; fi
        rm -f "$tmp" "$bak"
        exit 1
    fi
    rm -f "$tmp" "$bak"

    # The policy is loaded, so make it survive a reboot too. `disable` turns this
    # unit off, and forming a cluster means disabling the firewall on every node
    # first; without this, re-arming would look complete while silently leaving
    # the node bare after its next boot. Idempotent and quiet in the normal case.
    systemctl enable spinifex-firewall.service >/dev/null 2>&1 || true
    exit 0
fi

if [ ! -r "$PEERS" ]; then
    echo "no peer file at $PEERS: the daemon writes it once cluster membership" >&2
    echo "resolves. Ruleset left unchanged." >&2
    exit 1
fi

exec nft -f "$RULES"
APPLY
    $SUDO chown root:root "$FIREWALL_APPLY"
    $SUDO chmod 0755 "$FIREWALL_APPLY"
    info "  $FIREWALL_APPLY"

    # Say which state ssh is in either way. A node whose allowlist was never
    # narrowed reads as configured otherwise, and that is the whole risk here.
    _ssh_srcs="$(grep -o '{[^}]*}' "$FIREWALL_CUSTOM" 2>/dev/null | tr -d '{}' | tr -s ' ')"
    case "$_ssh_srcs" in
        *0.0.0.0/0*) info "  ssh (${_ssh_ports}) accepted from ANY source — narrow \$trusted_ssh_peers in $FIREWALL_CUSTOM" ;;
        ?*) info "  ssh (${_ssh_ports}) accepted from:${_ssh_srcs}" ;;
    esac

    if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
        info "Host firewall installed and armed (applied by spinifex-daemon once peers resolve)"
    else
        info "Host firewall installed but NOT armed — this machine may already be"
        info "running services a default-deny policy would cut off."
        info "  Allowed if armed: ssh (${_ssh_ports}), 53, 443, 3000, 8443, 9999"
        info "  Everything else on this host stops accepting new connections."
        info "  Arm it later by setting network.firewall_enabled = true in"
        info "  /etc/spinifex/spinifex.toml, or re-run this installer with --firewall=on"
    fi
}

# --- Host DNS ---
# The one resolver setup supported on a host with systemd-resolved installed:
# NetworkManager hands link DNS to resolved, and /etc/resolv.conf is its stub.
RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
# node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"

# systemd-resolved ships resolvconf as a link to resolvectl, which only works
# while resolved runs, so that one is not a resolvconf backend.
real_resolvconf() {
    local path
    path=$(command -v resolvconf) || return 1
    [ "$(basename "$(readlink -f "$path")")" != resolvectl ]
}

# Prints the NetworkManager setting that keeps it from handing DNS to resolved,
# or nothing. Only explicitly set keys are uncommented in --print-config.
nm_dns_blocker() {
    local config
    config=$($SUDO NetworkManager --print-config) || return 1
    printf '%s\n' "$config" | awk -F= '
        /^\[/ { main = ($0 == "[main]"); next }
        !main || /^#/ { next }
        $1 == "dns" && $2 != "default" && $2 != "systemd-resolved" { print; exit }
        $1 == "systemd-resolved" && ($2 == "false" || $2 == "no" || $2 == "0" || $2 == "off") { print; exit }'
}

# The URL of the first enabled package repository, which the install has
# reached unless system packages were skipped.
first_repo_url() {
    # shellcheck disable=SC2016  # apt's own $(FIELD) template, not a shell expansion
    case "$(distro_pkg_manager 2>/dev/null)" in
        apt) apt-get indextargets --format '$(REPO_URI)' 2>/dev/null | head -n 1 ;;
        dnf) dnf config-manager --dump '*' 2>/dev/null | awk '
                function flush() { if (enabled == "1" && url != "") { print url; done = 1; exit } }
                /^\[/ { flush(); enabled = ""; url = ""; next }
                /^(baseurl|mirrorlist|metalink) = / { v = $3; sub(/,.*/, "", v); if (url == "") url = v }
                /^enabled = / { enabled = $3 }
                END { if (!done) flush() }' ;;
    esac
}

# The host part of a URL. Prints nothing for a file:// or schemeless URL.
url_host() {
    local rest="${1#*://}"
    case "$1" in file://* | "$rest") return 0 ;; esac
    rest="${rest%%/*}"
    rest="${rest##*@}"
    case "$rest" in
        \[*) echo "${rest%%]*}]" ;;
        *) echo "${rest%%:*}" ;;
    esac
}

dns_handover_failed() {
    fatal "Host DNS through systemd-resolved does not work: $1.
    To put host DNS back as it was before this installer:
      sudo systemctl disable --now systemd-resolved
      sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
      sudo systemctl restart NetworkManager"
}

# Checks resolved has upstream servers and still resolves the package
# repository host, the one name this host is known to have resolved.
check_resolved_dns() {
    local i url host
    for i in 1 2 3 4 5 6 7 8 9 10; do
        resolvectl dns 2>/dev/null | grep -q ':[[:space:]]*[^[:space:]]' && break
        [ "$i" = 10 ] && dns_handover_failed "'resolvectl dns' lists no DNS servers 10 seconds after NetworkManager restarted"
        sleep 1
    done

    url=$(first_repo_url)
    host=$(url_host "$url")
    case "$host" in
        "")
            warn "No package repository with a host name to look up (${url:-none found}); not checking that names resolve"
            return 0 ;;
        \[*) ;;
        *[!0-9.]*)
            getent ahosts "$host" > /dev/null || dns_handover_failed "the package repository host $host no longer resolves"
            return 0 ;;
    esac
    warn "The package repository host $host is an IP address; not checking that names resolve"
}

# Where systemd-resolved is installed but not running and nothing else manages
# DNS, turns it on behind NetworkManager. The original resolv.conf is saved
# once, so a rerun never replaces it with one this installer wrote.
hand_dns_to_resolved() {
    stage "handing host DNS to systemd-resolved"
    local blocker
    [ "${ISO_BUILD:-0}" = "1" ] && return 0

    if systemctl is-active --quiet systemd-resolved.service; then
        # A handover made by an earlier run is checked again on every run.
        if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
            check_resolved_dns
        fi
        return 0
    fi
    if ! distro_unit_exists systemd-resolved.service; then
        return 0
    fi
    if real_resolvconf; then
        info "resolvconf manages host DNS; leaving it as it is"
        return 0
    fi
    if ! systemctl is-active --quiet NetworkManager.service; then
        warn "systemd-resolved is installed but not running, and NetworkManager is not running to hand it DNS. Leaving host DNS as it is: Spinifex names will not resolve on this host until its resolver forwards them to northstar"
        return 0
    fi
    blocker=$(nm_dns_blocker) || fatal "Cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
    if [ -n "$blocker" ]; then
        fatal "NetworkManager sets $blocker, so it cannot hand DNS to systemd-resolved. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
    fi

    info "Handing host DNS to systemd-resolved..."
    if [ ! -e "$RESOLV_CONF_SAVED" ] && [ ! -L "$RESOLV_CONF_SAVED" ]; then
        $SUDO mkdir -p "$(dirname "$RESOLV_CONF_SAVED")"
        $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
        info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
    fi
    $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
        || dns_handover_failed "systemd-resolved did not start"
    $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
    check_resolved_dns
    info "  $RESOLV_CONF -> $RESOLVED_STUB"
}

# Debian's stock chrony.conf carries `makestep 1 3`: step the clock only for the
# first three updates, then slew at ~83us/s, which is roughly 7 seconds of
# correction per day. A node that burns those three before the network is usable
# then crawls toward correct time for days, and SigV4 rejects its requests the
# whole time. A drop-in rather than an edit to chrony.conf, which is a dpkg
# conffile and would prompt on every future upgrade.
#
# EL's chrony.conf reads no drop-ins, so a confdir line is appended to it.
# chronyd keeps the last makestep it reads, so drop-ins must come after the
# stock line. DISTRO_ROOT prefixes the paths here as it does for the probes.
install_chrony_conf() {
    stage "installing chrony drop-in"
    local root="${DISTRO_ROOT:-}" conf dir="" confdir_line=0 makestep_line

    if ! conf=$(distro_chrony_conf); then
        warn "Skipping the chrony drop-in"
        return
    fi
    read -r confdir_line dir < <(awk '$1 == "confdir" { print NR, $2; exit }' "$root$conf") || true
    if [ -z "$dir" ]; then
        dir=/etc/chrony.d
        printf '\n# Added by the Spinifex installer. Last, so a drop-in overrides the settings above.\nconfdir %s\n' "$dir" \
            | $SUDO tee -a "$root$conf" > /dev/null
        confdir_line=$(wc -l < "$root$conf")
        info "  $conf: confdir $dir"
    fi
    makestep_line=$(awk '$1 == "makestep" { n = NR } END { print n + 0 }' "$root$conf")
    if [ "$makestep_line" -gt "$confdir_line" ]; then
        warn "$conf sets makestep on line $makestep_line, after its confdir line, which overrides the Spinifex drop-in. Move the confdir line to the end of $conf"
    fi

    $SUDO mkdir -p "$root$dir"
    $SUDO tee "$root$dir/spinifex.conf" > /dev/null << 'CHRONY'
# Managed by scripts/setup.sh — edits are overwritten.
# Step at any offset over a second, however many times it takes. The clock can
# jump backwards under load, which is the accepted cost: everything
# latency-sensitive here uses the monotonic clock, and a node minutes out of
# step fails every signed request until it converges.
makestep 1 -1
CHRONY
    $SUDO chmod 0644 "$root$dir/spinifex.conf"
    info "  $dir/spinifex.conf"
}

install_sudoers() {
    stage "installing scoped sudoers rules"
    install_endpoint_sysctl_helper
    install_ipsec_state_helper

    if sudo_is_sudo_rs; then
        $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
# pam_session is omitted: this host runs sudo-rs, which does not implement it.
SUDOERS
    else
        $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
# No PAM session for the service users. Every sudo call otherwise logs
# "pam_limits: Could not set limit for 'core'": the units set LimitCORE=0 and
# their CapabilityBoundingSet omits CAP_SYS_RESOURCE, so pam_limits cannot apply
# the limits.conf default and warns. It also means sudo children inherit the
# unit's rlimits rather than the system defaults, which is the RG-9 intent.
Defaults:spinifex-daemon !pam_session
SUDOERS
    fi

    $SUDO tee -a /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'

# The OVS/OVN client tools are deliberately absent. They do all their work over
# control sockets that setup-ovn.sh group-owns to `spinifex`, so they run as the
# service user. Granting them would be worse than pointless: a NOPASSWD rule for
# them takes unrestricted args, and ovs-vsctl/ovn-nbctl/ovn-appctl all accept
# --log-file=PATH, which writes a root-owned file wherever the caller points it.
#
# ovs-ofctl is the exception that keeps its grant: it talks to a per-bridge
# <bridge>.mgmt socket created by ovs-vswitchd when the bridge appears, including
# bridges spinifex creates at runtime, so it cannot be group-owned up front.

# spinifex-vpcd has no rules at all. Its ip/iptables/sysctl/arping work is done
# under the CAP_NET_ADMIN and CAP_NET_RAW its unit grants ambiently, which the
# kernel passes to each child on exec. Those grants were also root-equivalent:
# `sudo ip netns exec <ns> /bin/sh` is a root shell.

# Spinifex daemon: tap devices and OpenFlow rules. Its unit holds no
# CAP_NET_ADMIN, so unlike vpcd it cannot drop these. ovs-ofctl installs the
# per-tap IMDS datapath flows on br-imds; the sysctl helper sets the
# per-endpoint rp_filter/accept_local the asymmetric reply path needs.
spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-ofctl
SUDOERS

    # Separate append so the prose above can stay in a quoted heredoc: it carries
    # backticks that an expanding one would run as command substitution.
    $SUDO tee -a /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
spinifex-daemon ALL=(root) NOPASSWD: ${ENDPOINT_SYSCTL_HELPER}
spinifex-daemon ALL=(root) NOPASSWD: ${IPSEC_STATE_HELPER}
spinifex-daemon ALL=(root) NOPASSWD: ${FIREWALL_APPLY}
SUDOERS
    $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
    info "Scoped sudoers rules installed for spinifex-daemon (spinifex-vpcd needs none)"
}

# --- Install system dependencies ---
# NOTE: Runtime deps must stay in sync with the ISO package list at
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
# remove, or change a runtime package here, review that file too — drift means
# the ISO and `curl | bash` paths install different software.
#
# Hoisted out of the apt-get call so scripts/lint-apt-packages.sh can resolve the
# same list against each supported suite. The arch-specific qemu package is
# added by install_apt_deps from detect_arch.
APT_RUNTIME_PACKAGES="nbdkit
qemu-utils gdisk ovmf qemu-efi-aarch64 less
libvirt-daemon-system libvirt-clients
pciutils
jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
chrony nftables"

# The Oracle Linux equivalents, plus what a minimal OL install lacks. OVS 3.5
# and OVN 25.03 come from the CentOS NFV SIG; 25.03 is Debian 13's OVN, which
# the hand-written OVN NB models are written against.
OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
qemu-kvm qemu-img edk2-ovmf seabios-bin less
pciutils
jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
chrony nftables iptables-nft systemd-resolved"

# Oracle ships no OVS or OVN for OL10. The allowlist stops the SIG supplying
# anything but OVS, OVN and the two SIG-only packages they depend on.
NFV_SIG_REPO_ID="spinifex-nfv-sig"
NFV_SIG_KEY_URL="https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-NFV"
NFV_SIG_KEY_FILE="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV"
NFV_SIG_KEY_FPR="35154228174901BEFA8E69A621465E289D2A76A7"
NFV_SIG_INCLUDEPKGS="openvswitch3.5* ovn25.03* python3-openvswitch3.5 openvswitch-selinux-extra-policy"

# Only the KVM Utilities QEMU has the microvm machine. Excluding QEMU from
# AppStream stops a later AppStream build ever replacing part of that stack.
OL_KVM_UTILS_REPO="ol10_kvm_utils"
OL_APPSTREAM_REPO="ol10_appstream"
OL_APPSTREAM_EXCLUDES="qemu-kvm* qemu-img*"
# The running kernel's modules come from here, and KVM Utilities requires kernel-uek.
OL_UEK_REPO="ol10_UEKR8"

install_deps() {
    stage "installing system dependencies"
    if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
        info "Skipping system dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
    else
        local pm
        pm=$(distro_pkg_manager) || fatal "Cannot tell which package manager this host uses"
        case "$pm" in
            apt) install_apt_deps ;;
            dnf) install_dnf_deps ;;
        esac
    fi

    # Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
    # dhcpcd-base is present. It binds br-wan and competes with vpcd's
    # nclient4 for OFFERs, draining the upstream pool and causing
    # intermittent DORA failures. Must run even when apt is skipped (CI
    # bootstrap runs with INSTALL_SPINIFEX_SKIP_APT=1 against runners that
    # already have dhcpcd-base preinstalled). The ISO installer does the
    # same mask (spinifex/operator/installer/install/install.go).
    $SUDO systemctl disable --now dhcpcd.service 2>/dev/null || true
    $SUDO systemctl mask dhcpcd.service 2>/dev/null || true
}

install_apt_deps() {
    info "Installing system dependencies..."
    $SUDO apt-get update -qq

    # Unquoted on purpose: both variables are whitespace-separated package lists.
    # shellcheck disable=SC2086
    DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
        $QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
        > /dev/null

    info "System dependencies installed"

    # Only on a machine that already boots from ZFS. Pulling zfsutils-linux
    # in unconditionally would drag zfs-dkms onto every dev box and rebuild
    # the module on each kernel upgrade for no benefit.
    if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
        info "ZFS root detected, installing zfsutils-linux..."
        DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
    fi
}

# The package table is the other place keyed on the OS ID: dnf alone does not
# say which repositories or package names a host has.
install_dnf_deps() {
    case "${ID:-}" in
        ol) ;;
        *) fatal "No dnf package list for ${ID:-an unknown OS}. Spinifex installs with dnf on Oracle Linux only" ;;
    esac
    if [ "$(findmnt -no FSTYPE /)" = "zfs" ]; then
        fatal "A ZFS root is not supported on Oracle Linux"
    fi

    ensure_ol_repos
    install_running_kernel_modules

    info "Installing system dependencies..."
    # shellcheck disable=SC2086  # a whitespace-separated package list
    $SUDO dnf install -y -q $OL_RUNTIME_PACKAGES > /dev/null
    info "System dependencies installed"

    # shellcheck disable=SC2086  # a whitespace-separated module list
    load_kernel_modules $SPINIFEX_KERNEL_MODULES
}

# Enables KVM Utilities, pins QEMU to it, and adds the NFV SIG repository with
# a key whose fingerprint must match the one recorded above.
ensure_ol_repos() {
    info "Configuring Oracle Linux repositories..."
    local repo
    $SUDO dnf install -y -q dnf-plugins-core gnupg2 curl > /dev/null

    for repo in "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" "$OL_APPSTREAM_REPO"; do
        dnf config-manager --dump "$repo" > /dev/null 2>&1 \
            || fatal "Repository $repo is not defined on this host. Oracle Linux's release packages define it; reinstall oraclelinux-release-el10"
    done
    $SUDO dnf config-manager --set-enabled "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" > /dev/null
    exclude_from_appstream

    install_nfv_sig_key
    $SUDO tee "/etc/yum.repos.d/${NFV_SIG_REPO_ID}.repo" > /dev/null <<EOF
# Written by the Spinifex installer. Re-running it rewrites this file.
[${NFV_SIG_REPO_ID}]
name=CentOS NFV SIG Open vSwitch and OVN, for Spinifex
baseurl=https://mirror.stream.centos.org/SIGs/10-stream/nfv/\$basearch/openvswitch-2/
enabled=1
gpgcheck=1
gpgkey=file://${NFV_SIG_KEY_FILE}
includepkgs=${NFV_SIG_INCLUDEPKGS}
EOF

    for repo in "$OL_UEK_REPO" "$OL_APPSTREAM_REPO" "$OL_KVM_UTILS_REPO" "$NFV_SIG_REPO_ID"; do
        $SUDO dnf -q makecache --repo "$repo" > /dev/null \
            || fatal "Repository $repo is unreachable. Check this host's network access and proxy settings, then re-run this installer"
    done
}

# Adds the QEMU excludes to AppStream, keeping any exclude already set there.
exclude_from_appstream() {
    local current pattern merged="" patterns
    current=$(dnf config-manager --dump "$OL_APPSTREAM_REPO" | sed -n 's/^excludepkgs = //p' | tr ',' ' ')
    # read, not an unquoted expansion, so the patterns are never globbed.
    read -ra patterns <<< "$current $OL_APPSTREAM_EXCLUDES"
    for pattern in "${patterns[@]}"; do
        case " $merged " in
            *" $pattern "*) ;;
            *) merged="${merged:+$merged }$pattern" ;;
        esac
    done
    $SUDO dnf config-manager --save --setopt="${OL_APPSTREAM_REPO}.excludepkgs=${merged// /,}" > /dev/null
}

# gpg_fingerprints <file> — the primary-key fingerprint of each key in file.
gpg_fingerprints() {
    local home
    home=$(mktemp -d)
    GNUPGHOME="$home" gpg --batch --with-colons --show-keys "$1" 2>/dev/null \
        | awk -F: '$1 == "pub" { want = 1; next } want && $1 == "fpr" { print $10; want = 0 }'
    rm -rf "$home"
}

install_nfv_sig_key() {
    if [ -f "$NFV_SIG_KEY_FILE" ] && [ "$(gpg_fingerprints "$NFV_SIG_KEY_FILE")" = "$NFV_SIG_KEY_FPR" ]; then
        $SUDO rpm --import "$NFV_SIG_KEY_FILE"
        return 0
    fi
    local tmp got
    tmp=$(mktemp)
    curl -fsSL "$NFV_SIG_KEY_URL" -o "$tmp" \
        || { rm -f "$tmp"; fatal "Cannot download the CentOS NFV SIG key from $NFV_SIG_KEY_URL"; }
    got=$(gpg_fingerprints "$tmp")
    if [ "$got" != "$NFV_SIG_KEY_FPR" ]; then
        rm -f "$tmp"
        fatal "The key at $NFV_SIG_KEY_URL has fingerprint ${got:-<unreadable>}, not the CentOS NFV SIG's $NFV_SIG_KEY_FPR. Refusing to trust it"
    fi
    $SUDO install -D -m 0644 "$tmp" "$NFV_SIG_KEY_FILE"
    rm -f "$tmp"
    $SUDO rpm --import "$NFV_SIG_KEY_FILE"
}

# Oracle ships kernel modules per kernel version. The unversioned package
# follows the newest kernel and leaves the running one without openvswitch.ko.
install_running_kernel_modules() {
    local pkg
    pkg="kernel-uek-modules-$(uname -r)"
    if rpm -q "$pkg" > /dev/null 2>&1; then
        return 0
    fi
    if ! $SUDO dnf -q list --available "$pkg" > /dev/null 2>&1; then
        fatal "$pkg, the modules for the running kernel, is no longer in the repositories. Run 'sudo dnf update kernel-uek', reboot into the new kernel, then re-run this installer"
    fi
    info "Installing $pkg (it pulls in linux-firmware, about 800 MB)..."
    $SUDO dnf install -y -q "$pkg" > /dev/null
}

load_kernel_modules() {
    local mod failed=""
    for mod in "$@"; do
        $SUDO modprobe "$mod" || failed="${failed:+$failed }$mod"
    done
    if [ -n "$failed" ]; then
        fatal "Cannot load kernel modules for the running kernel $(uname -r): $failed"
    fi
}

# --- Install AWS CLI ---
install_aws_cli() {
    stage "installing AWS CLI v2"
    if [ "${INSTALL_SPINIFEX_SKIP_AWS}" = "1" ]; then
        info "Skipping AWS CLI (INSTALL_SPINIFEX_SKIP_AWS=1)"
        return
    fi

    # The bundled installer links /usr/local/bin/aws, which OL's sudo secure_path
    # leaves out, so a PATH lookup alone misses it and reinstalls on every run.
    local aws_bin
    aws_bin=$(command -v aws 2>/dev/null || echo /usr/local/bin/aws)
    if [ -x "$aws_bin" ]; then
        info "AWS CLI already installed: $("$aws_bin" --version 2>&1 | head -1)"
        return
    fi

    info "Installing AWS CLI v2..."
    AWS_TMPDIR=$(mktemp -d)
    curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_ARCH}.zip" -o "$AWS_TMPDIR/awscliv2.zip"
    unzip -q "$AWS_TMPDIR/awscliv2.zip" -d "$AWS_TMPDIR"
    $SUDO "$AWS_TMPDIR/aws/install" --update > /dev/null
    rm -rf "$AWS_TMPDIR"

    info "AWS CLI installed: $(/usr/local/bin/aws --version 2>&1 | head -1)"
}

# --- Download tarball ---
download_spinifex() {
    stage "downloading/extracting spinifex release tarball"
    SPINIFEX_TMPDIR=$(mktemp -d)
    TARBALL="$SPINIFEX_TMPDIR/spinifex.tar.gz"

    # Local tarball override — skip download (for testing and air-gapped installs)
    if [ -n "$INSTALL_SPINIFEX_TARBALL" ]; then
        info "Using local tarball: $INSTALL_SPINIFEX_TARBALL"
        cp "$INSTALL_SPINIFEX_TARBALL" "$TARBALL"
        info "Extracting..."
        tar -xzf "$TARBALL" -C "$SPINIFEX_TMPDIR"
        EXTRACT_DIR="$SPINIFEX_TMPDIR"
        return
    fi

    if [ -n "$INSTALL_SPINIFEX_VERSION" ]; then
        DOWNLOAD_URL="${INSTALL_BASE_URL}/download/${INSTALL_SPINIFEX_VERSION}/${ARCH}"
        info "Downloading Spinifex $INSTALL_SPINIFEX_VERSION ($ARCH)..."
    else
        DOWNLOAD_URL="${INSTALL_BASE_URL}/download/${INSTALL_SPINIFEX_CHANNEL}/${ARCH}"
        info "Downloading Spinifex ($INSTALL_SPINIFEX_CHANNEL channel, $ARCH)..."
    fi

    HTTP_CODE=$(curl -fsSL -w '%{http_code}' -o "$TARBALL" "$DOWNLOAD_URL" 2>/dev/null) || true
    if [ ! -f "$TARBALL" ] || [ "$HTTP_CODE" -ge 400 ] 2>/dev/null; then
        rm -rf "$SPINIFEX_TMPDIR"
        fatal "Failed to download Spinifex from $DOWNLOAD_URL (HTTP $HTTP_CODE)"
    fi

    # Verify checksum if available
    CHECKSUM_URL="${DOWNLOAD_URL}.sha256"
    if curl -fsSL -o "$SPINIFEX_TMPDIR/checksum.sha256" "$CHECKSUM_URL" 2>/dev/null; then
        info "Verifying checksum..."
        EXPECTED=$(awk '{print $1}' "$SPINIFEX_TMPDIR/checksum.sha256")
        ACTUAL=$(sha256sum "$TARBALL" | awk '{print $1}')
        if [ "$EXPECTED" != "$ACTUAL" ]; then
            rm -rf "$SPINIFEX_TMPDIR"
            fatal "Checksum verification failed. Expected: $EXPECTED, Got: $ACTUAL"
        fi
        info "Checksum verified"
    else
        rm -rf "$SPINIFEX_TMPDIR"
        fatal "Checksum not available at $CHECKSUM_URL. Cannot verify download integrity."
    fi

    # Extract
    info "Extracting..."
    tar -xzf "$TARBALL" -C "$SPINIFEX_TMPDIR"
    EXTRACT_DIR="$SPINIFEX_TMPDIR"
}

# OL's sudo secure_path leaves out /usr/local/bin, so `sudo spx` would find
# nothing there. A link in /usr/bin changes sudo for nothing else on the host.
link_spx_for_sudo() {
    local sudo_path
    [ "${ISO_BUILD:-0}" = "1" ] && return 0
    command -v sudo > /dev/null 2>&1 || return 0
    sudo_path=$(sudo -n /usr/bin/env 2> /dev/null | sed -n 's/^PATH=//p') || true
    if [ -z "$sudo_path" ]; then
        warn "Cannot read the PATH sudo gives commands. If 'sudo spx' is not found, run: sudo ln -s /usr/local/bin/spx /usr/bin/spx"
        return 0
    fi
    case ":$sudo_path:" in *:/usr/local/bin:*) return 0 ;; esac
    if [ -e /usr/bin/spx ] || [ -L /usr/bin/spx ]; then
        [ "$(readlink /usr/bin/spx)" = /usr/local/bin/spx ] && return 0
        fatal "/usr/bin/spx exists and is not a link to /usr/local/bin/spx. sudo's PATH ($sudo_path) leaves out /usr/local/bin, so 'sudo spx' would run that file. Remove it and re-run this installer"
    fi
    $SUDO ln -s /usr/local/bin/spx /usr/bin/spx
    info "  /usr/bin/spx -> /usr/local/bin/spx (sudo's PATH leaves out /usr/local/bin)"
}

# Sets NBDKIT_PLUGINDIR from nbdkit itself; no guessed path is safe, since the
# daemon loads the plugin from wherever systemd.env says it is.
resolve_nbdkit_plugindir() {
    NBDKIT_PLUGINDIR=$(nbdkit --dump-config 2> /dev/null | sed -n 's/^plugindir=//p') || true
    [ -n "$NBDKIT_PLUGINDIR" ] \
        || fatal "Cannot find nbdkit's plugin directory: 'nbdkit --dump-config' printed no plugindir. Check that nbdkit is installed and runs, then re-run this installer"
}

# --- Place files ---
install_files() {
    stage "installing binaries and scripts"
    info "Installing files..."

    # Binary
    $SUDO install -m 0755 "$EXTRACT_DIR/spx" /usr/local/bin/spx
    info "  /usr/local/bin/spx"

    link_spx_for_sudo

    # nbdkit plugin
    resolve_nbdkit_plugindir
    $SUDO mkdir -p "$NBDKIT_PLUGINDIR"
    $SUDO install -m 0755 "$EXTRACT_DIR/nbdkit-viperblock-plugin.so" "$NBDKIT_PLUGINDIR/nbdkit-viperblock-plugin.so"
    info "  $NBDKIT_PLUGINDIR/nbdkit-viperblock-plugin.so"

    # Setup scripts
    $SUDO mkdir -p /usr/local/share/spinifex
    if [ -f "$EXTRACT_DIR/setup-ovn.sh" ]; then
        $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
        info "  /usr/local/share/spinifex/setup-ovn.sh"
    fi

    # Sourced, not run, by setup-ovn.sh and uninstall-spx.sh wherever they are
    # installed, including the ISO's /usr/local/bin/setup-ovn.sh.
    if [ -f "$EXTRACT_DIR/lib/distro.sh" ]; then
        $SUDO install -D -m 0644 "$EXTRACT_DIR/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh
        info "  /usr/local/share/spinifex/lib/distro.sh"
    fi

    # Install setup.sh itself so firstboot and future re-runs can find it at a
    # stable path on both ISO and curl|bash installs.
    if [ -f "$EXTRACT_DIR/setup.sh" ]; then
        $SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
        info "  /usr/local/share/spinifex/setup.sh"
    fi

    # Formation. Shipping these means a multi-node cluster can be formed from a
    # `curl | bash` install with no checkout, which is what a cloud operator has.
    # smoke-test.sh comes too: install-node.sh scp's it from its own directory for
    # the post-formation verification, so without it formation fails at the last step.
    for formation in install-node.sh smoke-test.sh; do
        if [ -f "$EXTRACT_DIR/$formation" ]; then
            $SUDO install -m 0755 "$EXTRACT_DIR/$formation" "/usr/local/share/spinifex/$formation"
            info "  /usr/local/share/spinifex/$formation"
        fi
    done

    # Teardown scripts. An install delivered by `curl | bash` with no way to
    # undo it is a gap in the product, not just in our tooling, and node-reset.sh
    # ships alongside because uninstall-spx.sh delegates its state teardown to it
    # rather than reimplementing the QEMU and JetStream waits.
    for teardown in node-reset.sh uninstall-spx.sh; do
        if [ -f "$EXTRACT_DIR/$teardown" ]; then
            $SUDO install -m 0755 "$EXTRACT_DIR/$teardown" "/usr/local/share/spinifex/$teardown"
            info "  /usr/local/share/spinifex/$teardown"
        fi
    done

    # microVM kernel + initramfs
    $SUDO install -d /usr/share/spinifex/microvm
    if [ -d "$EXTRACT_DIR/microvm" ]; then
        $SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
        info "  /usr/share/spinifex/microvm/*"
    fi
    install_sqs_store_image
    install_sqs_worker_image
}

# The SQS images ship beside the load balancer's, not inside it: an older
# installer copies microvm/* without -r and would stop at a subdirectory.
install_sqs_store_image() { install_sqs_microvm_image sqs-store "SQS store"; }
install_sqs_worker_image() { install_sqs_microvm_image sqs-worker "SQS worker"; }

# install_sqs_microvm_image <name> <label>
install_sqs_microvm_image() {
    local name=$1 label=$2 f
    local src="$EXTRACT_DIR/microvm-$name"
    local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/$name"
    # A previously installed image belongs to an older release, and this
    # release's daemon must not boot it. Only the two image files are removed.
    if [ ! -d "$src" ]; then
        remove_sqs_microvm_image "$dest" "$label"
        warn "This release has no $label microVM image; $label launches will be refused"
        return 0
    fi
    for f in vmlinuz initramfs.cpio.gz; do
        if [ ! -s "$src/$f" ]; then
            remove_sqs_microvm_image "$dest" "$label"
            fatal "The $label microVM image in this release is incomplete: $f is missing"
        fi
    done
    # The initramfs goes last, so an interrupted install leaves no pair the
    # daemon would boot: it refuses a launch while either file is missing.
    $SUDO install -d "$dest"
    $SUDO rm -f "$dest/initramfs.cpio.gz"
    $SUDO install -m 0644 "$src/vmlinuz" "$dest/vmlinuz"
    $SUDO install -m 0644 "$src/initramfs.cpio.gz" "$dest/initramfs.cpio.gz"
    info "  $dest/vmlinuz, $dest/initramfs.cpio.gz"
}

# Running VMs keep their already-loaded kernel; the files are read at boot.
remove_sqs_microvm_image() {
    [ -e "$1/initramfs.cpio.gz" ] || [ -e "$1/vmlinuz" ] || return 0
    $SUDO rm -f "$1/initramfs.cpio.gz" "$1/vmlinuz"
    warn "Removed the $2 microVM image left by an earlier install from $1"
}

# --- Create directories ---
create_directories() {
    stage "creating spinifex directory layout"
    info "Creating directories..."

    # Top-level directories (root-owned, group-readable by spinifex)
    $SUDO mkdir -p /etc/spinifex
    $SUDO chmod 0750 /etc/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex

    $SUDO mkdir -p /var/lib/spinifex
    $SUDO chmod 0750 /var/lib/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex

    # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/.
    # -n so a re-run replaces the link rather than creating one inside the
    # directory it already points at.
    if $SUDO test ! -e /var/lib/spinifex/config; then
        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/config
    fi

    # Symlink so services that write logs to BaseDir/logs/ use /var/log/spinifex/
    if $SUDO test ! -e /var/lib/spinifex/logs; then
        $SUDO ln -sfn /var/log/spinifex /var/lib/spinifex/logs
    fi

    $SUDO mkdir -p /var/log/spinifex
    $SUDO chmod 0775 /var/log/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex

    # /run/spinifex and /run/spinifex/nbd are declared via tmpfiles.d because
    # /run is tmpfs — direct mkdir doesn't survive reboot, and units have
    # ReadWritePaths= on these paths which fails namespace setup with ENOENT
    # if the dirs are absent at service start.
    _tmpf=$(mktemp)
    cat > "$_tmpf" <<'TMPEOF'
# Type  Path               Mode  User                 Group                Age
d       /run/spinifex      0770  root                 spinifex             -
d       /run/spinifex/nbd  0770  spinifex-viperblock  spinifex-viperblock  -
TMPEOF
    $SUDO install -m 0644 "$_tmpf" /etc/tmpfiles.d/spinifex.conf
    rm -f "$_tmpf"
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        # Live systems: materialise the runtime dirs immediately so a re-run of
        # setup.sh on an existing host has correct /run state without a reboot.
        $SUDO systemd-tmpfiles --create /etc/tmpfiles.d/spinifex.conf 2>/dev/null || true
    fi

    # Per-service config directories
    $SUDO mkdir -p /etc/spinifex/nats
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
    $SUDO chmod 0750 /etc/spinifex/nats

    $SUDO mkdir -p /etc/spinifex/predastore
    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
    $SUDO chmod 0750 /etc/spinifex/predastore

    # Northstar holds northstar.toml (bucket-scoped S3 creds, written 0600 by
    # `spx admin init`); fix_file_ownership reassigns it to spinifex-northstar.
    $SUDO mkdir -p /etc/spinifex/northstar
    $SUDO chown "spinifex-northstar:$SPINIFEX_GROUP" /etc/spinifex/northstar
    $SUDO chmod 0750 /etc/spinifex/northstar

    $SUDO mkdir -p /etc/spinifex/awsgw
    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw
    $SUDO chmod 0750 /etc/spinifex/awsgw

    # Viperblock's at-rest encryption key dir. 0750 group-traversable; the key
    # itself is set to root:spinifex 0640 by SetServiceOwnership because both
    # viperblockd (spinifex-viperblock) and the awsgw handlers (spinifex-gw)
    # load it.
    $SUDO mkdir -p /etc/spinifex/viperblock
    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/viperblock
    $SUDO chmod 0750 /etc/spinifex/viperblock

    # Per-service data directories
    $SUDO mkdir -p /var/lib/spinifex/nats
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
    $SUDO chmod 0700 /var/lib/spinifex/nats

    $SUDO mkdir -p /var/lib/spinifex/spinifex
    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
    $SUDO chmod 0700 /var/lib/spinifex/spinifex

    $SUDO mkdir -p /var/lib/spinifex/predastore
    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
    $SUDO chmod 0700 /var/lib/spinifex/predastore

    $SUDO mkdir -p /var/lib/spinifex/northstar
    $SUDO chown "spinifex-northstar:$SPINIFEX_GROUP" /var/lib/spinifex/northstar
    $SUDO chmod 0700 /var/lib/spinifex/northstar

    $SUDO mkdir -p /var/lib/spinifex/viperblock
    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
    $SUDO chmod 0700 /var/lib/spinifex/viperblock

    $SUDO mkdir -p /var/lib/spinifex/vpcd
    $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
    $SUDO chmod 0700 /var/lib/spinifex/vpcd

    $SUDO mkdir -p /var/lib/spinifex/awsgw
    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /var/lib/spinifex/awsgw
    $SUDO chmod 0700 /var/lib/spinifex/awsgw

    $SUDO mkdir -p /var/lib/spinifex/spinifex-ui
    $SUDO chown "spinifex-ui:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex-ui
    $SUDO chmod 0700 /var/lib/spinifex/spinifex-ui

    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/.
    # The test runs under $SUDO because the parent is 0700 spinifex-gw: an
    # unprivileged -e cannot stat through it and always reports "missing", so
    # every re-run reached the ln below. That ln then resolved through the
    # existing link and tried to create /etc/spinifex/spinifex, which
    # CreateServiceDirectories already owns — failing the whole install.
    if $SUDO test ! -e /var/lib/spinifex/awsgw/config; then
        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/awsgw/config
    fi

    # Service helper scripts (root-owned, group-executable by all service users)
    if [ -d "$EXTRACT_DIR/scripts" ]; then
        for script in "$EXTRACT_DIR"/scripts/*.sh; do
            $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
                "$script" "/var/lib/spinifex/$(basename "$script")"
            info "  /var/lib/spinifex/$(basename "$script")"
        done
    fi
}

# --- Generate systemd environment file ---
# Split out of create_directories so the `env` stage can be refreshed
# independently (e.g. by inject-bins.sh when the plugin path changes).
install_systemd_env() {
    stage "generating /etc/spinifex/systemd.env"

    # The same directory install_files placed the plugin in.
    resolve_nbdkit_plugindir

    $SUDO mkdir -p /etc/spinifex
    $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
# Generated by setup.sh — install-specific environment variables
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${NBDKIT_PLUGINDIR}/nbdkit-viperblock-plugin.so
EOF
    $SUDO chown "spinifex-viperblock:${SPINIFEX_GROUP:-spinifex}" /etc/spinifex/systemd.env
    $SUDO chmod 0640 /etc/spinifex/systemd.env
    info "Generated /etc/spinifex/systemd.env"
}

# --- Fix file ownership for upgrades from v1 ---
# Also invoked from firstboot via SETUP_STAGES=fixown to correct ownership of
# files that `spx admin init` wrote as root:root.
fix_file_ownership() {
    stage "fixing file ownership for privilege separation"
    # create_service_users normally sets this; if we're invoked via
    # SETUP_STAGES=fixown on a host that already has the group, users isn't
    # run and SPINIFEX_GROUP is unset — default it here.
    SPINIFEX_GROUP="${SPINIFEX_GROUP:-spinifex}"
    info "Fixing file ownership for privilege separation..."

    # Per-service data dirs — recursive chown so existing files are accessible
    for entry in \
        nats:spinifex-nats \
        predastore:spinifex-storage \
        northstar:spinifex-northstar \
        spinifex:spinifex-daemon \
        viperblock:spinifex-viperblock \
        vpcd:spinifex-vpcd \
        awsgw:spinifex-gw; do
        IFS=: read -r dir svc_user <<< "$entry"
        if [ -d "/var/lib/spinifex/$dir" ]; then
            $SUDO chown -R "$svc_user:$SPINIFEX_GROUP" "/var/lib/spinifex/$dir" \
                || fatal "Failed to set ownership on /var/lib/spinifex/$dir"
        fi
    done

    # Per-service config dirs — recursive chown
    if [ -d /etc/spinifex/nats ]; then
        $SUDO chown -R "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats \
            || fatal "Failed to set ownership on /etc/spinifex/nats"
    fi
    if [ -d /etc/spinifex/predastore ]; then
        $SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
            || fatal "Failed to set ownership on /etc/spinifex/predastore"
    fi
    if [ -d /etc/spinifex/northstar ]; then
        $SUDO chown -R "spinifex-northstar:$SPINIFEX_GROUP" /etc/spinifex/northstar \
            || fatal "Failed to set ownership on /etc/spinifex/northstar"
    fi
    if [ -d /etc/spinifex/awsgw ]; then
        $SUDO chown -R "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw \
            || fatal "Failed to set ownership on /etc/spinifex/awsgw"
    fi

    # Shared config files — root:spinifex with per-file modes
    for f in spinifex.toml master.key server.key; do
        if [ -f "/etc/spinifex/$f" ]; then
            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
                || fatal "Failed to set ownership on /etc/spinifex/$f"
            $SUDO chmod 0640 "/etc/spinifex/$f" \
                || fatal "Failed to set permissions on /etc/spinifex/$f"
        fi
    done
    for f in server.pem ca.pem; do
        if [ -f "/etc/spinifex/$f" ]; then
            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
                || fatal "Failed to set ownership on /etc/spinifex/$f"
            $SUDO chmod 0644 "/etc/spinifex/$f" \
                || fatal "Failed to set permissions on /etc/spinifex/$f"
        fi
    done

    # CA private key — root-only
    if [ -f /etc/spinifex/ca.key ]; then
        $SUDO chown root:root /etc/spinifex/ca.key \
            || fatal "Failed to set ownership on /etc/spinifex/ca.key"
        $SUDO chmod 0600 /etc/spinifex/ca.key \
            || fatal "Failed to set permissions on /etc/spinifex/ca.key"
    fi

    # Shared data dirs — root:spinifex 0770 (daemon + admin CLI write, services read).
    # chmod must be recursive so pre-existing files (e.g. imported images originally
    # written as 0600 by another user) become group-readable.
    for d in images amis volumes state; do
        if [ -d "/var/lib/spinifex/$d" ]; then
            $SUDO chown -R "root:$SPINIFEX_GROUP" "/var/lib/spinifex/$d" \
                || fatal "Failed to set ownership on /var/lib/spinifex/$d"
            $SUDO chmod -R u+rwX,g+rwX,o-rwx "/var/lib/spinifex/$d" \
                || fatal "Failed to set permissions on /var/lib/spinifex/$d"
            # setgid on directories only, so subdirectories a root run creates
            # keep the spinifex group instead of falling back to root and locking
            # the admin CLI out of the tree it is meant to write.
            $SUDO find "/var/lib/spinifex/$d" -type d -exec chmod g+s {} + \
                || fatal "Failed to set setgid on /var/lib/spinifex/$d"
        fi
    done

    info "File ownership updated"
}

# --- Run config migrations ---
run_migrations() {
    # Only run if spinifex is already initialized (config exists)
    if [ ! -f /etc/spinifex/spinifex.toml ]; then
        info "Fresh install detected, skipping migrations"
        return
    fi

    if [ "${INSTALL_SPINIFEX_SKIP_MIGRATE:-0}" = "1" ]; then
        info "INSTALL_SPINIFEX_SKIP_MIGRATE=1, skipping migrations"
        info "Run 'spx admin upgrade' manually to apply pending migrations"
        return
    fi

    info "Running config migrations..."
    $SUDO /usr/local/bin/spx admin upgrade --yes \
        || fatal "Config migration failed. See errors above."
}

# --- Install systemd units ---
install_systemd() {
    stage "installing systemd units"
    info "Installing systemd units..."

    if [ ! -d "$EXTRACT_DIR/systemd" ]; then
        fatal "Systemd unit files not found in tarball (expected systemd/ directory)"
    fi

    for unit in "$EXTRACT_DIR"/systemd/*; do
        $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
        info "  /etc/systemd/system/$(basename "$unit")"
    done

    # Reserve RAM + CPU priority for system.slice (sshd, journald, the operator)
    # so a maxed spinifex.slice cannot starve them — the "stay sshable" guarantee.
    # Generated here rather than shipped as a staged file because the packaging
    # globs flatten the systemd/ dir and would skip a nested drop-in directory;
    # this mirrors the sshd-keygen drop-in pattern in build-rootfs.sh. MemoryMin
    # is a guaranteed-unreclaimable floor, not a cap.
    $SUDO mkdir -p /etc/systemd/system/system.slice.d
    $SUDO tee /etc/systemd/system/system.slice.d/spinifex-reserve.conf > /dev/null << 'EOF'
[Slice]
MemoryMin=1G
CPUWeight=300
EOF
    info "  /etc/systemd/system/system.slice.d/spinifex-reserve.conf"

    # The plaintext Geneve guard is not persisted, so it goes in when pluto
    # starts at boot, before ovn-controller builds the tunnels. `-`: the guard
    # must never fail the unit, and the daemon re-asserts it each pass.
    if [ -x /usr/libexec/ipsec/pluto ]; then
        $SUDO mkdir -p /etc/systemd/system/ipsec.service.d
        $SUDO tee /etc/systemd/system/ipsec.service.d/spinifex-guard.conf > /dev/null << EOF
[Service]
ExecStartPost=-${IPSEC_STATE_HELPER} guard
EOF
        info "  /etc/systemd/system/ipsec.service.d/spinifex-guard.conf"
    fi

    # daemon-reload / enable require a running systemd — skip inside the ISO
    # chroot. Unit files are still dropped into place; firstboot enables them.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        info "Systemd units installed (ISO_BUILD=1, skipping daemon-reload/enable)"
        return
    fi
    $SUDO systemctl daemon-reload
    $SUDO systemctl enable spinifex.target
    # The unit file alone does not self-activate (WantedBy=timers.target only
    # takes effect once enabled) — without this the JetStream ENOSPC-latch
    # watchdog is inert and a full disk requires a manual restart forever.
    $SUDO systemctl enable --now spinifex-nats-watchdog.timer
    # Enabled, not started: its ConditionPathExists holds it inert until the
    # daemon writes a peer file, so this only decides that a node which has one
    # gets the policy back at boot, before any service opens a socket.
    $SUDO systemctl enable spinifex-firewall.service
    info "Systemd units installed and enabled (per-service users)"
}

# --- Install kernel module list ---
# The file covers later boots; modprobe covers this one.
install_kernel_modules() {
    stage "installing kernel module list"
    $SUDO install -d /etc/modules-load.d
    # shellcheck disable=SC2086  # one module per line
    printf '# Managed by spinifex setup.sh — do not edit.\n%s\n' $SPINIFEX_KERNEL_MODULES \
        | $SUDO tee /etc/modules-load.d/spinifex.conf > /dev/null
    $SUDO chmod 0644 /etc/modules-load.d/spinifex.conf
    # A chroot would load them into the build host's kernel.
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        # shellcheck disable=SC2086  # a whitespace-separated module list
        load_kernel_modules $SPINIFEX_KERNEL_MODULES
    fi
    info "Kernel modules: $SPINIFEX_KERNEL_MODULES (/etc/modules-load.d/spinifex.conf)"
}

# --- Install kernel tunables ---
# Predastore carries blob, meta and raft over QUIC, and quic-go asks for a 7 MiB
# UDP receive buffer. Left at the kernel default of 208 KiB it silently drops
# datagrams under load, which fails shard writes and corrupts guest volumes.
install_sysctl() {
    stage "installing kernel tunables"
    $SUDO install -d /etc/sysctl.d
    $SUDO tee /etc/sysctl.d/99-spinifex-net.conf > /dev/null << 'SYSCTL'
# Managed by spinifex setup.sh — do not edit.
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
SYSCTL
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        $SUDO sysctl -q --system 2>/dev/null || true
    fi
    info "Kernel tunables installed (net.core.rmem_max/wmem_max = 16MB)"
}

# --- Keep needrestart off the Spinifex, OVS and OVN units ---
# Guest QEMU runs in spinifex-daemon's cgroup, so needrestart charges QEMU's
# replaced libraries to the daemon and restarts it after every dpkg run, which
# never clears them and ends at the unit's start limit. Restarting the stack
# after an upgrade is the operator's call; needrestart still lists the units.
install_needrestart_conf() {
    stage "installing needrestart override"
    local dir="${DISTRO_ROOT:-}/etc/needrestart/conf.d"
    $SUDO install -d "$dir"
    $SUDO tee "$dir/90-spinifex.conf" > /dev/null << 'NEEDRESTART'
# Managed by spinifex setup.sh — edits are overwritten.
# Never restart these automatically; restart them by hand when it suits.
$nrconf{override_rc}{qr(^spinifex-)} = 0;
$nrconf{override_rc}{qr(^ovs-)} = 0;
$nrconf{override_rc}{qr(^ovsdb-)} = 0;
$nrconf{override_rc}{qr(^ovn-)} = 0;
$nrconf{override_rc}{qr(^openvswitch-)} = 0;
NEEDRESTART
    $SUDO chmod 0644 "$dir/90-spinifex.conf"
    info "needrestart will not restart the spinifex, OVS or OVN units"
}

# --- Install logrotate ---
install_logrotate() {
    stage "installing logrotate config"
    if [ -f "$EXTRACT_DIR/logrotate-spinifex" ]; then
        $SUDO install -m 0644 "$EXTRACT_DIR/logrotate-spinifex" /etc/logrotate.d/spinifex
    else
        warn "Logrotate config not found in tarball, skipping"
        return
    fi
    info "Logrotate config installed"
}

# --- Install udev rules ---
install_udev() {
    stage "installing udev rules"
    if [ ! -d "$EXTRACT_DIR/udev" ]; then
        return
    fi
    $SUDO install -d /etc/udev/rules.d
    for rule in "$EXTRACT_DIR"/udev/*; do
        $SUDO install -m 0644 "$rule" "/etc/udev/rules.d/$(basename "$rule")"
        info "  /etc/udev/rules.d/$(basename "$rule")"
    done
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        $SUDO udevadm control --reload-rules
        $SUDO udevadm trigger --subsystem-match=vfio 2>/dev/null || true
    fi
    info "udev rules installed"
}

# --- Upgrade handling ---
handle_upgrade() {
    if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
        warn "Spinifex services are running. Stopping for upgrade..."
        $SUDO systemctl stop spinifex.target
        RESTART_AFTER=true
    fi

    # Pre-tightening hosts have /run/spinifex/nbd as root:spinifex 0770, which
    # grants access to every spinifex-* service. While services are stopped,
    # bring it up to the new spinifex-viperblock:spinifex-viperblock 0770
    # policy so restart picks up the fresh group membership without a reboot.
    if [ -d /run/spinifex/nbd ]; then
        $SUDO chown spinifex-viperblock:spinifex-viperblock /run/spinifex/nbd 2>/dev/null || true
        $SUDO chmod 0770 /run/spinifex/nbd 2>/dev/null || true
    fi
}

restart_if_needed() {
    if [ "${RESTART_AFTER}" = "true" ]; then
        info "Restarting Spinifex services..."
        $SUDO systemctl start spinifex.target
    fi
}

# --- Print summary ---
print_summary() {
    INSTALLED_VERSION=$(/usr/local/bin/spx version 2>/dev/null || echo "unknown")

    echo ""
    echo "============================================"
    echo "  Spinifex installed successfully"
    echo "============================================"
    echo ""
    echo "  Version:      $INSTALLED_VERSION"
    echo "  Architecture: $ARCH"
    echo "  Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
    echo "  Binary:       /usr/local/bin/spx"
    echo "  Config:       /etc/spinifex/"
    echo "  Data:         /var/lib/spinifex/"
    echo "  Logs:         /var/log/spinifex/"
    echo ""
    echo "  Next steps:"
    echo ""
    echo "  1. Setup OVN networking:"
    echo "     If your WAN interface is already a bridge (e.g. br-wan):"
    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management"
    echo ""
    echo "     If your WAN is a physical NIC:"
    echo "       # Dedicated WAN NIC (not your SSH connection):"
    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
    echo ""
    echo "  2. Initialize:"
    echo "     sudo spx admin init --node node1 --nodes 1"
    echo ""
    echo "  3. Start services:"
    echo "     sudo systemctl start spinifex.target"
    echo ""
    echo "  4. Verify:"
    echo "     export AWS_PROFILE=spinifex"
    echo "     aws ec2 describe-instance-types"
    echo ""
}

# --- Configure host swap ---
# Provisions an 8G swapfile and lowers vm.swappiness so spinifex.slice
# (MemorySwapMax=100%) has a backing device for graceful degradation under
# memory pressure. Reverses the historical swap=0 assumption. Idempotent.
setup_swap() {
    stage "configuring host swap"

    # ISO build runs in a chroot — cannot swapon, and baking an 8G file into the
    # rootfs bloats the image. ISO hosts provision swap at firstboot instead.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        info "Swap setup skipped (ISO_BUILD=1; firstboot provisions swap)"
        return
    fi

    local swapfile=/swapfile size_mb=8192

    # Swap is a safety buffer, not a routine path: reclaim page cache first.
    $SUDO tee /etc/sysctl.d/99-spinifex-swap.conf > /dev/null << 'EOF'
# Spinifex: minimise swapping; swap backs spinifex.slice graceful degradation.
vm.swappiness = 10
EOF
    $SUDO chmod 0644 /etc/sysctl.d/99-spinifex-swap.conf
    $SUDO sysctl -q -w vm.swappiness=10 || true

    if swapon --show=NAME --noheadings 2>/dev/null | grep -qx "$swapfile"; then
        info "Swap already active ($swapfile)"
        return
    fi

    if [ ! -f "$swapfile" ]; then
        info "Creating ${size_mb}MiB $swapfile"
        $SUDO fallocate -l "${size_mb}M" "$swapfile" 2>/dev/null \
            || $SUDO dd if=/dev/zero of="$swapfile" bs=1M count="$size_mb" status=none
        $SUDO chmod 0600 "$swapfile"
        $SUDO mkswap "$swapfile" > /dev/null
    fi
    $SUDO swapon "$swapfile"
    grep -q "^$swapfile " /etc/fstab 2>/dev/null \
        || echo "$swapfile none swap sw 0 0" | $SUDO tee -a /etc/fstab > /dev/null
    info "Swap enabled: $swapfile (${size_mb}MiB), vm.swappiness=10"
}

# --- Main ---
main() {
    while [ $# -gt 0 ]; do
        case "$1" in
            --firewall=*) INSTALL_SPINIFEX_FIREWALL="${1#*=}" ;;
            --firewall)   INSTALL_SPINIFEX_FIREWALL="${2:-}"; shift ;;
            --channel=*)  INSTALL_SPINIFEX_CHANNEL="${1#*=}" ;;
            --channel)    INSTALL_SPINIFEX_CHANNEL="${2:-}"; shift ;;
            --version=*)  INSTALL_SPINIFEX_VERSION="${1#*=}" ;;
            --version)    INSTALL_SPINIFEX_VERSION="${2:-}"; shift ;;
            *) fatal "unknown option: $1" ;;
        esac
        shift
    done

    case "$INSTALL_SPINIFEX_CHANNEL" in
        latest|dev) ;;
        # Anything else is passed to the install endpoint as a literal tag, so a
        # typo would 404 on the download rather than here. Name the two channels.
        *) fatal "--channel must be 'latest' or 'dev', got: ${INSTALL_SPINIFEX_CHANNEL} (use --version for a specific tag)" ;;
    esac

    info "Spinifex installer"
    echo ""

    # Always needed: sudo resolution, OS/arch detection.
    setup_sudo
    resolve_firewall_mode
    detect_os
    detect_arch

    # Orchestration (stop running services, fix stale /run/spinifex/nbd perms)
    # only applies in full-install mode on a live host.
    if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then
        handle_upgrade
    fi

    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
    # Only download/extract when at least one such stage is enabled.
    if stage_enabled files || stage_enabled directories \
        || stage_enabled systemd || stage_enabled logrotate \
        || stage_enabled udev; then
        download_spinifex
    fi

    stage_enabled deps       && install_deps
    stage_enabled aws        && install_aws_cli
    stage_enabled dns        && hand_dns_to_resolved
    stage_enabled users      && create_service_users
    stage_enabled sudoers    && install_sudoers
    stage_enabled firewall   && install_firewall
    stage_enabled timesync   && install_chrony_conf
    stage_enabled files      && install_files
    stage_enabled directories && create_directories
    stage_enabled env        && install_systemd_env
    stage_enabled fixown     && fix_file_ownership
    stage_enabled systemd    && install_systemd
    stage_enabled sysctl     && install_sysctl
    stage_enabled needrestart && install_needrestart_conf
    stage_enabled modules    && install_kernel_modules
    stage_enabled logrotate  && install_logrotate
    stage_enabled udev       && install_udev
    stage_enabled swap       && setup_swap

    # Migrations are only safe on a live system (need a running NATS and a
    # persisted config file). Skip under ISO_BUILD and under any explicit
    # SETUP_STAGES filter that doesn't list migrations.
    if [ "${ISO_BUILD:-0}" != "1" ] && stage_enabled migrations; then
        run_migrations
    fi

    [ -n "${SPINIFEX_TMPDIR:-}" ] && rm -rf "$SPINIFEX_TMPDIR"

    # Post-install orchestration (service restart, summary, newgrp) only in
    # full-install mode on a live host.
    if [ "${ISO_BUILD:-0}" = "1" ] || [ -n "${SETUP_STAGES:-}" ]; then
        return
    fi

    restart_if_needed
    print_summary

    # Activate spinifex group membership in the invoking shell. Under curl|bash
    # stdin is the drained pipe, so redirect from /dev/tty and exec so the new
    # shell becomes the foreground process. Skip when we can't actually open
    # /dev/tty (CI, cloud-init, ssh -T — stat passes but open fails with ENXIO).
    if [ "${INSTALL_SPINIFEX_SKIP_NEWGRP:-0}" != "1" ] \
        && ! id -Gn 2>/dev/null | grep -qw "$SPINIFEX_GROUP" \
        && ( : </dev/tty ) 2>/dev/null; then
        echo ""
        echo "  Activating '$SPINIFEX_GROUP' group in a subshell — type 'exit' when done."
        echo ""
        exec newgrp "$SPINIFEX_GROUP" < /dev/tty
    fi
}

# Guarded so the package lint can source this file for APT_RUNTIME_PACKAGES
# without running the installer.
if [ "${INSTALL_SPINIFEX_LIB_ONLY:-0}" != "1" ]; then
    main "$@"
fi