spinifex/scripts/setup.sh

#!/bin/bash
# Spinifex binary installer
# Usage: curl -sfL https://install.mulgadc.com | bash
#        curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
#
# Environment variables:
#   INSTALL_SPINIFEX_CHANNEL   Release channel: latest (default), dev
#   INSTALL_SPINIFEX_VERSION   Pin to specific version (overrides channel)
#   INSTALL_SPINIFEX_TARBALL   Path to local tarball (skips download, for testing/air-gapped)
#   INSTALL_SPINIFEX_SKIP_APT  Set to 1 to skip apt dependency install
#   INSTALL_SPINIFEX_SKIP_AWS  Set to 1 to skip AWS CLI install
#   INSTALL_SPINIFEX_SKIP_NEWGRP  Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
#   ISO_BUILD                  Set to 1 when running inside a debootstrap chroot from the ISO
#                              builder: skip handle_upgrade/restart/migrations/newgrp/print_summary,
#                              skip systemctl daemon-reload + enable, short-circuit setup_sudo.
#   VERBOSE                    Set to 1 to echo "[setup] <stage>" before each top-level step.
#   SETUP_STAGES               Comma-separated subset of stages to run:
#                                deps, aws, users, sudoers, firewall, timesync, files,
#                                directories, env, systemd, sysctl, logrotate, udev,
#                                fixown, migrations
#                              Unset = run every stage appropriate for the current mode.

set -e

INSTALL_SPINIFEX_CHANNEL="${INSTALL_SPINIFEX_CHANNEL:-latest}"
INSTALL_BASE_URL="${INSTALL_BASE_URL:-https://install.mulgadc.com}"

# Referenced by both the sudoers grant and the daemon, so the paths are fixed here.
ENDPOINT_SYSCTL_HELPER="/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl"
IPSEC_STATE_HELPER="/usr/local/lib/spinifex/spinifex-set-ipsec-state"

# The firewall policy and the peer addresses it scopes cluster ports to. The
# policy ships with the node; the peers are written by spinifex-daemon, which is
# the only component that knows every node's resolved planes.
FIREWALL_DIR="/etc/spinifex/firewall"
FIREWALL_RULES="${FIREWALL_DIR}/spinifex.nft"
# shellcheck disable=SC2034  # part of the firewall dir layout; written by install-node.sh
FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
FIREWALL_MODE_FILE="${FIREWALL_DIR}/mode"
FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"

# Whether the policy is armed, not merely installed. The ISO is an appliance on
# hardware we define, so a default-deny policy is the product; a curl-to-bash
# install lands on a machine that was already doing something, where arming a
# drop policy uninvited can cut off services we know nothing about.
INSTALL_SPINIFEX_FIREWALL="${INSTALL_SPINIFEX_FIREWALL:-}"

# --- Colors ---
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

info()  { echo -e "${GREEN}[INFO]${NC} $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC} $*"; }
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }

stage() {
    [ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
    return 0
}

stage_enabled() {
    [ -z "${SETUP_STAGES:-}" ] && return 0
    case ",${SETUP_STAGES}," in
        *",$1,"*) return 0 ;;
        *) return 1 ;;
    esac
}

# --- Sudo setup ---
setup_sudo() {
    # Inside a debootstrap chroot we are already root and sudo may not be installed.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        SUDO=""
        return
    fi
    if [ "$(id -u)" -eq 0 ]; then
        SUDO=""
    elif command -v sudo >/dev/null 2>&1; then
        SUDO="sudo"
        if ! $SUDO -n true 2>/dev/null; then
            info "This installer requires sudo access for system-level operations"
            $SUDO true || fatal "Failed to obtain sudo access"
        fi
    else
        fatal "This script requires root or sudo access"
    fi
}

# --- OS detection ---
detect_os() {
    if [ ! -f /etc/os-release ]; then
        fatal "Cannot detect OS: /etc/os-release not found"
    fi

    . /etc/os-release

    case "$ID" in
        debian)
            if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
                fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
            fi
            ;;
        ubuntu)
            major="${VERSION_ID%%.*}"
            if [ "$major" -lt 24 ] 2>/dev/null; then
                fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
            fi
            ;;
        *)
            fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 24.04+"
            ;;
    esac

    info "Detected OS: $PRETTY_NAME"
}

# --- Architecture detection ---
detect_arch() {
    MACHINE=$(uname -m)
    case "$MACHINE" in
        x86_64)
            ARCH="amd64"
            QEMU_PACKAGES="qemu-system-x86"
            AWS_ARCH="x86_64"
            ;;
        aarch64|arm64)
            ARCH="arm64"
            QEMU_PACKAGES="qemu-system-arm"
            AWS_ARCH="aarch64"
            ;;
        *)
            fatal "Unsupported architecture: $MACHINE. Spinifex requires x86_64 or aarch64"
            ;;
    esac

    info "Detected architecture: $MACHINE ($ARCH)"
}

# --- Create per-service system users ---
create_service_users() {
    stage "creating spinifex group and per-service users"
    SPINIFEX_GROUP="spinifex"

    # Create shared group
    if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
        $SUDO groupadd --system "$SPINIFEX_GROUP"
    fi

    # Producer-typed access group for viperblock's runtime resources
    # (/run/spinifex/nbd socket dir). Viperblock writes as owner; daemon reads
    # via supplementary membership; every other spinifex-* service is excluded.
    if ! getent group spinifex-viperblock > /dev/null 2>&1; then
        $SUDO groupadd --system spinifex-viperblock
    fi

    # Create per-service users with correct home directories
    declare -A SERVICE_HOMES=(
        [nats]="/var/lib/spinifex/nats"
        [gw]="/var/lib/spinifex/awsgw"
        [daemon]="/var/lib/spinifex/spinifex"
        [storage]="/var/lib/spinifex/predastore"
        [northstar]="/var/lib/spinifex/northstar"
        [viperblock]="/var/lib/spinifex/viperblock"
        [vpcd]="/var/lib/spinifex"
        [ui]="/var/lib/spinifex"
    )
    for svc in nats gw daemon storage northstar viperblock vpcd ui; do
        local user="spinifex-${svc}"
        if ! id "$user" > /dev/null 2>&1; then
            $SUDO useradd --system --no-create-home \
                --home-dir "${SERVICE_HOMES[$svc]}" \
                --gid "$SPINIFEX_GROUP" \
                --shell /usr/sbin/nologin \
                "$user"
        fi
    done

    # Add invoking user to spinifex group for admin CLI access.
    # Skip under ISO_BUILD: the chroot has no invoking user (tf-user,
    # whoever ran `sudo make`, etc. don't exist in the rootfs). The ISO's
    # interactive 'spinifex' login account is created later in Phase 4 of
    # build-rootfs.sh with spinifex as its primary gid. In curl|bash mode
    # guard against a stale/missing SUDO_USER too.
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        ADMIN_USER="${SUDO_USER:-$(whoami)}"
        if [ "$ADMIN_USER" != "root" ] && id -u "$ADMIN_USER" > /dev/null 2>&1; then
            $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
        fi
    fi

    # KVM access for daemon
    if getent group kvm > /dev/null 2>&1; then
        $SUDO usermod -aG kvm spinifex-daemon
    fi

    # Daemon consumes viperblock's NBD socket — join the producer-typed group.
    $SUDO usermod -aG spinifex-viperblock spinifex-daemon

    info "Service users created (spinifex-{nats,gw,daemon,storage,northstar,viperblock,vpcd,ui})"
}

# --- Install scoped sudoers rules ---
# sudo-rs (Ubuntu's default sudo since 25.10) implements only a subset of
# Defaults and rejects an unknown one outright, so pam_session must be omitted
# there rather than emitted and ignored.
sudo_is_sudo_rs() {
    sudo --version 2>/dev/null | head -1 | grep -qi 'sudo-rs'
}

# The daemon only ever sets rp_filter and accept_local on an endpoint it just
# created. A fixed-verb helper keeps that grant expressible without a wildcard,
# which sudo-rs forbids inside command arguments.
install_endpoint_sysctl_helper() {
    $SUDO install -d -m 0755 /usr/local/lib/spinifex
    $SUDO tee "${ENDPOINT_SYSCTL_HELPER}" > /dev/null << 'HELPER'
#!/bin/sh
# Sets one of two per-endpoint sysctls for the asymmetric IMDS reply path.
# Interface, key and value are all constrained: this runs as root under a
# NOPASSWD grant, so it must never pass an arbitrary key through to sysctl.
set -eu

if [ "$#" -ne 3 ]; then
    echo "usage: $0 <interface> <rp_filter|accept_local> <0|1>" >&2
    exit 2
fi

iface=$1
key=$2
value=$3

# Reject anything that is not a plain interface name, so the key cannot be
# steered elsewhere in the sysctl tree with a "." or a "/".
case "${iface}" in
    ''|*[!A-Za-z0-9_-]*) echo "invalid interface: ${iface}" >&2; exit 2 ;;
esac

case "${key}" in
    rp_filter|accept_local) ;;
    *) echo "key not permitted: ${key}" >&2; exit 2 ;;
esac

case "${value}" in
    0|1) ;;
    *) echo "value not permitted: ${value}" >&2; exit 2 ;;
esac

exec sysctl -qw "net.ipv4.conf.${iface}.${key}=${value}"
HELPER
    $SUDO chown root:root "${ENDPOINT_SYSCTL_HELPER}"
    $SUDO chmod 0755 "${ENDPOINT_SYSCTL_HELPER}"
}

# The daemon turns OVN IPsec on and off as the cluster topology resolves, so it
# needs unit control. A NOPASSWD systemctl grant takes an arbitrary unit name and
# is root-equivalent, so the units are fixed here and only the state is an input.
install_ipsec_state_helper() {
    $SUDO install -d -m 0755 /usr/local/lib/spinifex
    $SUDO tee "${IPSEC_STATE_HELPER}" > /dev/null << 'HELPER'
#!/bin/sh
# Turns the host's OVN IPsec services on or off. Runs as root under a NOPASSWD
# grant, so it must never act on a unit the caller names.
set -eu

if [ "$#" -ne 1 ]; then
    echo "usage: $0 <on|off>" >&2
    exit 2
fi

# mask/unmask rather than disable/enable. The daemon's unit sets
# ProtectSystem=full, so /etc is read-only in the namespace its children inherit,
# and disabling a unit that also ships a SysV script shells out to update-rc.d
# there and fails. Masking is done by PID 1 over D-Bus and is unaffected.

# ovs-monitor-ipsec execs the strongSwan starter itself rather than going through
# this unit, so leaving it enabled only contends for UDP 500/4500. Off in both
# states. Absent on a host without strongswan, which is not an error.
systemctl mask --now strongswan-starter.service >/dev/null 2>&1 || true

case "$1" in
    on)
        systemctl unmask openvswitch-ipsec.service
        exec systemctl start openvswitch-ipsec.service
        ;;
    off)
        exec systemctl mask --now openvswitch-ipsec.service
        ;;
    *)
        echo "state not permitted: $1" >&2
        exit 2
        ;;
esac
HELPER
    $SUDO chown root:root "${IPSEC_STATE_HELPER}"
    $SUDO chmod 0755 "${IPSEC_STATE_HELPER}"
}

# Every port sshd actually listens on, so a host hardened onto a non-standard
# port is not locked out by its own firewall. Reads the Include drop-ins too,
# since Debian's default config ends with one. Falls back to 22, which is what
# sshd itself does when no Port is set.
sshd_ports() {
    _cfg="/etc/ssh/sshd_config"
    _files="$_cfg"
    if [ -r "$_cfg" ]; then
        for _inc in $(awk '$1 == "Include" { $1 = ""; print }' "$_cfg" 2>/dev/null); do
            for _f in $_inc; do
                [ -r "$_f" ] && _files="$_files $_f"
            done
        done
    fi

    # shellcheck disable=SC2086
    _ports=$(awk '$1 == "Port" && $2 ~ /^[0-9]+$/ && $2+0 > 0 && $2+0 < 65536 { print $2 }' \
        $_files 2>/dev/null | sort -un | tr '\n' ' ')
    [ -n "$_ports" ] || _ports="22"

    _out=""
    for _p in $_ports; do
        if [ -z "$_out" ]; then _out="$_p"; else _out="$_out, $_p"; fi
    done
    printf '%s' "$_out"
}

# The policy is written here rather than shipped in the tarball so the ISO chroot
# and the ansible path get an identical file without a packaging step. Installed
# but NOT enabled: the peer set is empty until spinifex-daemon resolves the
# cluster's planes, and a drop policy with no peers would break formation.
install_firewall() {
    stage "installing host firewall policy"
    info "Installing host firewall policy..."

    $SUDO install -d -m 0755 "$FIREWALL_DIR" /usr/local/lib/spinifex
    $SUDO tee "$FIREWALL_RULES" > /dev/null << 'RULES'
#!/usr/sbin/nft -f
# Spinifex host firewall. Managed by scripts/setup.sh — edits are overwritten.
#
# Only `table inet spinifex_filter` is created, deleted and replaced. vpcd writes
# MASQUERADE and per-EIP FORWARD ACCEPT rules into the `ip nat` and `ip filter`
# tables and reinstalls them only when the service starts, so anything that
# flushes the whole ruleset breaks elastic IPs silently until the next restart.
#
# INPUT only. nftables runs every table registered on a hook and any drop is
# final, so a forward-hook policy here could not be rescued by vpcd's ACCEPTs in
# the other table. OUTPUT is untouched: the IMDS reply path egresses under
# per-tap policy routing.

include "/etc/spinifex/firewall/local.nft"
include "/etc/spinifex/firewall/open-ports.nft"
include "/etc/spinifex/firewall/peers.nft"
include "/etc/spinifex/firewall/custom.nft"

# Create-then-delete so the replace is atomic and the delete cannot fail on a
# first run. nft applies the whole file as one transaction or none of it.
table inet spinifex_filter
delete table inet spinifex_filter

table inet spinifex_filter {
    chain input {
        type filter hook input priority filter; policy drop;

        # First, or every reply to a connection this node opened is dropped.
        ct state established,related accept
        ct state invalid drop
        iif lo accept

        # Guest metadata and VPC DNS terminate on per-ENI OVS internal ports in
        # the host netns, so guest traffic to 169.254.169.254 and .253 arrives
        # here. Without this, cloud-init, instance-role credentials and all
        # guest DNS break. The ports exist only while instances run, so this
        # matches the prefix rather than an enumerated list.
        #
        # Scoped to the two link-local endpoints and their ports, not a blanket
        # accept: the OVS demux delivers any guest packet to 169.254.169.254/.253
        # onto an ime-* port regardless of L4 port, so `iifname "ime-*" accept`
        # would expose every wildcard-bound host service (sshd, the AWS gateway,
        # northstar, the UI) to any tenant. IMDS is HTTP on .254:80; VPC DNS is
        # 53 on .253. Everything else on these ports falls through to policy drop.
        iifname "ime-*" ip daddr 169.254.169.254 tcp dport 80 accept
        iifname "ime-*" ip daddr 169.254.169.253 udp dport 53 accept
        iifname "ime-*" ip daddr 169.254.169.253 tcp dport 53 accept

        # PMTUD is not optional under a Geneve overlay.
        icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
        icmpv6 type { echo-request, destination-unreachable, packet-too-big, time-exceeded, parameter-problem, nd-neighbor-solicit, nd-neighbor-advert, nd-router-solicit, nd-router-advert } accept

        # A broadcast DHCP reply does not reliably match an established
        # conntrack entry, so the uplink and external-pool leases need this.
        udp sport 67 udp dport 68 accept

        # Scoped by $trusted_ssh_peers from custom.nft, which the installer
        # creates once and never rewrites. That default accepts every source,
        # so a node is no less reachable than before until an operator narrows it.
        ip saddr $trusted_ssh_peers tcp dport $spinifex_ssh_ports accept

        # Public plane. 9999 is also how every guest agent (lb, eks, ecs, rds)
        # reaches its control plane over br-mgmt: they speak SigV4 to the AWS
        # gateway, never NATS, so no cluster port needs to face the guests.
        # 443 has no listener yet and is held for the nginx TLS edge, so the
        # rollout does not need a firewall change on every node.
        # nfproto ipv4: these accepts carry no source scope, so without a family
        # qualifier they would also serve on any global IPv6 a node autoconfigures
        # from an upstream RA. The scoped rules are v4-only via `ip saddr` already.
        meta nfproto ipv4 tcp dport { 443, 3000, 8443, 9999 } accept

        # Transiently open to any source, for cluster formation. A node dialling
        # in to join is not a peer yet, so the peer-scoped rule below cannot let
        # it in; `spx admin init` opens the formation port here for the length of
        # the formation window and closes it again afterwards. Normally holds
        # only the sentinel, which no packet can match.
        meta nfproto ipv4 tcp dport $spinifex_open_ports accept

        # 53 is open on every node, deliberately: northstar serves public
        # authoritative DNS and binds its advertise address. A node running no
        # public zone answers here too, which is the accepted cost of not
        # templating the policy per node.
        meta nfproto ipv4 tcp dport 53 accept
        meta nfproto ipv4 udp dport 53 accept

        # Cluster plane, peer-scoped. On a single-NIC node the planes collapse
        # onto the public address, which is why these are peer addresses rather
        # than an interface or a CIDR. 4432 stays here rather than joining the
        # public rule above: outside formation it is the daemon cluster manager,
        # whose /health and /local/* routes report node topology, service
        # inventory and running instances to anyone who asks. 8660 is
        # predastore's unauthenticated /healthz and /readyz, which name the
        # peers a gate cannot reach, so it belongs here for the same reason.
        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
        ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept

        # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
        # kernel UDP-tunnel socket, so encapsulated packets are delivered
        # locally and pass this hook before the OVS vport sees them.
        ip saddr $spinifex_encap_peers udp dport { 6081, 500, 4500 } accept
        ip saddr $spinifex_encap_peers meta l4proto esp accept

        # Rate-limited so a scan cannot fill the journal. This is the only way
        # to tell a policy gap from an application fault after the fact.
        limit rate 5/minute burst 10 packets log prefix "spinifex-fw drop: " level info
    }
}
RULES
    $SUDO chmod 0644 "$FIREWALL_RULES"
    info "  $FIREWALL_RULES"

    # Host-local facts the policy needs that the daemon has no view of. Separate
    # from peers.nft because the daemon rewrites that file and would drop this.
    # Refreshed on every setup.sh run, so an sshd port change is picked up by an
    # upgrade rather than needing the file edited by hand.
    _ssh_ports="$(sshd_ports)"
    $SUDO tee "$FIREWALL_LOCAL" > /dev/null << LOCAL
# Managed by scripts/setup.sh — edits are overwritten on upgrade.
define spinifex_ssh_ports = { ${_ssh_ports} }
LOCAL
    $SUDO chmod 0644 "$FIREWALL_LOCAL"
    info "  $FIREWALL_LOCAL (sshd ports: ${_ssh_ports})"

    # Reset closed on every run, so an install or upgrade that interrupts a
    # formation window cannot leave the port open indefinitely.
    $SUDO tee "$FIREWALL_OPEN" > /dev/null << 'OPENPORTS'
# Managed by spinifex-firewall-apply. Rewritten for the formation window only.
define spinifex_open_ports = { 0 }
OPENPORTS
    $SUDO chmod 0644 "$FIREWALL_OPEN"
    info "  $FIREWALL_OPEN (closed)"

    # Operator-owned, so it is created once and never rewritten: an upgrade, a
    # re-install and a node reset all leave an edited file alone. Never write it
    # empty — an undefined variable fails the whole ruleset, not just its rule.
    if [ ! -e "$FIREWALL_CUSTOM" ]; then
        $SUDO tee "$FIREWALL_CUSTOM" > /dev/null << 'CUSTOM'
# Operator-owned host firewall settings. setup.sh creates this file once and
# never overwrites it, so edits here survive upgrades, re-installs and resets.
#
# Check a change without applying it, then apply it:
#   sudo nft -c -f /etc/spinifex/firewall/spinifex.nft
#   sudo nft -f  /etc/spinifex/firewall/spinifex.nft
#
# Check spinifex.nft, not this file: on its own this parses as a bare define
# and proves nothing, because the rule that uses the variable lives there.
#
# Sources allowed to reach SSH. 0.0.0.0/0 accepts from anywhere; narrow it to
# your management networks. IPv4 only — sshd reached over IPv6 is not matched.
define trusted_ssh_peers = { 0.0.0.0/0 }
CUSTOM
        $SUDO chmod 0644 "$FIREWALL_CUSTOM"
        info "  $FIREWALL_CUSTOM (created — ssh open to every source)"
    else
        info "  $FIREWALL_CUSTOM (already present, left unchanged)"
    fi

    # Armed or merely installed. The daemon reads this when the config carries no
    # explicit firewall_enabled, which is the normal case on both install paths.
    printf '%s\n' "$INSTALL_SPINIFEX_FIREWALL" | $SUDO tee "$FIREWALL_MODE_FILE" > /dev/null
    $SUDO chmod 0644 "$FIREWALL_MODE_FILE"
    info "  $FIREWALL_MODE_FILE ($INSTALL_SPINIFEX_FIREWALL)"

    $SUDO tee "$FIREWALL_APPLY" > /dev/null << 'APPLY'
#!/bin/sh
# Applies the spinifex host firewall. Fails without changing the ruleset when
# the peer file is missing, so a node whose daemon has not yet resolved cluster
# membership is left reachable rather than cut off from a cluster it cannot name.
set -eu

RULES="/etc/spinifex/firewall/spinifex.nft"
PEERS="/etc/spinifex/firewall/peers.nft"
OPEN="/etc/spinifex/firewall/open-ports.nft"

# 0 is a sentinel that keeps the set non-empty, because nft rejects an empty
# one. No packet can match it: the kernel has no destination port 0.
write_open_ports() {
    _tmp=$(mktemp)
    printf '%s\n' \
        '# Managed by spinifex-firewall-apply. Rewritten for the formation window only.' \
        "define spinifex_open_ports = { $1 }" > "$_tmp"
    install -m 0644 "$_tmp" "$OPEN"
    rm -f "$_tmp"
}

# set-peers reads two comma-separated address lists on stdin (cluster peers,
# then encap peers) and rewrites the peer file before applying. Every address is
# re-validated here as a bare dotted quad: this runs as root under a NOPASSWD
# grant, so a caller must not be able to inject nft syntax through it.
# disable removes spinifex's own table and the peer file, leaving every other
# table alone. Idempotent, so it is safe on a node that never had the policy.
# Deliberately ahead of the policy-file check below: turning the firewall off is
# the recovery path, and refusing to run it because the policy is missing leaves
# an operator with a node they can neither arm nor disarm.
if [ "${1:-}" = "disable" ]; then
    nft delete table inet spinifex_filter 2>/dev/null || true
    rm -f "$PEERS"
    systemctl disable --now spinifex-firewall.service >/dev/null 2>&1 || true
    exit 0
fi

# Every remaining verb writes the policy, so it has to exist. Failing here
# changes nothing, which leaves a node whose daemon has not yet resolved cluster
# membership reachable rather than cut off from a cluster it cannot name.
[ -r "$RULES" ] || { echo "no firewall policy at $RULES" >&2; exit 1; }

# open-port and close-port hold a single port open to any source while a cluster
# forms. One slot, not a list: the only caller is `spx admin init` opening its
# formation port, and a set that can only ever hold one entry cannot be grown
# into a general-purpose hole in the policy.
if [ "${1:-}" = "open-port" ] || [ "${1:-}" = "close-port" ]; then
    port="${2:-}"
    case "$port" in
        '' | *[!0-9]*) echo "port must be a number: '$port'" >&2; exit 2 ;;
    esac
    [ "$port" -gt 0 ] && [ "$port" -lt 65536 ] || {
        echo "port out of range: $port" >&2
        exit 2
    }

    if [ "$1" = "open-port" ]; then write_open_ports "0, $port"; else write_open_ports "0"; fi

    # Nothing to reload on a node that is installed but not armed: the table is
    # not loaded, so every port is already open and the file is enough.
    [ -r "$PEERS" ] || exit 0
    exec nft -f "$RULES"
fi

if [ "${1:-}" = "set-peers" ]; then
    read -r peers_in || peers_in=""
    read -r encap_in || encap_in=""

    # Octet ranges are checked, not just the shape: nft rejects 10.0.0.999 and
    # the whole transaction with it, which would leave a peer file that fails on
    # every boot. A loop, not a pipeline — a rejection in a `while read` subshell
    # cannot fail the function.
    OCTET='(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
    validate() {
        _out=""
        for a in $(echo "$1" | tr ',' ' '); do
            echo "$a" | grep -Eq "^${OCTET}(\.${OCTET}){3}$" || {
                echo "address not permitted: $a" >&2
                return 2
            }
            if [ -z "$_out" ]; then _out="$a"; else _out="$_out, $a"; fi
        done
        printf '%s' "$_out"
    }

    peers=$(validate "$peers_in") || exit 2
    encap=$(validate "$encap_in") || exit 2
    [ -n "$peers" ] && [ -n "$encap" ] || {
        echo "set-peers needs a non-empty peer and encap list" >&2
        exit 2
    }

    tmp=$(mktemp)
    bak=$(mktemp)
    printf '%s\n' \
        '# Managed by spinifex-daemon. Regenerated from cluster membership.' \
        "define spinifex_peers = { $peers }" \
        "define spinifex_encap_peers = { $encap }" > "$tmp"
    if [ -r "$PEERS" ]; then cp "$PEERS" "$bak"; fi
    install -m 0644 "$tmp" "$PEERS"

    # Roll the peer file back if the ruleset will not load, so the boot-time
    # unit never inherits a file already known to fail.
    if ! nft -f "$RULES"; then
        if [ -s "$bak" ]; then install -m 0644 "$bak" "$PEERS"; else rm -f "$PEERS"; fi
        rm -f "$tmp" "$bak"
        exit 1
    fi
    rm -f "$tmp" "$bak"

    # The policy is loaded, so make it survive a reboot too. `disable` turns this
    # unit off, and forming a cluster means disabling the firewall on every node
    # first; without this, re-arming would look complete while silently leaving
    # the node bare after its next boot. Idempotent and quiet in the normal case.
    systemctl enable spinifex-firewall.service >/dev/null 2>&1 || true
    exit 0
fi

if [ ! -r "$PEERS" ]; then
    echo "no peer file at $PEERS: the daemon writes it once cluster membership" >&2
    echo "resolves. Ruleset left unchanged." >&2
    exit 1
fi

exec nft -f "$RULES"
APPLY
    $SUDO chown root:root "$FIREWALL_APPLY"
    $SUDO chmod 0755 "$FIREWALL_APPLY"
    info "  $FIREWALL_APPLY"

    # Say which state ssh is in either way. A node whose allowlist was never
    # narrowed reads as configured otherwise, and that is the whole risk here.
    _ssh_srcs="$(grep -o '{[^}]*}' "$FIREWALL_CUSTOM" 2>/dev/null | tr -d '{}' | tr -s ' ')"
    case "$_ssh_srcs" in
        *0.0.0.0/0*) info "  ssh (${_ssh_ports}) accepted from ANY source — narrow \$trusted_ssh_peers in $FIREWALL_CUSTOM" ;;
        ?*) info "  ssh (${_ssh_ports}) accepted from:${_ssh_srcs}" ;;
    esac

    if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
        info "Host firewall installed and armed (applied by spinifex-daemon once peers resolve)"
    else
        info "Host firewall installed but NOT armed — this machine may already be"
        info "running services a default-deny policy would cut off."
        info "  Allowed if armed: ssh (${_ssh_ports}), 53, 443, 3000, 8443, 9999"
        info "  Everything else on this host stops accepting new connections."
        info "  Arm it later by setting network.firewall_enabled = true in"
        info "  /etc/spinifex/spinifex.toml, or re-run this installer with --firewall=on"
    fi
}

# Debian's stock chrony.conf carries `makestep 1 3`: step the clock only for the
# first three updates, then slew at ~83us/s, which is roughly 7 seconds of
# correction per day. A node that burns those three before the network is usable
# then crawls toward correct time for days, and SigV4 rejects its requests the
# whole time. A drop-in rather than an edit to chrony.conf, which is a dpkg
# conffile and would prompt on every future upgrade.
install_chrony_conf() {
    stage "installing chrony drop-in"

    if [ ! -d /etc/chrony/conf.d ]; then
        warn "No /etc/chrony/conf.d — skipping chrony drop-in"
        return
    fi

    $SUDO tee /etc/chrony/conf.d/spinifex.conf > /dev/null << 'CHRONY'
# Managed by scripts/setup.sh — edits are overwritten.
# Step at any offset over a second, however many times it takes. The clock can
# jump backwards under load, which is the accepted cost: everything
# latency-sensitive here uses the monotonic clock, and a node minutes out of
# step fails every signed request until it converges.
makestep 1 -1
CHRONY
    $SUDO chmod 0644 /etc/chrony/conf.d/spinifex.conf
    info "  /etc/chrony/conf.d/spinifex.conf"
}

install_sudoers() {
    stage "installing scoped sudoers rules"
    install_endpoint_sysctl_helper
    install_ipsec_state_helper

    if sudo_is_sudo_rs; then
        $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
# pam_session is omitted: this host runs sudo-rs, which does not implement it.
SUDOERS
    else
        $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
# No PAM session for the service users. Every sudo call otherwise logs
# "pam_limits: Could not set limit for 'core'": the units set LimitCORE=0 and
# their CapabilityBoundingSet omits CAP_SYS_RESOURCE, so pam_limits cannot apply
# the limits.conf default and warns. It also means sudo children inherit the
# unit's rlimits rather than the system defaults, which is the RG-9 intent.
Defaults:spinifex-daemon !pam_session
SUDOERS
    fi

    $SUDO tee -a /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'

# The OVS/OVN client tools are deliberately absent. They do all their work over
# control sockets that setup-ovn.sh group-owns to `spinifex`, so they run as the
# service user. Granting them would be worse than pointless: a NOPASSWD rule for
# them takes unrestricted args, and ovs-vsctl/ovn-nbctl/ovn-appctl all accept
# --log-file=PATH, which writes a root-owned file wherever the caller points it.
#
# ovs-ofctl is the exception that keeps its grant: it talks to a per-bridge
# <bridge>.mgmt socket created by ovs-vswitchd when the bridge appears, including
# bridges spinifex creates at runtime, so it cannot be group-owned up front.

# spinifex-vpcd has no rules at all. Its ip/iptables/sysctl/arping work is done
# under the CAP_NET_ADMIN and CAP_NET_RAW its unit grants ambiently, which the
# kernel passes to each child on exec. Those grants were also root-equivalent:
# `sudo ip netns exec <ns> /bin/sh` is a root shell.

# Spinifex daemon: tap devices and OpenFlow rules. Its unit holds no
# CAP_NET_ADMIN, so unlike vpcd it cannot drop these. ovs-ofctl installs the
# per-tap IMDS datapath flows on br-imds; the sysctl helper sets the
# per-endpoint rp_filter/accept_local the asymmetric reply path needs.
spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-ofctl
SUDOERS

    # Separate append so the prose above can stay in a quoted heredoc: it carries
    # backticks that an expanding one would run as command substitution.
    $SUDO tee -a /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
spinifex-daemon ALL=(root) NOPASSWD: ${ENDPOINT_SYSCTL_HELPER}
spinifex-daemon ALL=(root) NOPASSWD: ${IPSEC_STATE_HELPER}
spinifex-daemon ALL=(root) NOPASSWD: ${FIREWALL_APPLY}
SUDOERS
    $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
    info "Scoped sudoers rules installed for spinifex-daemon (spinifex-vpcd needs none)"
}

# --- Install apt dependencies ---
# NOTE: Runtime deps must stay in sync with the ISO package list at
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
# remove, or change a runtime package here, review that file too — drift means
# the ISO and `curl | bash` paths install different software.
#
# Hoisted out of the apt-get call so scripts/lint-apt-packages.sh can resolve the
# same list against each supported suite. The arch-specific qemu package is
# added by install_apt_deps from detect_arch.
APT_RUNTIME_PACKAGES="nbdkit
qemu-utils gdisk ovmf qemu-efi-aarch64 less
libvirt-daemon-system libvirt-clients
pciutils
jq curl iproute2 ethtool netcat-openbsd wget unzip xz-utils file
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
chrony nftables"

install_apt_deps() {
    stage "installing apt dependencies"
    if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
        info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
    else
        info "Installing system dependencies..."
        $SUDO apt-get update -qq

        # Unquoted on purpose: both variables are whitespace-separated package lists.
        # shellcheck disable=SC2086
        DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
            $QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
            > /dev/null

        info "System dependencies installed"

        # Only on a machine that already boots from ZFS. Pulling zfsutils-linux
        # in unconditionally would drag zfs-dkms onto every dev box and rebuild
        # the module on each kernel upgrade for no benefit.
        if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
            info "ZFS root detected, installing zfsutils-linux..."
            DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
        fi
    fi

    # Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
    # dhcpcd-base is present. It binds br-wan and competes with vpcd's
    # nclient4 for OFFERs, draining the upstream pool and causing
    # intermittent DORA failures. Must run even when apt is skipped (CI
    # bootstrap runs with INSTALL_SPINIFEX_SKIP_APT=1 against runners that
    # already have dhcpcd-base preinstalled). The ISO installer does the
    # same mask (spinifex/operator/installer/install/install.go).
    $SUDO systemctl disable --now dhcpcd.service 2>/dev/null || true
    $SUDO systemctl mask dhcpcd.service 2>/dev/null || true
}

# --- Install AWS CLI ---
install_aws_cli() {
    stage "installing AWS CLI v2"
    if [ "${INSTALL_SPINIFEX_SKIP_AWS}" = "1" ]; then
        info "Skipping AWS CLI (INSTALL_SPINIFEX_SKIP_AWS=1)"
        return
    fi

    if command -v aws >/dev/null 2>&1; then
        info "AWS CLI already installed: $(aws --version 2>&1 | head -1)"
        return
    fi

    info "Installing AWS CLI v2..."
    AWS_TMPDIR=$(mktemp -d)
    curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${AWS_ARCH}.zip" -o "$AWS_TMPDIR/awscliv2.zip"
    unzip -q "$AWS_TMPDIR/awscliv2.zip" -d "$AWS_TMPDIR"
    $SUDO "$AWS_TMPDIR/aws/install" --update > /dev/null
    rm -rf "$AWS_TMPDIR"

    info "AWS CLI installed: $(aws --version 2>&1 | head -1)"
}

# --- Download tarball ---
download_spinifex() {
    stage "downloading/extracting spinifex release tarball"
    SPINIFEX_TMPDIR=$(mktemp -d)
    TARBALL="$SPINIFEX_TMPDIR/spinifex.tar.gz"

    # Local tarball override — skip download (for testing and air-gapped installs)
    if [ -n "$INSTALL_SPINIFEX_TARBALL" ]; then
        info "Using local tarball: $INSTALL_SPINIFEX_TARBALL"
        cp "$INSTALL_SPINIFEX_TARBALL" "$TARBALL"
        info "Extracting..."
        tar -xzf "$TARBALL" -C "$SPINIFEX_TMPDIR"
        EXTRACT_DIR="$SPINIFEX_TMPDIR"
        return
    fi

    if [ -n "$INSTALL_SPINIFEX_VERSION" ]; then
        DOWNLOAD_URL="${INSTALL_BASE_URL}/download/${INSTALL_SPINIFEX_VERSION}/${ARCH}"
        info "Downloading Spinifex $INSTALL_SPINIFEX_VERSION ($ARCH)..."
    else
        DOWNLOAD_URL="${INSTALL_BASE_URL}/download/${INSTALL_SPINIFEX_CHANNEL}/${ARCH}"
        info "Downloading Spinifex ($INSTALL_SPINIFEX_CHANNEL channel, $ARCH)..."
    fi

    HTTP_CODE=$(curl -fsSL -w '%{http_code}' -o "$TARBALL" "$DOWNLOAD_URL" 2>/dev/null) || true
    if [ ! -f "$TARBALL" ] || [ "$HTTP_CODE" -ge 400 ] 2>/dev/null; then
        rm -rf "$SPINIFEX_TMPDIR"
        fatal "Failed to download Spinifex from $DOWNLOAD_URL (HTTP $HTTP_CODE)"
    fi

    # Verify checksum if available
    CHECKSUM_URL="${DOWNLOAD_URL}.sha256"
    if curl -fsSL -o "$SPINIFEX_TMPDIR/checksum.sha256" "$CHECKSUM_URL" 2>/dev/null; then
        info "Verifying checksum..."
        EXPECTED=$(awk '{print $1}' "$SPINIFEX_TMPDIR/checksum.sha256")
        ACTUAL=$(sha256sum "$TARBALL" | awk '{print $1}')
        if [ "$EXPECTED" != "$ACTUAL" ]; then
            rm -rf "$SPINIFEX_TMPDIR"
            fatal "Checksum verification failed. Expected: $EXPECTED, Got: $ACTUAL"
        fi
        info "Checksum verified"
    else
        rm -rf "$SPINIFEX_TMPDIR"
        fatal "Checksum not available at $CHECKSUM_URL. Cannot verify download integrity."
    fi

    # Extract
    info "Extracting..."
    tar -xzf "$TARBALL" -C "$SPINIFEX_TMPDIR"
    EXTRACT_DIR="$SPINIFEX_TMPDIR"
}

# --- Place files ---
install_files() {
    stage "installing binaries and scripts"
    info "Installing files..."

    # Binary
    $SUDO install -m 0755 "$EXTRACT_DIR/spx" /usr/local/bin/spx
    info "  /usr/local/bin/spx"

    # nbdkit plugin
    PLUGINDIR=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2)
    if [ -z "$PLUGINDIR" ]; then
        warn "Could not detect nbdkit plugin directory, using default"
        if [ "$ARCH" = "arm64" ]; then
            PLUGINDIR="/usr/lib/aarch64-linux-gnu/nbdkit/plugins"
        else
            PLUGINDIR="/usr/lib/x86_64-linux-gnu/nbdkit/plugins"
        fi
    fi
    $SUDO mkdir -p "$PLUGINDIR"
    $SUDO install -m 0755 "$EXTRACT_DIR/nbdkit-viperblock-plugin.so" "$PLUGINDIR/nbdkit-viperblock-plugin.so"
    info "  $PLUGINDIR/nbdkit-viperblock-plugin.so"

    # Setup scripts
    $SUDO mkdir -p /usr/local/share/spinifex
    if [ -f "$EXTRACT_DIR/setup-ovn.sh" ]; then
        $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
        info "  /usr/local/share/spinifex/setup-ovn.sh"
    fi

    # Install setup.sh itself so firstboot and future re-runs can find it at a
    # stable path on both ISO and curl|bash installs.
    if [ -f "$EXTRACT_DIR/setup.sh" ]; then
        $SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
        info "  /usr/local/share/spinifex/setup.sh"
    fi

    # Formation. Shipping these means a multi-node cluster can be formed from a
    # `curl | bash` install with no checkout, which is what a cloud operator has.
    # smoke-test.sh comes too: install-node.sh scp's it from its own directory for
    # the post-formation verification, so without it formation fails at the last step.
    for formation in install-node.sh smoke-test.sh; do
        if [ -f "$EXTRACT_DIR/$formation" ]; then
            $SUDO install -m 0755 "$EXTRACT_DIR/$formation" "/usr/local/share/spinifex/$formation"
            info "  /usr/local/share/spinifex/$formation"
        fi
    done

    # Teardown scripts. An install delivered by `curl | bash` with no way to
    # undo it is a gap in the product, not just in our tooling, and node-reset.sh
    # ships alongside because uninstall-spx.sh delegates its state teardown to it
    # rather than reimplementing the QEMU and JetStream waits.
    for teardown in node-reset.sh uninstall-spx.sh; do
        if [ -f "$EXTRACT_DIR/$teardown" ]; then
            $SUDO install -m 0755 "$EXTRACT_DIR/$teardown" "/usr/local/share/spinifex/$teardown"
            info "  /usr/local/share/spinifex/$teardown"
        fi
    done

    # microVM kernel + initramfs
    $SUDO install -d /usr/share/spinifex/microvm
    if [ -d "$EXTRACT_DIR/microvm" ]; then
        $SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
        info "  /usr/share/spinifex/microvm/*"
    fi
}

# --- Create directories ---
create_directories() {
    stage "creating spinifex directory layout"
    info "Creating directories..."

    # Top-level directories (root-owned, group-readable by spinifex)
    $SUDO mkdir -p /etc/spinifex
    $SUDO chmod 0750 /etc/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex

    $SUDO mkdir -p /var/lib/spinifex
    $SUDO chmod 0750 /var/lib/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex

    # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/.
    # -n so a re-run replaces the link rather than creating one inside the
    # directory it already points at.
    if $SUDO test ! -e /var/lib/spinifex/config; then
        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/config
    fi

    # Symlink so services that write logs to BaseDir/logs/ use /var/log/spinifex/
    if $SUDO test ! -e /var/lib/spinifex/logs; then
        $SUDO ln -sfn /var/log/spinifex /var/lib/spinifex/logs
    fi

    $SUDO mkdir -p /var/log/spinifex
    $SUDO chmod 0775 /var/log/spinifex
    $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex

    # /run/spinifex and /run/spinifex/nbd are declared via tmpfiles.d because
    # /run is tmpfs — direct mkdir doesn't survive reboot, and units have
    # ReadWritePaths= on these paths which fails namespace setup with ENOENT
    # if the dirs are absent at service start.
    _tmpf=$(mktemp)
    cat > "$_tmpf" <<'TMPEOF'
# Type  Path               Mode  User                 Group                Age
d       /run/spinifex      0770  root                 spinifex             -
d       /run/spinifex/nbd  0770  spinifex-viperblock  spinifex-viperblock  -
TMPEOF
    $SUDO install -m 0644 "$_tmpf" /etc/tmpfiles.d/spinifex.conf
    rm -f "$_tmpf"
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        # Live systems: materialise the runtime dirs immediately so a re-run of
        # setup.sh on an existing host has correct /run state without a reboot.
        $SUDO systemd-tmpfiles --create /etc/tmpfiles.d/spinifex.conf 2>/dev/null || true
    fi

    # Per-service config directories
    $SUDO mkdir -p /etc/spinifex/nats
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
    $SUDO chmod 0750 /etc/spinifex/nats

    $SUDO mkdir -p /etc/spinifex/predastore
    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
    $SUDO chmod 0750 /etc/spinifex/predastore

    # Northstar holds northstar.toml (bucket-scoped S3 creds, written 0600 by
    # `spx admin init`); fix_file_ownership reassigns it to spinifex-northstar.
    $SUDO mkdir -p /etc/spinifex/northstar
    $SUDO chown "spinifex-northstar:$SPINIFEX_GROUP" /etc/spinifex/northstar
    $SUDO chmod 0750 /etc/spinifex/northstar

    $SUDO mkdir -p /etc/spinifex/awsgw
    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw
    $SUDO chmod 0750 /etc/spinifex/awsgw

    # Viperblock's at-rest encryption key dir. 0750 group-traversable; the key
    # itself is set to root:spinifex 0640 by SetServiceOwnership because both
    # viperblockd (spinifex-viperblock) and the awsgw handlers (spinifex-gw)
    # load it.
    $SUDO mkdir -p /etc/spinifex/viperblock
    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/viperblock
    $SUDO chmod 0750 /etc/spinifex/viperblock

    # Per-service data directories
    $SUDO mkdir -p /var/lib/spinifex/nats
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
    $SUDO chmod 0700 /var/lib/spinifex/nats

    $SUDO mkdir -p /var/lib/spinifex/spinifex
    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
    $SUDO chmod 0700 /var/lib/spinifex/spinifex

    $SUDO mkdir -p /var/lib/spinifex/predastore
    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
    $SUDO chmod 0700 /var/lib/spinifex/predastore

    $SUDO mkdir -p /var/lib/spinifex/northstar
    $SUDO chown "spinifex-northstar:$SPINIFEX_GROUP" /var/lib/spinifex/northstar
    $SUDO chmod 0700 /var/lib/spinifex/northstar

    $SUDO mkdir -p /var/lib/spinifex/viperblock
    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
    $SUDO chmod 0700 /var/lib/spinifex/viperblock

    $SUDO mkdir -p /var/lib/spinifex/vpcd
    $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
    $SUDO chmod 0700 /var/lib/spinifex/vpcd

    $SUDO mkdir -p /var/lib/spinifex/awsgw
    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /var/lib/spinifex/awsgw
    $SUDO chmod 0700 /var/lib/spinifex/awsgw

    $SUDO mkdir -p /var/lib/spinifex/spinifex-ui
    $SUDO chown "spinifex-ui:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex-ui
    $SUDO chmod 0700 /var/lib/spinifex/spinifex-ui

    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/.
    # The test runs under $SUDO because the parent is 0700 spinifex-gw: an
    # unprivileged -e cannot stat through it and always reports "missing", so
    # every re-run reached the ln below. That ln then resolved through the
    # existing link and tried to create /etc/spinifex/spinifex, which
    # CreateServiceDirectories already owns — failing the whole install.
    if $SUDO test ! -e /var/lib/spinifex/awsgw/config; then
        $SUDO ln -sfn /etc/spinifex /var/lib/spinifex/awsgw/config
    fi

    # Service helper scripts (root-owned, group-executable by all service users)
    if [ -d "$EXTRACT_DIR/scripts" ]; then
        for script in "$EXTRACT_DIR"/scripts/*.sh; do
            $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
                "$script" "/var/lib/spinifex/$(basename "$script")"
            info "  /var/lib/spinifex/$(basename "$script")"
        done
    fi
}

# --- Generate systemd environment file ---
# Split out of create_directories so the `env` stage can be refreshed
# independently (e.g. by inject-bins.sh when the plugin path changes).
install_systemd_env() {
    stage "generating /etc/spinifex/systemd.env"

    # nbdkit plugin path is arch-dependent (`nbdkit --dump-config` returns
    # /usr/lib/{x86_64,aarch64}-linux-gnu/nbdkit/plugins). Resolve it here so
    # systemd.env always matches wherever install_files placed the .so.
    local plugindir
    plugindir=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2)
    if [ -z "$plugindir" ]; then
        if [ "${ARCH:-}" = "arm64" ]; then
            plugindir="/usr/lib/aarch64-linux-gnu/nbdkit/plugins"
        else
            plugindir="/usr/lib/x86_64-linux-gnu/nbdkit/plugins"
        fi
    fi

    $SUDO mkdir -p /etc/spinifex
    $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
# Generated by setup.sh — install-specific environment variables
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${plugindir}/nbdkit-viperblock-plugin.so
EOF
    $SUDO chown "spinifex-viperblock:${SPINIFEX_GROUP:-spinifex}" /etc/spinifex/systemd.env
    $SUDO chmod 0640 /etc/spinifex/systemd.env
    info "Generated /etc/spinifex/systemd.env"
}

# --- Fix file ownership for upgrades from v1 ---
# Also invoked from firstboot via SETUP_STAGES=fixown to correct ownership of
# files that `spx admin init` wrote as root:root.
fix_file_ownership() {
    stage "fixing file ownership for privilege separation"
    # create_service_users normally sets this; if we're invoked via
    # SETUP_STAGES=fixown on a host that already has the group, users isn't
    # run and SPINIFEX_GROUP is unset — default it here.
    SPINIFEX_GROUP="${SPINIFEX_GROUP:-spinifex}"
    info "Fixing file ownership for privilege separation..."

    # Per-service data dirs — recursive chown so existing files are accessible
    for entry in \
        nats:spinifex-nats \
        predastore:spinifex-storage \
        northstar:spinifex-northstar \
        spinifex:spinifex-daemon \
        viperblock:spinifex-viperblock \
        vpcd:spinifex-vpcd \
        awsgw:spinifex-gw; do
        IFS=: read -r dir svc_user <<< "$entry"
        if [ -d "/var/lib/spinifex/$dir" ]; then
            $SUDO chown -R "$svc_user:$SPINIFEX_GROUP" "/var/lib/spinifex/$dir" \
                || fatal "Failed to set ownership on /var/lib/spinifex/$dir"
        fi
    done

    # Per-service config dirs — recursive chown
    if [ -d /etc/spinifex/nats ]; then
        $SUDO chown -R "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats \
            || fatal "Failed to set ownership on /etc/spinifex/nats"
    fi
    if [ -d /etc/spinifex/predastore ]; then
        $SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
            || fatal "Failed to set ownership on /etc/spinifex/predastore"
    fi
    if [ -d /etc/spinifex/northstar ]; then
        $SUDO chown -R "spinifex-northstar:$SPINIFEX_GROUP" /etc/spinifex/northstar \
            || fatal "Failed to set ownership on /etc/spinifex/northstar"
    fi
    if [ -d /etc/spinifex/awsgw ]; then
        $SUDO chown -R "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw \
            || fatal "Failed to set ownership on /etc/spinifex/awsgw"
    fi

    # Shared config files — root:spinifex with per-file modes
    for f in spinifex.toml master.key server.key; do
        if [ -f "/etc/spinifex/$f" ]; then
            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
                || fatal "Failed to set ownership on /etc/spinifex/$f"
            $SUDO chmod 0640 "/etc/spinifex/$f" \
                || fatal "Failed to set permissions on /etc/spinifex/$f"
        fi
    done
    for f in server.pem ca.pem; do
        if [ -f "/etc/spinifex/$f" ]; then
            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
                || fatal "Failed to set ownership on /etc/spinifex/$f"
            $SUDO chmod 0644 "/etc/spinifex/$f" \
                || fatal "Failed to set permissions on /etc/spinifex/$f"
        fi
    done

    # CA private key — root-only
    if [ -f /etc/spinifex/ca.key ]; then
        $SUDO chown root:root /etc/spinifex/ca.key \
            || fatal "Failed to set ownership on /etc/spinifex/ca.key"
        $SUDO chmod 0600 /etc/spinifex/ca.key \
            || fatal "Failed to set permissions on /etc/spinifex/ca.key"
    fi

    # Shared data dirs — root:spinifex 0770 (daemon + admin CLI write, services read).
    # chmod must be recursive so pre-existing files (e.g. imported images originally
    # written as 0600 by another user) become group-readable.
    for d in images amis volumes state; do
        if [ -d "/var/lib/spinifex/$d" ]; then
            $SUDO chown -R "root:$SPINIFEX_GROUP" "/var/lib/spinifex/$d" \
                || fatal "Failed to set ownership on /var/lib/spinifex/$d"
            $SUDO chmod -R u+rwX,g+rwX,o-rwx "/var/lib/spinifex/$d" \
                || fatal "Failed to set permissions on /var/lib/spinifex/$d"
            # setgid on directories only, so subdirectories a root run creates
            # keep the spinifex group instead of falling back to root and locking
            # the admin CLI out of the tree it is meant to write.
            $SUDO find "/var/lib/spinifex/$d" -type d -exec chmod g+s {} + \
                || fatal "Failed to set setgid on /var/lib/spinifex/$d"
        fi
    done

    info "File ownership updated"
}

# --- Run config migrations ---
run_migrations() {
    # Only run if spinifex is already initialized (config exists)
    if [ ! -f /etc/spinifex/spinifex.toml ]; then
        info "Fresh install detected, skipping migrations"
        return
    fi

    if [ "${INSTALL_SPINIFEX_SKIP_MIGRATE:-0}" = "1" ]; then
        info "INSTALL_SPINIFEX_SKIP_MIGRATE=1, skipping migrations"
        info "Run 'spx admin upgrade' manually to apply pending migrations"
        return
    fi

    info "Running config migrations..."
    $SUDO /usr/local/bin/spx admin upgrade --yes \
        || fatal "Config migration failed. See errors above."
}

# --- Install systemd units ---
install_systemd() {
    stage "installing systemd units"
    info "Installing systemd units..."

    if [ ! -d "$EXTRACT_DIR/systemd" ]; then
        fatal "Systemd unit files not found in tarball (expected systemd/ directory)"
    fi

    for unit in "$EXTRACT_DIR"/systemd/*; do
        $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
        info "  /etc/systemd/system/$(basename "$unit")"
    done

    # Reserve RAM + CPU priority for system.slice (sshd, journald, the operator)
    # so a maxed spinifex.slice cannot starve them — the "stay sshable" guarantee.
    # Generated here rather than shipped as a staged file because the packaging
    # globs flatten the systemd/ dir and would skip a nested drop-in directory;
    # this mirrors the sshd-keygen drop-in pattern in build-rootfs.sh. MemoryMin
    # is a guaranteed-unreclaimable floor, not a cap.
    $SUDO mkdir -p /etc/systemd/system/system.slice.d
    $SUDO tee /etc/systemd/system/system.slice.d/spinifex-reserve.conf > /dev/null << 'EOF'
[Slice]
MemoryMin=1G
CPUWeight=300
EOF
    info "  /etc/systemd/system/system.slice.d/spinifex-reserve.conf"

    # daemon-reload / enable require a running systemd — skip inside the ISO
    # chroot. Unit files are still dropped into place; firstboot enables them.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        info "Systemd units installed (ISO_BUILD=1, skipping daemon-reload/enable)"
        return
    fi
    $SUDO systemctl daemon-reload
    $SUDO systemctl enable spinifex.target
    # The unit file alone does not self-activate (WantedBy=timers.target only
    # takes effect once enabled) — without this the JetStream ENOSPC-latch
    # watchdog is inert and a full disk requires a manual restart forever.
    $SUDO systemctl enable --now spinifex-nats-watchdog.timer
    # Enabled, not started: its ConditionPathExists holds it inert until the
    # daemon writes a peer file, so this only decides that a node which has one
    # gets the policy back at boot, before any service opens a socket.
    $SUDO systemctl enable spinifex-firewall.service
    info "Systemd units installed and enabled (per-service users)"
}

# --- Install kernel tunables ---
# Predastore carries blob, meta and raft over QUIC, and quic-go asks for a 7 MiB
# UDP receive buffer. Left at the kernel default of 208 KiB it silently drops
# datagrams under load, which fails shard writes and corrupts guest volumes.
install_sysctl() {
    stage "installing kernel tunables"
    $SUDO install -d /etc/sysctl.d
    $SUDO tee /etc/sysctl.d/99-spinifex-net.conf > /dev/null << 'SYSCTL'
# Managed by spinifex setup.sh — do not edit.
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
SYSCTL
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        $SUDO sysctl -q --system 2>/dev/null || true
    fi
    info "Kernel tunables installed (net.core.rmem_max/wmem_max = 16MB)"
}

# --- Install logrotate ---
install_logrotate() {
    stage "installing logrotate config"
    if [ -f "$EXTRACT_DIR/logrotate-spinifex" ]; then
        $SUDO install -m 0644 "$EXTRACT_DIR/logrotate-spinifex" /etc/logrotate.d/spinifex
    else
        warn "Logrotate config not found in tarball, skipping"
        return
    fi
    info "Logrotate config installed"
}

# --- Install udev rules ---
install_udev() {
    stage "installing udev rules"
    if [ ! -d "$EXTRACT_DIR/udev" ]; then
        return
    fi
    $SUDO install -d /etc/udev/rules.d
    for rule in "$EXTRACT_DIR"/udev/*; do
        $SUDO install -m 0644 "$rule" "/etc/udev/rules.d/$(basename "$rule")"
        info "  /etc/udev/rules.d/$(basename "$rule")"
    done
    if [ "${ISO_BUILD:-0}" != "1" ]; then
        $SUDO udevadm control --reload-rules
        $SUDO udevadm trigger --subsystem-match=vfio 2>/dev/null || true
    fi
    info "udev rules installed"
}

# --- Upgrade handling ---
handle_upgrade() {
    if $SUDO systemctl is-active --quiet spinifex.target 2>/dev/null; then
        warn "Spinifex services are running. Stopping for upgrade..."
        $SUDO systemctl stop spinifex.target
        RESTART_AFTER=true
    fi

    # Pre-tightening hosts have /run/spinifex/nbd as root:spinifex 0770, which
    # grants access to every spinifex-* service. While services are stopped,
    # bring it up to the new spinifex-viperblock:spinifex-viperblock 0770
    # policy so restart picks up the fresh group membership without a reboot.
    if [ -d /run/spinifex/nbd ]; then
        $SUDO chown spinifex-viperblock:spinifex-viperblock /run/spinifex/nbd 2>/dev/null || true
        $SUDO chmod 0770 /run/spinifex/nbd 2>/dev/null || true
    fi
}

restart_if_needed() {
    if [ "${RESTART_AFTER}" = "true" ]; then
        info "Restarting Spinifex services..."
        $SUDO systemctl start spinifex.target
    fi
}

# --- Print summary ---
print_summary() {
    INSTALLED_VERSION=$(/usr/local/bin/spx version 2>/dev/null || echo "unknown")

    echo ""
    echo "============================================"
    echo "  Spinifex installed successfully"
    echo "============================================"
    echo ""
    echo "  Version:      $INSTALLED_VERSION"
    echo "  Architecture: $ARCH"
    echo "  Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
    echo "  Binary:       /usr/local/bin/spx"
    echo "  Config:       /etc/spinifex/"
    echo "  Data:         /var/lib/spinifex/"
    echo "  Logs:         /var/log/spinifex/"
    echo ""
    echo "  Next steps:"
    echo ""
    echo "  1. Setup OVN networking:"
    echo "     If your WAN interface is already a bridge (e.g. br-wan):"
    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management"
    echo ""
    echo "     If your WAN is a physical NIC:"
    echo "       # Dedicated WAN NIC (not your SSH connection):"
    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
    echo ""
    echo "  2. Initialize:"
    echo "     sudo spx admin init --node node1 --nodes 1"
    echo ""
    echo "  3. Start services:"
    echo "     sudo systemctl start spinifex.target"
    echo ""
    echo "  4. Verify:"
    echo "     export AWS_PROFILE=spinifex"
    echo "     aws ec2 describe-instance-types"
    echo ""
}

# --- Configure host swap ---
# Provisions an 8G swapfile and lowers vm.swappiness so spinifex.slice
# (MemorySwapMax=100%) has a backing device for graceful degradation under
# memory pressure. Reverses the historical swap=0 assumption. Idempotent.
setup_swap() {
    stage "configuring host swap"

    # ISO build runs in a chroot — cannot swapon, and baking an 8G file into the
    # rootfs bloats the image. ISO hosts provision swap at firstboot instead.
    if [ "${ISO_BUILD:-0}" = "1" ]; then
        info "Swap setup skipped (ISO_BUILD=1; firstboot provisions swap)"
        return
    fi

    local swapfile=/swapfile size_mb=8192

    # Swap is a safety buffer, not a routine path: reclaim page cache first.
    $SUDO tee /etc/sysctl.d/99-spinifex-swap.conf > /dev/null << 'EOF'
# Spinifex: minimise swapping; swap backs spinifex.slice graceful degradation.
vm.swappiness = 10
EOF
    $SUDO chmod 0644 /etc/sysctl.d/99-spinifex-swap.conf
    $SUDO sysctl -q -w vm.swappiness=10 || true

    if swapon --show=NAME --noheadings 2>/dev/null | grep -qx "$swapfile"; then
        info "Swap already active ($swapfile)"
        return
    fi

    if [ ! -f "$swapfile" ]; then
        info "Creating ${size_mb}MiB $swapfile"
        $SUDO fallocate -l "${size_mb}M" "$swapfile" 2>/dev/null \
            || $SUDO dd if=/dev/zero of="$swapfile" bs=1M count="$size_mb" status=none
        $SUDO chmod 0600 "$swapfile"
        $SUDO mkswap "$swapfile" > /dev/null
    fi
    $SUDO swapon "$swapfile"
    grep -q "^$swapfile " /etc/fstab 2>/dev/null \
        || echo "$swapfile none swap sw 0 0" | $SUDO tee -a /etc/fstab > /dev/null
    info "Swap enabled: $swapfile (${size_mb}MiB), vm.swappiness=10"
}

# --- Main ---
main() {
    while [ $# -gt 0 ]; do
        case "$1" in
            --firewall=*) INSTALL_SPINIFEX_FIREWALL="${1#*=}" ;;
            --firewall)   INSTALL_SPINIFEX_FIREWALL="${2:-}"; shift ;;
            --channel=*)  INSTALL_SPINIFEX_CHANNEL="${1#*=}" ;;
            --channel)    INSTALL_SPINIFEX_CHANNEL="${2:-}"; shift ;;
            --version=*)  INSTALL_SPINIFEX_VERSION="${1#*=}" ;;
            --version)    INSTALL_SPINIFEX_VERSION="${2:-}"; shift ;;
            *) fatal "unknown option: $1" ;;
        esac
        shift
    done

    case "$INSTALL_SPINIFEX_CHANNEL" in
        latest|dev) ;;
        # Anything else is passed to the install endpoint as a literal tag, so a
        # typo would 404 on the download rather than here. Name the two channels.
        *) fatal "--channel must be 'latest' or 'dev', got: ${INSTALL_SPINIFEX_CHANNEL} (use --version for a specific tag)" ;;
    esac

    case "${INSTALL_SPINIFEX_FIREWALL}" in
        on|off) ;;
        # Unset resolves by install path: the ISO arms, everything else does not.
        "") if [ "${ISO_BUILD:-0}" = "1" ]; then
                INSTALL_SPINIFEX_FIREWALL="on"
            else
                INSTALL_SPINIFEX_FIREWALL="off"
            fi ;;
        *) fatal "--firewall must be 'on' or 'off', got: ${INSTALL_SPINIFEX_FIREWALL}" ;;
    esac

    info "Spinifex installer"
    echo ""

    # Always needed: sudo resolution, OS/arch detection.
    setup_sudo
    detect_os
    detect_arch

    # Orchestration (stop running services, fix stale /run/spinifex/nbd perms)
    # only applies in full-install mode on a live host.
    if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then
        handle_upgrade
    fi

    # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
    # Only download/extract when at least one such stage is enabled.
    if stage_enabled files || stage_enabled directories \
        || stage_enabled systemd || stage_enabled logrotate \
        || stage_enabled udev; then
        download_spinifex
    fi

    stage_enabled deps       && install_apt_deps
    stage_enabled aws        && install_aws_cli
    stage_enabled users      && create_service_users
    stage_enabled sudoers    && install_sudoers
    stage_enabled firewall   && install_firewall
    stage_enabled timesync   && install_chrony_conf
    stage_enabled files      && install_files
    stage_enabled directories && create_directories
    stage_enabled env        && install_systemd_env
    stage_enabled fixown     && fix_file_ownership
    stage_enabled systemd    && install_systemd
    stage_enabled sysctl     && install_sysctl
    stage_enabled logrotate  && install_logrotate
    stage_enabled udev       && install_udev
    stage_enabled swap       && setup_swap

    # Migrations are only safe on a live system (need a running NATS and a
    # persisted config file). Skip under ISO_BUILD and under any explicit
    # SETUP_STAGES filter that doesn't list migrations.
    if [ "${ISO_BUILD:-0}" != "1" ] && stage_enabled migrations; then
        run_migrations
    fi

    [ -n "${SPINIFEX_TMPDIR:-}" ] && rm -rf "$SPINIFEX_TMPDIR"

    # Post-install orchestration (service restart, summary, newgrp) only in
    # full-install mode on a live host.
    if [ "${ISO_BUILD:-0}" = "1" ] || [ -n "${SETUP_STAGES:-}" ]; then
        return
    fi

    restart_if_needed
    print_summary

    # Activate spinifex group membership in the invoking shell. Under curl|bash
    # stdin is the drained pipe, so redirect from /dev/tty and exec so the new
    # shell becomes the foreground process. Skip when we can't actually open
    # /dev/tty (CI, cloud-init, ssh -T — stat passes but open fails with ENXIO).
    if [ "${INSTALL_SPINIFEX_SKIP_NEWGRP:-0}" != "1" ] \
        && ! id -Gn 2>/dev/null | grep -qw "$SPINIFEX_GROUP" \
        && ( : </dev/tty ) 2>/dev/null; then
        echo ""
        echo "  Activating '$SPINIFEX_GROUP' group in a subshell — type 'exit' when done."
        echo ""
        exec newgrp "$SPINIFEX_GROUP" < /dev/tty
    fi
}

# Guarded so the package lint can source this file for APT_RUNTIME_PACKAGES
# without running the installer.
if [ "${INSTALL_SPINIFEX_LIB_ONLY:-0}" != "1" ]; then
    main "$@"
fi