diff --git a/cmd/installer/firstboot/firstboot.go b/cmd/installer/firstboot/firstboot.go index e3279fda7..99c1bde31 100644 --- a/cmd/installer/firstboot/firstboot.go +++ b/cmd/installer/firstboot/firstboot.go @@ -97,16 +97,6 @@ fi # Set hostname hostnamectl set-hostname %s -# NOTE: We do NOT invoke /usr/local/share/spinifex/setup.sh here. The ISO -# rootfs (scripts/iso-builder/build/build-rootfs.sh) already pre-stages -# everything setup.sh would do at install time: spx binary, viperblock -# nbdkit plugin, service users (spinifex-{nats,gw,daemon,storage,viperblock, -# vpcd,ui}), data directories with per-service ownership, sudoers rules, -# systemd units, and tmpfiles.d entries. Calling setup.sh would also fail -# in the air-gapped boot environment because no tarball is staged at -# /opt/spinifex/spinifex.tar.gz — the firstboot would never complete and -# every spinifex-* service depending on it would refuse to start. - # Pre-start OVS and OVN central so their databases are initialised before # setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes # longer than setup-ovn.sh's internal 15-second timeout allows. Starting them @@ -139,28 +129,16 @@ resolvconf -u # Cluster formation — capture credentials to file for display on console. %s 2>&1 -# Fix ownership of files spx admin init created. spx runs as root (no -# SUDO_USER under systemd), so /etc/spinifex/{spinifex.toml,master.key} and -# any per-service files written under /var/lib/spinifex/* land as root:root. -# Re-chown them so each service can read/write its own subtree at -# Type=notify start. The directories themselves were pre-created with the -# correct ownership in build-rootfs.sh — these chowns only affect the -# runtime-generated config and key files inside them. -chown root:spinifex /etc/spinifex && chmod 750 /etc/spinifex -find /etc/spinifex -type f -exec chmod 640 {} \; -chown -R spinifex-gw:spinifex /var/lib/spinifex/awsgw -chown -R spinifex-daemon:spinifex /var/lib/spinifex/spinifex -chown -R spinifex-nats:spinifex /var/lib/spinifex/nats -chown -R spinifex-storage:spinifex /var/lib/spinifex/predastore -chown -R spinifex-viperblock:spinifex /var/lib/spinifex/viperblock -chown -R spinifex-vpcd:spinifex /var/lib/spinifex/vpcd - -# awsgw looks for master.key at /config/master.key. In production -# BaseDir is /var/lib/spinifex/awsgw/ (set by SPINIFEX_BASE_DIR), but the -# key lives in /etc/spinifex/. Symlink so both paths resolve to the same -# file. spx admin init writes the key, so this must run after it. -mkdir -p /var/lib/spinifex/awsgw/config -ln -sf /etc/spinifex/master.key /var/lib/spinifex/awsgw/config/master.key +# Fix ownership of files spx admin init wrote. spx runs as root under +# systemd, so /etc/spinifex/{spinifex.toml,master.key,ca.key,*.pem} and any +# per-service files written under /var/lib/spinifex/* land as root:root. +# Delegate to setup.sh's fix_file_ownership (single source of truth) — it +# knows the per-file modes (ca.key 0600, *.pem 0644, toml/key 0640) that a +# blunt recursive chmod would clobber. +SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh +# The /var/lib/spinifex/awsgw/config symlink was created at build time by +# setup.sh's create_directories — it resolves to /etc/spinifex, so +# {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key. # Copy AWS credentials to the spinifex user's home directory. # spx admin init runs with HOME=/root (set by the systemd unit), so credentials diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index cca285a4b..24c1698dd 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -13,7 +13,7 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp | `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | TLS cert/key, base-path, config-path required | Creates predastore service instance with S3-compatible storage backend → starts service | **DONE** | | `spx service predastore stop` | — | Predastore must be running | Stops the predastore service | **DONE** | | `spx service predastore status` | — | None | Reports predastore service status | **DONE** | -| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (default: /opt/spinifex/lib/nbdkit-viperblock-plugin.so) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** | +| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** | | `spx service viperblock stop` | — | Viperblock must be running | Stops the viperblock service | **DONE** | | `spx service viperblock status` | — | None | Reports viperblock service status | **DONE** | | `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | None | Starts embedded NATS server with optional JetStream | **DONE** | diff --git a/scripts/setup.sh b/scripts/setup.sh index 005428aff..190d87a72 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -9,6 +9,14 @@ # INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip apt dependency install # INSTALL_SPINIFEX_SKIP_AWS Set to 1 to skip AWS CLI install # INSTALL_SPINIFEX_SKIP_NEWGRP Set to 1 to skip newgrp exec at end (for callers like dev-install.sh) +# ISO_BUILD Set to 1 when running inside a debootstrap chroot from the ISO +# builder: skip handle_upgrade/restart/migrations/newgrp/print_summary, +# skip systemctl daemon-reload + enable, short-circuit setup_sudo. +# VERBOSE Set to 1 to echo "[setup] " before each top-level step. +# SETUP_STAGES Comma-separated subset of stages to run: +# deps, aws, users, sudoers, files, directories, +# env, systemd, logrotate, fixown, migrations +# Unset = run every stage appropriate for the current mode. set -e @@ -25,8 +33,26 @@ info() { echo -e "${GREEN}[INFO]${NC} $*"; } warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } +stage() { + [ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*" + return 0 +} + +stage_enabled() { + [ -z "${SETUP_STAGES:-}" ] && return 0 + case ",${SETUP_STAGES}," in + *",$1,"*) return 0 ;; + *) return 1 ;; + esac +} + # --- Sudo setup --- setup_sudo() { + # Inside a debootstrap chroot we are already root and sudo may not be installed. + if [ "${ISO_BUILD:-0}" = "1" ]; then + SUDO="" + return + fi if [ "$(id -u)" -eq 0 ]; then SUDO="" elif command -v sudo >/dev/null 2>&1; then @@ -92,6 +118,7 @@ detect_arch() { # --- Create per-service system users --- create_service_users() { + stage "creating spinifex group and per-service users" SPINIFEX_GROUP="spinifex" # Create shared group @@ -99,6 +126,13 @@ create_service_users() { $SUDO groupadd --system "$SPINIFEX_GROUP" fi + # Producer-typed access group for viperblock's runtime resources + # (/run/spinifex/nbd socket dir). Viperblock writes as owner; daemon reads + # via supplementary membership; every other spinifex-* service is excluded. + if ! getent group spinifex-viperblock > /dev/null 2>&1; then + $SUDO groupadd --system spinifex-viperblock + fi + # Create per-service users with correct home directories declare -A SERVICE_HOMES=( [nats]="/var/lib/spinifex/nats" @@ -120,10 +154,17 @@ create_service_users() { fi done - # Add invoking user to spinifex group for admin CLI access - ADMIN_USER="${SUDO_USER:-$(whoami)}" - if [ "$ADMIN_USER" != "root" ]; then - $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER" + # Add invoking user to spinifex group for admin CLI access. + # Skip under ISO_BUILD: the chroot has no invoking user (tf-user, + # whoever ran `sudo make`, etc. don't exist in the rootfs). The ISO's + # interactive 'spinifex' login account is created later in Phase 4 of + # build-rootfs.sh with spinifex as its primary gid. In curl|bash mode + # guard against a stale/missing SUDO_USER too. + if [ "${ISO_BUILD:-0}" != "1" ]; then + ADMIN_USER="${SUDO_USER:-$(whoami)}" + if [ "$ADMIN_USER" != "root" ] && id -u "$ADMIN_USER" > /dev/null 2>&1; then + $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER" + fi fi # KVM access for daemon @@ -131,11 +172,15 @@ create_service_users() { $SUDO usermod -aG kvm spinifex-daemon fi + # Daemon consumes viperblock's NBD socket — join the producer-typed group. + $SUDO usermod -aG spinifex-viperblock spinifex-daemon + info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})" } # --- Install scoped sudoers rules --- install_sudoers() { + stage "installing scoped sudoers rules" $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS' # Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip @@ -154,7 +199,12 @@ SUDOERS } # --- Install apt dependencies --- +# NOTE: Runtime deps must stay in sync with the ISO package list at +# scripts/iso-builder/build/packages.list in the mulga repo. When you add, +# remove, or change a runtime package here, review that file too — drift means +# the ISO and `curl | bash` paths install different software. install_apt_deps() { + stage "installing apt dependencies" if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)" return @@ -176,6 +226,7 @@ install_apt_deps() { # --- Install AWS CLI --- install_aws_cli() { + stage "installing AWS CLI v2" if [ "${INSTALL_SPINIFEX_SKIP_AWS}" = "1" ]; then info "Skipping AWS CLI (INSTALL_SPINIFEX_SKIP_AWS=1)" return @@ -198,6 +249,7 @@ install_aws_cli() { # --- Download tarball --- download_spinifex() { + stage "downloading/extracting spinifex release tarball" SPINIFEX_TMPDIR=$(mktemp -d) TARBALL="$SPINIFEX_TMPDIR/spinifex.tar.gz" @@ -249,6 +301,7 @@ download_spinifex() { # --- Place files --- install_files() { + stage "installing binaries and scripts" info "Installing files..." # Binary @@ -276,10 +329,17 @@ install_files() { info " /usr/local/share/spinifex/setup-ovn.sh" fi + # Install setup.sh itself so firstboot and future re-runs can find it at a + # stable path on both ISO and curl|bash installs. + if [ -f "$EXTRACT_DIR/setup.sh" ]; then + $SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh + info " /usr/local/share/spinifex/setup.sh" + fi } # --- Create directories --- create_directories() { + stage "creating spinifex directory layout" info "Creating directories..." # Top-level directories (root-owned, group-readable by spinifex) @@ -305,13 +365,20 @@ create_directories() { $SUDO chmod 0775 /var/log/spinifex $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex - $SUDO mkdir -p /run/spinifex - $SUDO chmod 0775 /run/spinifex - $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex - - $SUDO mkdir -p /run/spinifex/nbd - $SUDO chmod 0770 /run/spinifex/nbd - $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex/nbd + # /run/spinifex and /run/spinifex/nbd are declared via tmpfiles.d because + # /run is tmpfs — direct mkdir doesn't survive reboot, and units have + # ReadWritePaths= on these paths which fails namespace setup with ENOENT + # if the dirs are absent at service start. + $SUDO install -m 0644 /dev/stdin /etc/tmpfiles.d/spinifex.conf <<'TMPEOF' +# Type Path Mode User Group Age +d /run/spinifex 0770 root spinifex - +d /run/spinifex/nbd 0770 spinifex-viperblock spinifex-viperblock - +TMPEOF + if [ "${ISO_BUILD:-0}" != "1" ]; then + # Live systems: materialise the runtime dirs immediately so a re-run of + # setup.sh on an existing host has correct /run state without a reboot. + $SUDO systemd-tmpfiles --create /etc/tmpfiles.d/spinifex.conf 2>/dev/null || true + fi # Per-service config directories $SUDO mkdir -p /etc/spinifex/nats @@ -356,16 +423,6 @@ create_directories() { $SUDO ln -s /etc/spinifex /var/lib/spinifex/awsgw/config fi - # Top-level config files (root-owned, group-readable) - # Generate environment file with install-specific values (e.g. arch-dependent paths) - $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF -# Generated by setup.sh — install-specific environment variables -SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so -EOF - $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env - $SUDO chmod 0640 /etc/spinifex/systemd.env - info "Generated /etc/spinifex/systemd.env" - # Service helper scripts (root-owned, group-executable by all service users) if [ -d "$EXTRACT_DIR/scripts" ]; then for script in "$EXTRACT_DIR"/scripts/*.sh; do @@ -376,8 +433,44 @@ EOF fi } +# --- Generate systemd environment file --- +# Split out of create_directories so the `env` stage can be refreshed +# independently (e.g. by inject-bins.sh when the plugin path changes). +install_systemd_env() { + stage "generating /etc/spinifex/systemd.env" + + # nbdkit plugin path is arch-dependent (`nbdkit --dump-config` returns + # /usr/lib/{x86_64,aarch64}-linux-gnu/nbdkit/plugins). Resolve it here so + # systemd.env always matches wherever install_files placed the .so. + local plugindir + plugindir=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2) + if [ -z "$plugindir" ]; then + if [ "${ARCH:-}" = "arm64" ]; then + plugindir="/usr/lib/aarch64-linux-gnu/nbdkit/plugins" + else + plugindir="/usr/lib/x86_64-linux-gnu/nbdkit/plugins" + fi + fi + + $SUDO mkdir -p /etc/spinifex + $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF +# Generated by setup.sh — install-specific environment variables +SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${plugindir}/nbdkit-viperblock-plugin.so +EOF + $SUDO chown "spinifex-viperblock:${SPINIFEX_GROUP:-spinifex}" /etc/spinifex/systemd.env + $SUDO chmod 0640 /etc/spinifex/systemd.env + info "Generated /etc/spinifex/systemd.env" +} + # --- Fix file ownership for upgrades from v1 --- +# Also invoked from firstboot via SETUP_STAGES=fixown to correct ownership of +# files that `spx admin init` wrote as root:root. fix_file_ownership() { + stage "fixing file ownership for privilege separation" + # create_service_users normally sets this; if we're invoked via + # SETUP_STAGES=fixown on a host that already has the group, users isn't + # run and SPINIFEX_GROUP is unset — default it here. + SPINIFEX_GROUP="${SPINIFEX_GROUP:-spinifex}" info "Fixing file ownership for privilege separation..." # Per-service data dirs — recursive chown so existing files are accessible @@ -471,6 +564,7 @@ run_migrations() { # --- Install systemd units --- install_systemd() { + stage "installing systemd units" info "Installing systemd units..." if [ ! -d "$EXTRACT_DIR/systemd" ]; then @@ -482,6 +576,12 @@ install_systemd() { info " /etc/systemd/system/$(basename "$unit")" done + # daemon-reload / enable require a running systemd — skip inside the ISO + # chroot. Unit files are still dropped into place; firstboot enables them. + if [ "${ISO_BUILD:-0}" = "1" ]; then + info "Systemd units installed (ISO_BUILD=1, skipping daemon-reload/enable)" + return + fi $SUDO systemctl daemon-reload $SUDO systemctl enable spinifex.target info "Systemd units installed and enabled (per-service users)" @@ -489,6 +589,7 @@ install_systemd() { # --- Install logrotate --- install_logrotate() { + stage "installing logrotate config" if [ -f "$EXTRACT_DIR/logrotate-spinifex" ]; then $SUDO install -m 0644 "$EXTRACT_DIR/logrotate-spinifex" /etc/logrotate.d/spinifex else @@ -505,6 +606,15 @@ handle_upgrade() { $SUDO systemctl stop spinifex.target RESTART_AFTER=true fi + + # Pre-tightening hosts have /run/spinifex/nbd as root:spinifex 0770, which + # grants access to every spinifex-* service. While services are stopped, + # bring it up to the new spinifex-viperblock:spinifex-viperblock 0770 + # policy so restart picks up the fresh group membership without a reboot. + if [ -d /run/spinifex/nbd ]; then + $SUDO chown spinifex-viperblock:spinifex-viperblock /run/spinifex/nbd 2>/dev/null || true + $SUDO chmod 0770 /run/spinifex/nbd 2>/dev/null || true + fi } restart_if_needed() { @@ -558,22 +668,50 @@ main() { info "Spinifex installer" echo "" + # Always needed: sudo resolution, OS/arch detection. setup_sudo detect_os detect_arch - handle_upgrade - install_apt_deps - install_aws_cli - create_service_users - install_sudoers - download_spinifex - install_files - create_directories - fix_file_ownership - install_systemd - run_migrations - install_logrotate - rm -rf "$SPINIFEX_TMPDIR" + + # Orchestration (stop running services, fix stale /run/spinifex/nbd perms) + # only applies in full-install mode on a live host. + if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then + handle_upgrade + fi + + # Stages that need EXTRACT_DIR: files, directories, systemd, logrotate. + # Only download/extract when at least one such stage is enabled. + if stage_enabled files || stage_enabled directories \ + || stage_enabled systemd || stage_enabled logrotate; then + download_spinifex + fi + + stage_enabled deps && install_apt_deps + stage_enabled aws && install_aws_cli + stage_enabled users && create_service_users + stage_enabled sudoers && install_sudoers + stage_enabled files && install_files + stage_enabled directories && create_directories + stage_enabled env && install_systemd_env + stage_enabled fixown && fix_file_ownership + stage_enabled systemd && install_systemd + stage_enabled logrotate && install_logrotate + + # Migrations are only safe on a live system (need a running NATS and a + # persisted config file). Skip under ISO_BUILD and under any explicit + # SETUP_STAGES filter that doesn't list migrations. + if [ "${ISO_BUILD:-0}" != "1" ] && stage_enabled migrations; then + run_migrations + fi + + [ -n "${SPINIFEX_TMPDIR:-}" ] && rm -rf "$SPINIFEX_TMPDIR" + + # Post-install orchestration (service restart, summary, newgrp) only in + # full-install mode on a live host. + if [ "${ISO_BUILD:-0}" = "1" ] || [ -n "${SETUP_STAGES:-}" ]; then + return + fi + restart_if_needed print_summary