Latest changes re ISO and binary install path

This commit is contained in:
Ben Duncan
2026-04-20 15:14:32 +10:00
parent 728cdd972b
commit 27d7fdd12b
3 changed files with 183 additions and 67 deletions
+10 -32
View File
@@ -97,16 +97,6 @@ fi
# Set hostname
hostnamectl set-hostname %s
# NOTE: We do NOT invoke /usr/local/share/spinifex/setup.sh here. The ISO
# rootfs (scripts/iso-builder/build/build-rootfs.sh) already pre-stages
# everything setup.sh would do at install time: spx binary, viperblock
# nbdkit plugin, service users (spinifex-{nats,gw,daemon,storage,viperblock,
# vpcd,ui}), data directories with per-service ownership, sudoers rules,
# systemd units, and tmpfiles.d entries. Calling setup.sh would also fail
# in the air-gapped boot environment because no tarball is staged at
# /opt/spinifex/spinifex.tar.gz — the firstboot would never complete and
# every spinifex-* service depending on it would refuse to start.
# Pre-start OVS and OVN central so their databases are initialised before
# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
# longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
@@ -139,28 +129,16 @@ resolvconf -u
# Cluster formation — capture credentials to file for display on console.
%s 2>&1
# Fix ownership of files spx admin init created. spx runs as root (no
# SUDO_USER under systemd), so /etc/spinifex/{spinifex.toml,master.key} and
# any per-service files written under /var/lib/spinifex/* land as root:root.
# Re-chown them so each service can read/write its own subtree at
# Type=notify start. The directories themselves were pre-created with the
# correct ownership in build-rootfs.sh — these chowns only affect the
# runtime-generated config and key files inside them.
chown root:spinifex /etc/spinifex && chmod 750 /etc/spinifex
find /etc/spinifex -type f -exec chmod 640 {} \;
chown -R spinifex-gw:spinifex /var/lib/spinifex/awsgw
chown -R spinifex-daemon:spinifex /var/lib/spinifex/spinifex
chown -R spinifex-nats:spinifex /var/lib/spinifex/nats
chown -R spinifex-storage:spinifex /var/lib/spinifex/predastore
chown -R spinifex-viperblock:spinifex /var/lib/spinifex/viperblock
chown -R spinifex-vpcd:spinifex /var/lib/spinifex/vpcd
# awsgw looks for master.key at <BaseDir>/config/master.key. In production
# BaseDir is /var/lib/spinifex/awsgw/ (set by SPINIFEX_BASE_DIR), but the
# key lives in /etc/spinifex/. Symlink so both paths resolve to the same
# file. spx admin init writes the key, so this must run after it.
mkdir -p /var/lib/spinifex/awsgw/config
ln -sf /etc/spinifex/master.key /var/lib/spinifex/awsgw/config/master.key
# Fix ownership of files spx admin init wrote. spx runs as root under
# systemd, so /etc/spinifex/{spinifex.toml,master.key,ca.key,*.pem} and any
# per-service files written under /var/lib/spinifex/* land as root:root.
# Delegate to setup.sh's fix_file_ownership (single source of truth) — it
# knows the per-file modes (ca.key 0600, *.pem 0644, toml/key 0640) that a
# blunt recursive chmod would clobber.
SETUP_STAGES=fixown /usr/local/share/spinifex/setup.sh
# The /var/lib/spinifex/awsgw/config symlink was created at build time by
# setup.sh's create_directories — it resolves to /etc/spinifex, so
# {BaseDir}/config/master.key automatically points at /etc/spinifex/master.key.
# Copy AWS credentials to the spinifex user's home directory.
# spx admin init runs with HOME=/root (set by the systemd unit), so credentials
+1 -1
View File
@@ -13,7 +13,7 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp
| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | TLS cert/key, base-path, config-path required | Creates predastore service instance with S3-compatible storage backend → starts service | **DONE** |
| `spx service predastore stop` | — | Predastore must be running | Stops the predastore service | **DONE** |
| `spx service predastore status` | — | None | Reports predastore service status | **DONE** |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (default: /opt/spinifex/lib/nbdkit-viperblock-plugin.so) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** |
| `spx service viperblock stop` | — | Viperblock must be running | Stops the viperblock service | **DONE** |
| `spx service viperblock status` | — | None | Reports viperblock service status | **DONE** |
| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | None | Starts embedded NATS server with optional JetStream | **DONE** |
+172 -34
View File
@@ -9,6 +9,14 @@
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip apt dependency install
# INSTALL_SPINIFEX_SKIP_AWS Set to 1 to skip AWS CLI install
# INSTALL_SPINIFEX_SKIP_NEWGRP Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
# ISO_BUILD Set to 1 when running inside a debootstrap chroot from the ISO
# builder: skip handle_upgrade/restart/migrations/newgrp/print_summary,
# skip systemctl daemon-reload + enable, short-circuit setup_sudo.
# VERBOSE Set to 1 to echo "[setup] <stage>" before each top-level step.
# SETUP_STAGES Comma-separated subset of stages to run:
# deps, aws, users, sudoers, files, directories,
# env, systemd, logrotate, fixown, migrations
# Unset = run every stage appropriate for the current mode.
set -e
@@ -25,8 +33,26 @@ info() { echo -e "${GREEN}[INFO]${NC} $*"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
stage() {
[ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
return 0
}
stage_enabled() {
[ -z "${SETUP_STAGES:-}" ] && return 0
case ",${SETUP_STAGES}," in
*",$1,"*) return 0 ;;
*) return 1 ;;
esac
}
# --- Sudo setup ---
setup_sudo() {
# Inside a debootstrap chroot we are already root and sudo may not be installed.
if [ "${ISO_BUILD:-0}" = "1" ]; then
SUDO=""
return
fi
if [ "$(id -u)" -eq 0 ]; then
SUDO=""
elif command -v sudo >/dev/null 2>&1; then
@@ -92,6 +118,7 @@ detect_arch() {
# --- Create per-service system users ---
create_service_users() {
stage "creating spinifex group and per-service users"
SPINIFEX_GROUP="spinifex"
# Create shared group
@@ -99,6 +126,13 @@ create_service_users() {
$SUDO groupadd --system "$SPINIFEX_GROUP"
fi
# Producer-typed access group for viperblock's runtime resources
# (/run/spinifex/nbd socket dir). Viperblock writes as owner; daemon reads
# via supplementary membership; every other spinifex-* service is excluded.
if ! getent group spinifex-viperblock > /dev/null 2>&1; then
$SUDO groupadd --system spinifex-viperblock
fi
# Create per-service users with correct home directories
declare -A SERVICE_HOMES=(
[nats]="/var/lib/spinifex/nats"
@@ -120,10 +154,17 @@ create_service_users() {
fi
done
# Add invoking user to spinifex group for admin CLI access
ADMIN_USER="${SUDO_USER:-$(whoami)}"
if [ "$ADMIN_USER" != "root" ]; then
$SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
# Add invoking user to spinifex group for admin CLI access.
# Skip under ISO_BUILD: the chroot has no invoking user (tf-user,
# whoever ran `sudo make`, etc. don't exist in the rootfs). The ISO's
# interactive 'spinifex' login account is created later in Phase 4 of
# build-rootfs.sh with spinifex as its primary gid. In curl|bash mode
# guard against a stale/missing SUDO_USER too.
if [ "${ISO_BUILD:-0}" != "1" ]; then
ADMIN_USER="${SUDO_USER:-$(whoami)}"
if [ "$ADMIN_USER" != "root" ] && id -u "$ADMIN_USER" > /dev/null 2>&1; then
$SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
fi
fi
# KVM access for daemon
@@ -131,11 +172,15 @@ create_service_users() {
$SUDO usermod -aG kvm spinifex-daemon
fi
# Daemon consumes viperblock's NBD socket — join the producer-typed group.
$SUDO usermod -aG spinifex-viperblock spinifex-daemon
info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})"
}
# --- Install scoped sudoers rules ---
install_sudoers() {
stage "installing scoped sudoers rules"
$SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs
spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
@@ -154,7 +199,12 @@ SUDOERS
}
# --- Install apt dependencies ---
# NOTE: Runtime deps must stay in sync with the ISO package list at
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
# remove, or change a runtime package here, review that file too — drift means
# the ISO and `curl | bash` paths install different software.
install_apt_deps() {
stage "installing apt dependencies"
if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
return
@@ -176,6 +226,7 @@ install_apt_deps() {
# --- Install AWS CLI ---
install_aws_cli() {
stage "installing AWS CLI v2"
if [ "${INSTALL_SPINIFEX_SKIP_AWS}" = "1" ]; then
info "Skipping AWS CLI (INSTALL_SPINIFEX_SKIP_AWS=1)"
return
@@ -198,6 +249,7 @@ install_aws_cli() {
# --- Download tarball ---
download_spinifex() {
stage "downloading/extracting spinifex release tarball"
SPINIFEX_TMPDIR=$(mktemp -d)
TARBALL="$SPINIFEX_TMPDIR/spinifex.tar.gz"
@@ -249,6 +301,7 @@ download_spinifex() {
# --- Place files ---
install_files() {
stage "installing binaries and scripts"
info "Installing files..."
# Binary
@@ -276,10 +329,17 @@ install_files() {
info " /usr/local/share/spinifex/setup-ovn.sh"
fi
# Install setup.sh itself so firstboot and future re-runs can find it at a
# stable path on both ISO and curl|bash installs.
if [ -f "$EXTRACT_DIR/setup.sh" ]; then
$SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
info " /usr/local/share/spinifex/setup.sh"
fi
}
# --- Create directories ---
create_directories() {
stage "creating spinifex directory layout"
info "Creating directories..."
# Top-level directories (root-owned, group-readable by spinifex)
@@ -305,13 +365,20 @@ create_directories() {
$SUDO chmod 0775 /var/log/spinifex
$SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex
$SUDO mkdir -p /run/spinifex
$SUDO chmod 0775 /run/spinifex
$SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex
$SUDO mkdir -p /run/spinifex/nbd
$SUDO chmod 0770 /run/spinifex/nbd
$SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex/nbd
# /run/spinifex and /run/spinifex/nbd are declared via tmpfiles.d because
# /run is tmpfs — direct mkdir doesn't survive reboot, and units have
# ReadWritePaths= on these paths which fails namespace setup with ENOENT
# if the dirs are absent at service start.
$SUDO install -m 0644 /dev/stdin /etc/tmpfiles.d/spinifex.conf <<'TMPEOF'
# Type Path Mode User Group Age
d /run/spinifex 0770 root spinifex -
d /run/spinifex/nbd 0770 spinifex-viperblock spinifex-viperblock -
TMPEOF
if [ "${ISO_BUILD:-0}" != "1" ]; then
# Live systems: materialise the runtime dirs immediately so a re-run of
# setup.sh on an existing host has correct /run state without a reboot.
$SUDO systemd-tmpfiles --create /etc/tmpfiles.d/spinifex.conf 2>/dev/null || true
fi
# Per-service config directories
$SUDO mkdir -p /etc/spinifex/nats
@@ -356,16 +423,6 @@ create_directories() {
$SUDO ln -s /etc/spinifex /var/lib/spinifex/awsgw/config
fi
# Top-level config files (root-owned, group-readable)
# Generate environment file with install-specific values (e.g. arch-dependent paths)
$SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
# Generated by setup.sh — install-specific environment variables
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
EOF
$SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
$SUDO chmod 0640 /etc/spinifex/systemd.env
info "Generated /etc/spinifex/systemd.env"
# Service helper scripts (root-owned, group-executable by all service users)
if [ -d "$EXTRACT_DIR/scripts" ]; then
for script in "$EXTRACT_DIR"/scripts/*.sh; do
@@ -376,8 +433,44 @@ EOF
fi
}
# --- Generate systemd environment file ---
# Split out of create_directories so the `env` stage can be refreshed
# independently (e.g. by inject-bins.sh when the plugin path changes).
install_systemd_env() {
stage "generating /etc/spinifex/systemd.env"
# nbdkit plugin path is arch-dependent (`nbdkit --dump-config` returns
# /usr/lib/{x86_64,aarch64}-linux-gnu/nbdkit/plugins). Resolve it here so
# systemd.env always matches wherever install_files placed the .so.
local plugindir
plugindir=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2)
if [ -z "$plugindir" ]; then
if [ "${ARCH:-}" = "arm64" ]; then
plugindir="/usr/lib/aarch64-linux-gnu/nbdkit/plugins"
else
plugindir="/usr/lib/x86_64-linux-gnu/nbdkit/plugins"
fi
fi
$SUDO mkdir -p /etc/spinifex
$SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
# Generated by setup.sh — install-specific environment variables
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${plugindir}/nbdkit-viperblock-plugin.so
EOF
$SUDO chown "spinifex-viperblock:${SPINIFEX_GROUP:-spinifex}" /etc/spinifex/systemd.env
$SUDO chmod 0640 /etc/spinifex/systemd.env
info "Generated /etc/spinifex/systemd.env"
}
# --- Fix file ownership for upgrades from v1 ---
# Also invoked from firstboot via SETUP_STAGES=fixown to correct ownership of
# files that `spx admin init` wrote as root:root.
fix_file_ownership() {
stage "fixing file ownership for privilege separation"
# create_service_users normally sets this; if we're invoked via
# SETUP_STAGES=fixown on a host that already has the group, users isn't
# run and SPINIFEX_GROUP is unset — default it here.
SPINIFEX_GROUP="${SPINIFEX_GROUP:-spinifex}"
info "Fixing file ownership for privilege separation..."
# Per-service data dirs — recursive chown so existing files are accessible
@@ -471,6 +564,7 @@ run_migrations() {
# --- Install systemd units ---
install_systemd() {
stage "installing systemd units"
info "Installing systemd units..."
if [ ! -d "$EXTRACT_DIR/systemd" ]; then
@@ -482,6 +576,12 @@ install_systemd() {
info " /etc/systemd/system/$(basename "$unit")"
done
# daemon-reload / enable require a running systemd — skip inside the ISO
# chroot. Unit files are still dropped into place; firstboot enables them.
if [ "${ISO_BUILD:-0}" = "1" ]; then
info "Systemd units installed (ISO_BUILD=1, skipping daemon-reload/enable)"
return
fi
$SUDO systemctl daemon-reload
$SUDO systemctl enable spinifex.target
info "Systemd units installed and enabled (per-service users)"
@@ -489,6 +589,7 @@ install_systemd() {
# --- Install logrotate ---
install_logrotate() {
stage "installing logrotate config"
if [ -f "$EXTRACT_DIR/logrotate-spinifex" ]; then
$SUDO install -m 0644 "$EXTRACT_DIR/logrotate-spinifex" /etc/logrotate.d/spinifex
else
@@ -505,6 +606,15 @@ handle_upgrade() {
$SUDO systemctl stop spinifex.target
RESTART_AFTER=true
fi
# Pre-tightening hosts have /run/spinifex/nbd as root:spinifex 0770, which
# grants access to every spinifex-* service. While services are stopped,
# bring it up to the new spinifex-viperblock:spinifex-viperblock 0770
# policy so restart picks up the fresh group membership without a reboot.
if [ -d /run/spinifex/nbd ]; then
$SUDO chown spinifex-viperblock:spinifex-viperblock /run/spinifex/nbd 2>/dev/null || true
$SUDO chmod 0770 /run/spinifex/nbd 2>/dev/null || true
fi
}
restart_if_needed() {
@@ -558,22 +668,50 @@ main() {
info "Spinifex installer"
echo ""
# Always needed: sudo resolution, OS/arch detection.
setup_sudo
detect_os
detect_arch
handle_upgrade
install_apt_deps
install_aws_cli
create_service_users
install_sudoers
download_spinifex
install_files
create_directories
fix_file_ownership
install_systemd
run_migrations
install_logrotate
rm -rf "$SPINIFEX_TMPDIR"
# Orchestration (stop running services, fix stale /run/spinifex/nbd perms)
# only applies in full-install mode on a live host.
if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then
handle_upgrade
fi
# Stages that need EXTRACT_DIR: files, directories, systemd, logrotate.
# Only download/extract when at least one such stage is enabled.
if stage_enabled files || stage_enabled directories \
|| stage_enabled systemd || stage_enabled logrotate; then
download_spinifex
fi
stage_enabled deps && install_apt_deps
stage_enabled aws && install_aws_cli
stage_enabled users && create_service_users
stage_enabled sudoers && install_sudoers
stage_enabled files && install_files
stage_enabled directories && create_directories
stage_enabled env && install_systemd_env
stage_enabled fixown && fix_file_ownership
stage_enabled systemd && install_systemd
stage_enabled logrotate && install_logrotate
# Migrations are only safe on a live system (need a running NATS and a
# persisted config file). Skip under ISO_BUILD and under any explicit
# SETUP_STAGES filter that doesn't list migrations.
if [ "${ISO_BUILD:-0}" != "1" ] && stage_enabled migrations; then
run_migrations
fi
[ -n "${SPINIFEX_TMPDIR:-}" ] && rm -rf "$SPINIFEX_TMPDIR"
# Post-install orchestration (service restart, summary, newgrp) only in
# full-install mode on a live host.
if [ "${ISO_BUILD:-0}" = "1" ] || [ -n "${SETUP_STAGES:-}" ]; then
return
fi
restart_if_needed
print_summary