feat: privilege separation (#114)

scripts/dev-install.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# dev-install.sh — Full local development setup via production installer.
+# Builds from source, assembles a tarball, runs setup.sh for scaffolding
+# (user creation, directories, systemd units), initializes the cluster,
+# and starts services via systemd.
+#
+# Usage: ./scripts/dev-install.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
+
+echo "=== Building ==="
+cd "$PROJECT_ROOT" && make build
+cd "$MULGA_ROOT/viperblock" && make go_build_nbd
+
+echo "=== Assembling tarball for setup.sh ==="
+STAGING=$(mktemp -d)
+trap 'rm -rf "$STAGING"' EXIT
+cp "$PROJECT_ROOT/bin/spx" "$STAGING/spx"
+cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
+cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
+mkdir -p "$STAGING/systemd"
+cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
+mkdir -p "$STAGING/scripts"
+cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
+cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
+tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
+
+echo "=== Cleaning stale state from previous installs ==="
+# Stop any running services before modifying files
+sudo systemctl stop spinifex.target 2>/dev/null || true
+sudo systemctl reset-failed 2>/dev/null || true
+
+# Remove stale files owned by the dev user in production paths.
+# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave
+# files owned by tf-user that service users (spinifex-nats, etc.) can't read
+# under systemd's ProtectSystem=strict sandboxing.
+for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do
+    if [ -d "$dir" ]; then
+        # Remove PID files, stale logs, and the legacy ~/spinifex/config symlink
+        sudo find "$dir" -name '*.pid' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log.*' -delete 2>/dev/null || true
+    fi
+done
+# Remove legacy data dir contents that conflict with production layout
+if [ -d /var/lib/spinifex/config ]; then
+    sudo rm -rf /var/lib/spinifex/config
+fi
+
+echo "=== Running setup.sh (creates users, dirs, systemd units) ==="
+sudo INSTALL_SPINIFEX_TARBALL=/tmp/spinifex-local.tar.gz bash "$PROJECT_ROOT/scripts/setup.sh"
+rm -f /tmp/spinifex-local.tar.gz
+
+echo "=== Setting up OVN ==="
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+
+echo "=== Initializing ==="
+sudo spx admin init --force --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+
+echo "=== Installing CA certificate ==="
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+
+echo "=== Starting services ==="
+sudo systemctl start spinifex.target
+
+echo "=== Done ==="
+echo "Services: sudo systemctl status spinifex.target"
+echo "Logs:     journalctl -u 'spinifex-*' -f"
+echo "Test:     AWS_PROFILE=spinifex aws ec2 describe-instances"
+echo "Iterate:  make deploy (rebuild + restart all)"
+
+# Check if user needs to re-login for spinifex group membership
+if ! id -Gn 2>/dev/null | grep -qw spinifex; then
+    echo ""
+    echo "NOTE: You were added to the 'spinifex' group but your current"
+    echo "session doesn't reflect it yet. Log out and back in, or run:"
+    echo "  newgrp spinifex"
+fi

scripts/dev-setup.sh

@@ -165,24 +165,16 @@ else
 fi
 
 echo ""
-echo "🎉 Development environment setup complete!"
+echo "Development environment setup complete!"
 echo ""
-echo "📋 Next steps:"
+echo "Recommended: Use scripts/dev-install.sh for a production-parity setup"
+echo "  (systemd, privilege separation, production paths)."
 echo ""
-echo "   1. Setup OVN networking:"
-echo "      ./scripts/setup-ovn.sh --management"
+echo "  scripts/dev-install.sh  # One-time: build, install, init, start via systemd"
+echo "  make deploy             # Daily: rebuild + copy binary + restart all"
+echo "  make reinstall          # Changed a .service file? Re-run full setup"
 echo ""
-echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):"
-echo "      sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
-echo ""
-echo "   3. Start services:"
-echo "      ./scripts/start-dev.sh"
-echo ""
-echo "🔧 Development endpoints:"
-echo "   - Spinifex Gateway:  https://localhost:$SPINIFEX_GATEWAY_PORT"
-echo "   - Predastore S3: https://localhost:$PREDASTORE_PORT"
-echo "   - NATS:          nats://localhost:$NATS_PORT"
-echo ""
-echo "🧪 Test with AWS CLI:"
-echo "   export AWS_PROFILE=spinifex"
-echo "   aws ec2 describe-instances"
+echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):"
+echo "  ./scripts/setup-ovn.sh --management"
+echo "  sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
+echo "  ./scripts/start-dev.sh"

scripts/setup-ovn.sh

@@ -565,21 +565,13 @@ else
     echo "  systemd override: already exists"
 fi
 
-# Create sudoers rule for network commands that always need root
-# (ip tuntap, ip link set — NET_ADMIN operations)
+# Sudoers rules for spinifex-daemon and spinifex-vpcd are managed by setup.sh
+# (install_sudoers). Skip writing here to avoid conflicts.
 SUDOERS_FILE="/etc/sudoers.d/spinifex-network"
-if [ ! -f "$SUDOERS_FILE" ]; then
-    CURRENT_USER=$(whoami)
-    sudo tee "$SUDOERS_FILE" >/dev/null <<EOF
-# Spinifex VPC networking: allow non-root daemon to manage tap devices and OVS
-$CURRENT_USER ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
-EOF
-    sudo chmod 0440 "$SUDOERS_FILE"
-    echo "  sudoers rule: created ($SUDOERS_FILE)"
+if [ -f "$SUDOERS_FILE" ]; then
+    echo "  sudoers rule: already exists ($SUDOERS_FILE, managed by setup.sh)"
 else
-    echo "  sudoers rule: already exists"
+    echo "  sudoers rule: not found — run setup.sh first, or install manually"
 fi
 
 # --- Step 9: Configure OVN log rotation ---

scripts/setup.sh

@@ -89,27 +89,66 @@ detect_arch() {
     info "Detected architecture: $MACHINE ($ARCH)"
 }
 
-# --- Detect service user ---
-detect_service_user() {
-    # Use the real user — either the current user, or SUDO_USER if running via sudo
-    SPINIFEX_USER="${SUDO_USER:-$(whoami)}"
-    SPINIFEX_GROUP="$(id -gn "$SPINIFEX_USER")"
+# --- Create per-service system users ---
+create_service_users() {
+    SPINIFEX_GROUP="spinifex"
 
-    if [ "$SPINIFEX_USER" = "root" ]; then
-        fatal "Cannot determine service user. Do not run as root directly — use: curl ... | bash (as a normal user)"
+    # Create shared group
+    if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
+        $SUDO groupadd --system "$SPINIFEX_GROUP"
     fi
 
-    info "Services will run as: $SPINIFEX_USER:$SPINIFEX_GROUP"
+    # Create per-service users with correct home directories
+    declare -A SERVICE_HOMES=(
+        [nats]="/var/lib/spinifex/nats"
+        [gw]="/var/lib/spinifex"
+        [daemon]="/var/lib/spinifex/spinifex"
+        [storage]="/var/lib/spinifex/predastore"
+        [viperblock]="/var/lib/spinifex/viperblock"
+        [vpcd]="/var/lib/spinifex"
+        [ui]="/var/lib/spinifex"
+    )
+    for svc in nats gw daemon storage viperblock vpcd ui; do
+        local user="spinifex-${svc}"
+        if ! id "$user" > /dev/null 2>&1; then
+            $SUDO useradd --system --no-create-home \
+                --home-dir "${SERVICE_HOMES[$svc]}" \
+                --gid "$SPINIFEX_GROUP" \
+                --shell /usr/sbin/nologin \
+                "$user"
+        fi
+    done
 
-    # Allow the service user to manage OVN/OVS and tap devices
-    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
-# Spinifex VPC networking: allow service user to manage tap devices, OVS, and OVN
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+    # Add invoking user to spinifex group for admin CLI access
+    ADMIN_USER="${SUDO_USER:-$(whoami)}"
+    if [ "$ADMIN_USER" != "root" ]; then
+        $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
+    fi
+
+    # KVM access for daemon
+    if getent group kvm > /dev/null 2>&1; then
+        $SUDO usermod -aG kvm spinifex-daemon
+    fi
+
+    info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})"
+}
+
+# --- Install scoped sudoers rules ---
+install_sudoers() {
+    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
+# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs
+spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
+spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
+
+# Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
 SUDOERS
     $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
-    info "Sudoers rules installed for $SPINIFEX_USER"
+    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
+    info "Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd"
 }
 
 # --- Install apt dependencies ---
@@ -241,12 +280,15 @@ install_files() {
 create_directories() {
     info "Creating directories..."
 
+    # Top-level directories (root-owned, group-readable by spinifex)
     $SUDO mkdir -p /etc/spinifex
-    $SUDO chmod 0700 /etc/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex
+    $SUDO chmod 0750 /etc/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex
 
     $SUDO mkdir -p /var/lib/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex
+    # TODO: tighten to 0750 once awsgw gets its own data dir (currently writes awsgw.pid here)
+    $SUDO chmod 0770 /var/lib/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex
 
     # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/
     if [ ! -e /var/lib/spinifex/config ]; then
@@ -259,26 +301,57 @@ create_directories() {
     fi
 
     $SUDO mkdir -p /var/log/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/log/spinifex
+    $SUDO chmod 0775 /var/log/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex
 
     $SUDO mkdir -p /run/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /run/spinifex
+    $SUDO chmod 0775 /run/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex
+
+    # Per-service config directories
+    $SUDO mkdir -p /etc/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
+    $SUDO chmod 0750 /etc/spinifex/nats
+
+    $SUDO mkdir -p /etc/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
+    $SUDO chmod 0750 /etc/spinifex/predastore
+
+    # Per-service data directories
+    $SUDO mkdir -p /var/lib/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
+    $SUDO chmod 0700 /var/lib/spinifex/nats
+
+    $SUDO mkdir -p /var/lib/spinifex/spinifex
+    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
+    $SUDO chmod 0700 /var/lib/spinifex/spinifex
+
+    $SUDO mkdir -p /var/lib/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
+    $SUDO chmod 0700 /var/lib/spinifex/predastore
 
     $SUDO mkdir -p /var/lib/spinifex/viperblock
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chmod 0700 /var/lib/spinifex/viperblock
 
+    $SUDO mkdir -p /var/lib/spinifex/vpcd
+    $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
+    $SUDO chmod 0700 /var/lib/spinifex/vpcd
+
+    # Top-level config files (root-owned, group-readable)
     # Generate environment file with install-specific values (e.g. arch-dependent paths)
     $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
 # Generated by setup.sh — install-specific environment variables
 SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
 EOF
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chmod 0640 /etc/spinifex/systemd.env
     info "Generated /etc/spinifex/systemd.env"
 
-    # Service helper scripts
+    # Service helper scripts (root-owned, group-executable by all service users)
     if [ -d "$EXTRACT_DIR/scripts" ]; then
         for script in "$EXTRACT_DIR"/scripts/*.sh; do
-            $SUDO install -o "$SPINIFEX_USER" -g "$SPINIFEX_GROUP" -m 0755 \
+            $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
                 "$script" "/var/lib/spinifex/$(basename "$script")"
             info "  /var/lib/spinifex/$(basename "$script")"
         done
@@ -294,15 +367,12 @@ install_systemd() {
     fi
 
     for unit in "$EXTRACT_DIR"/systemd/*; do
-        # Substitute User= and Group= with the detected service user
-        sed "s/^User=spinifex$/User=$SPINIFEX_USER/;s/^Group=spinifex$/Group=$SPINIFEX_GROUP/" \
-            "$unit" | $SUDO tee "/etc/systemd/system/$(basename "$unit")" > /dev/null
-        $SUDO chmod 0644 "/etc/systemd/system/$(basename "$unit")"
+        $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
         info "  /etc/systemd/system/$(basename "$unit")"
     done
 
     $SUDO systemctl daemon-reload
-    info "Systemd units installed (running as $SPINIFEX_USER:$SPINIFEX_GROUP)"
+    info "Systemd units installed (per-service users)"
 }
 
 # --- Install logrotate ---
@@ -343,7 +413,7 @@ print_summary() {
     echo ""
     echo "  Version:      $INSTALLED_VERSION"
     echo "  Architecture: $ARCH"
-    echo "  Service user: $SPINIFEX_USER"
+    echo "  Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
     echo "  Binary:       /usr/local/bin/spx"
     echo "  Config:       /etc/spinifex/"
     echo "  Data:         /var/lib/spinifex/"
@@ -384,7 +454,8 @@ main() {
     handle_upgrade
     install_apt_deps
     install_aws_cli
-    detect_service_user
+    create_service_users
+    install_sudoers
     download_spinifex
     install_files
     create_directories

scripts/start-dev.sh

@@ -11,6 +11,11 @@
 
 set -euo pipefail
 
+echo "WARNING: start-dev.sh is deprecated for local development." >&2
+echo "  Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2
+echo "  start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2
+echo "" >&2
+
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
@@ -33,6 +38,8 @@ echo "Using configuration directory: $CONFIG_DIR"
 LOGS_DIR="$DATA_DIR/logs"
 WAL_DIR="$DATA_DIR/spinifex"
 
+echo "⚠️  Dev mode: all services run as $(whoami) without privilege separation"
+echo ""
 echo "🚀 Starting Spinifex development environment..."
 echo "Project root: $PROJECT_ROOT"
 echo "Data directory: $DATA_DIR"