diff --git a/scripts/clone-deps.sh b/scripts/clone-deps.sh index bb8bffe88..8eb63de19 100755 --- a/scripts/clone-deps.sh +++ b/scripts/clone-deps.sh @@ -75,5 +75,5 @@ echo "" echo "๐ŸŽ‰ Development environment setup complete!" echo "" echo "Next step:" -echo "Run development setup to build services: ./scripts/dev-setup.sh" +echo "Run development setup to build, install, and start services: ./scripts/dev-install.sh" echo "" diff --git a/scripts/dev-install.sh b/scripts/dev-install.sh index e3a86969a..408fc9160 100755 --- a/scripts/dev-install.sh +++ b/scripts/dev-install.sh @@ -44,9 +44,9 @@ sudo systemctl stop spinifex.target 2>/dev/null || true sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true # Remove stale files owned by the dev user in production paths. -# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave -# files owned by tf-user that service users (spinifex-nats, etc.) can't read -# under systemd's ProtectSystem=strict sandboxing. +# A prior admin init run without sudo can leave files owned by tf-user that +# service users (spinifex-nats, etc.) can't read under systemd's +# ProtectSystem=strict sandboxing. for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do if [ -d "$dir" ]; then # Remove PID files, stale logs, and the legacy ~/spinifex/config symlink diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh deleted file mode 100755 index 3412a2984..000000000 --- a/scripts/dev-setup.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash - -# Spinifex Platform Development Environment Setup -# This script sets up a complete development environment for the Spinifex platform - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" - -# Default configuration -NATS_PORT=4222 -PREDASTORE_PORT=8443 -SPINIFEX_GATEWAY_PORT=9999 - -echo "๐Ÿš€ Setting up Spinifex development environment..." -echo "Project root: $PROJECT_ROOT" - -# Function to check if command exists -command_exists() { - command -v "$1" >/dev/null 2>&1 -} - -# Function to check if port is available -port_available() { - ! nc -z localhost "$1" 2>/dev/null -} - -# Check dependencies -echo "๐Ÿ” Checking dependencies..." - -# Check for required commands -required_commands=("go" "make") -missing_required=false -for cmd in "${required_commands[@]}"; do - if command_exists "$cmd"; then - echo "โœ… $cmd found" - else - echo "โŒ $cmd not found. Please install $cmd" - missing_required=true - case "$cmd" in - "go") - echo " Install Go: https://golang.org/dl/" - ;; - esac - fi -done -if [ "$missing_required" = true ]; then - echo "โŒ Missing required dependencies. Cannot continue." - exit 1 -fi - -# Check optional commands -optional_commands=("air" "nbdkit") -for cmd in "${optional_commands[@]}"; do - if command_exists "$cmd"; then - echo "โœ… $cmd found (optional)" - else - echo "โš ๏ธ $cmd not found (optional)" - case "$cmd" in - "air") - echo " Install air for hot reloading: go install github.com/air-verse/air@latest" - ;; - "nbdkit") - echo " Install nbdkit: sudo apt-get install nbdkit (Ubuntu/Debian)" - ;; - esac - fi -done - -# Check OVN/OVS commands (required for VPC networking) -echo "" -echo "๐Ÿ” Checking OVN/OVS dependencies..." -ovn_commands=("ovs-vsctl" "ovn-controller" "ovn-nbctl") -ovn_ready=true -for cmd in "${ovn_commands[@]}"; do - if command_exists "$cmd"; then - echo "โœ… $cmd found" - else - echo "โš ๏ธ $cmd not found (required for VPC networking)" - ovn_ready=false - fi -done -if [ "$ovn_ready" = false ]; then - echo " Install OVN: sudo apt-get install ovn-central ovn-host openvswitch-switch" - echo " Or run: make quickinstall" -fi - -# Initialize br-int if OVS is available -if command_exists "ovs-vsctl"; then - echo "" - echo "๐ŸŒ Checking OVS br-int bridge..." - if ovs-vsctl br-exists br-int 2>/dev/null; then - echo "โœ… br-int already exists" - else - echo "โš ๏ธ br-int not found. Run ./scripts/setup-ovn.sh to initialize OVN networking" - fi -fi - -# Check component repositories -echo "" -echo "๐Ÿ“ฆ Checking component repositories..." - -components=("viperblock" "predastore") -for component in "${components[@]}"; do - component_path="$MULGA_ROOT/$component" - if [[ -d "$component_path" ]]; then - echo "โœ… $component found at $component_path" - if [[ -f "$component_path/go.mod" ]]; then - echo " Go module verified" - else - echo " โš ๏ธ No go.mod found in $component" - fi - else - echo "โŒ $component not found at $component_path" - echo " Run: ./scripts/clone-deps.sh" - fi -done - -# Check port availability -echo "" -echo "๐Ÿ”Œ Checking port availability..." - -ports=("$NATS_PORT:NATS" "$PREDASTORE_PORT:Predastore" "$SPINIFEX_GATEWAY_PORT:Spinifex Gateway") -for port_info in "${ports[@]}"; do - port="${port_info%:*}" - service="${port_info#*:}" - if port_available "$port"; then - echo "โœ… Port $port available for $service" - else - echo "โš ๏ธ Port $port already in use (needed for $service)" - fi -done - -# Build Spinifex first (needed for admin init) -echo "" -echo "๐Ÿ”จ Building Spinifex..." -cd "$PROJECT_ROOT" -make build -echo "โœ… Spinifex built successfully" - -# Build components -echo "" -echo "๐Ÿ”จ Building components..." - -# Build Viperblock -if [[ -d "$MULGA_ROOT/viperblock" ]]; then - echo "๐Ÿ“ฆ Building Viperblock..." - cd "$MULGA_ROOT/viperblock" - make build - echo "โœ… Viperblock built successfully" -else - echo "โš ๏ธ Viperblock directory not found, skipping build" -fi - -# Build Predastore -if [[ -d "$MULGA_ROOT/predastore" ]]; then - echo "๐Ÿ“ฆ Building Predastore..." - cd "$MULGA_ROOT/predastore" - make build - echo "โœ… Predastore built successfully" -else - echo "โš ๏ธ Predastore directory not found, skipping build" -fi - -echo "" -echo "Development environment setup complete!" -echo "" -echo "Recommended: Use scripts/dev-install.sh for a production-parity setup" -echo " (systemd, privilege separation, production paths)." -echo "" -echo " scripts/dev-install.sh # One-time: build, install, init, start via systemd" -echo " make deploy # Daily: rebuild + copy binary + restart all" -echo " make reinstall # Changed a .service file? Re-run full setup" -echo "" -echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):" -echo " ./scripts/setup-ovn.sh --management" -echo " sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1" -echo " ./scripts/start-dev.sh" diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh deleted file mode 100755 index c12a0889a..000000000 --- a/scripts/start-dev.sh +++ /dev/null @@ -1,514 +0,0 @@ -#!/bin/bash - -# Start Spinifex development environment -# This script starts all required services in the correct order using Spinifex service commands -# Usage: ./scripts/start-dev.sh [--build] [data-dir] -# --build: Rebuild all binaries before starting (default: skip build) -# data-dir: Optional data directory path (default: ~/spinifex) -# -# Environment variables: -# UI=false Skip starting Spinifex UI (e.g., UI=false ./scripts/start-dev.sh) - -set -euo pipefail - -echo "WARNING: start-dev.sh is deprecated for local development." >&2 -echo " Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2 -echo " start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2 -echo "" >&2 - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" - -# Parse arguments -BUILD=false -DATA_DIR="$HOME/spinifex" -for arg in "$@"; do - case "$arg" in - --build) BUILD=true ;; - *) DATA_DIR="$arg" ;; - esac -done - -# Configuration paths -# Use CONFIG_DIR environment variable if set, otherwise derive from DATA_DIR -CONFIG_DIR="${CONFIG_DIR:-$DATA_DIR/config}" -echo "Using data directory: $DATA_DIR" -echo "Using configuration directory: $CONFIG_DIR" -LOGS_DIR="$DATA_DIR/logs" -WAL_DIR="$DATA_DIR/spinifex" - -echo "โš ๏ธ Dev mode: all services run as $(whoami) without privilege separation" -echo "" -echo "๐Ÿš€ Starting Spinifex development environment..." -echo "Project root: $PROJECT_ROOT" -echo "Data directory: $DATA_DIR" - -# Parse services from spinifex.toml โ€” defaults to all if not set -parse_services() { - local config_file="$CONFIG_DIR/spinifex.toml" - if [ -f "$config_file" ]; then - local svc_line=$(grep -m1 '^services' "$config_file" | sed 's/.*\[//;s/\].*//;s/"//g;s/,/ /g') - if [ -n "$svc_line" ]; then - echo "$svc_line" - return - fi - fi - echo "nats predastore viperblock daemon awsgw vpcd ui" -} - -SERVICES=$(parse_services) -has_service() { - local svc="$1" - echo "$SERVICES" | grep -qw "$svc" -} -echo "Services: $SERVICES" - -# Detect multi-node cluster from config -is_multinode() { - local config_file="$CONFIG_DIR/spinifex.toml" - if [ -f "$config_file" ]; then - local node_count=$(grep -c '^\[nodes\.' "$config_file") - [ "$node_count" -gt 1 ] - else - return 1 - fi -} - -# Confirm configuration directory exists -if [ ! -d "$CONFIG_DIR" ]; then - echo "โš ๏ธ Configuration directory $CONFIG_DIR does not exist." - echo "Please init the spinifex environment using the CLI." - echo "spx admin init" - exit 1 -fi - - -if [ -d "/mnt/ramdisk" ]; then - - # Check if /mnt/ramdisk is mounted, if not mount it as tmpfs - if ! mountpoint -q /mnt/ramdisk; then - echo "๐Ÿ’พ Mounting /mnt/ramdisk as tmpfs" - sudo mount -t tmpfs -o size=8G tmpfs /mnt/ramdisk/ - fi - - # If /mnt/ramdisk is mounted, use it for the WAL directory (for development) - if mountpoint -q "/mnt/ramdisk"; then - WAL_DIR="/mnt/ramdisk/" - fi - -else - echo "โš ๏ธ /mnt/ramdisk not available, using $DATA_DIR/viperblock" -fi - -# Change to project root for all commands -cd "$PROJECT_ROOT" - -# Function to start service in background -start_service() { - local name="$1" - local command="$2" - local pidfile="$LOGS_DIR/$name.pid" - local logfile="$LOGS_DIR/$name.log" - - echo "๐Ÿ“ก Starting $name..." - echo " Command: $command" - - # Start service and capture PID - nohup $command > "$logfile" 2>&1 & - local pid=$! - echo $pid > "$pidfile" - - echo " PID: $pid, Log: $logfile" - - # Brief pause to let service start, then verify it's still alive - sleep 1 - if ! kill -0 "$pid" 2>/dev/null; then - echo " โŒ $name failed to start (exited immediately)" - echo " Check log: $logfile" - tail -5 "$logfile" 2>/dev/null || true - exit 1 - fi -} - -# Function to start service in foreground (for final daemon) -start_service_foreground() { - local name="$1" - local command="$2" - - echo "๐Ÿ“ก Starting $name in foreground..." - echo " Command: $command" - - # Setup signal handler to stop background services when daemon stops - trap 'echo ""; echo "๐Ÿ›‘ Stopping all services..."; ./scripts/stop-dev.sh; exit 0' INT TERM - - # Start the service in foreground - $command -} - -# Function to set OOM score for a service (Linux only) -# Protects infrastructure services from OOM killer (-500 = less likely to be killed) -set_oom_score() { - local name="$1" - local score="$2" - local pid_file="$LOGS_DIR/${name}.pid" - if [ -f "$pid_file" ]; then - local pid=$(cat "$pid_file") - if [ -d "/proc/$pid" ]; then - echo "$score" > "/proc/$pid/oom_score_adj" 2>/dev/null && \ - echo " OOM score for $name (PID $pid): $score" || \ - echo " Warning: Could not set OOM score for $name" - fi - fi -} - -# Function to check if service is responsive -check_service() { - local name="$1" - local host="$2" - local port="$3" - local max_attempts="${4:-30}" - local attempt=1 - - echo "๐Ÿ” Checking $name connectivity on $host:$port..." - - while [ $attempt -le $max_attempts ]; do - if nc -z "$host" "$port" 2>/dev/null; then - echo " โœ… $name is responding on $host:$port" - return 0 - fi - echo " โณ Attempt $attempt/$max_attempts - waiting for $name..." - sleep 2 - ((attempt++)) - done - - echo " โŒ $name failed to start on $host:$port after $max_attempts attempts" - exit 1 -} - -# Pre-flight, compile latest (only with --build flag) -if [ "$BUILD" = "true" ]; then - echo "โœˆ๏ธ Pre-flight, compiling latest..." - - echo " Building spinifex..." - make build - - echo " Building predastore..." - cd "$MULGA_ROOT/predastore" && make build - cd "$PROJECT_ROOT" - - echo " Building viperblock (nbdkit plugin)..." - cd "$MULGA_ROOT/viperblock" && make build - cd "$PROJECT_ROOT" - - echo " โœ… Build complete" -else - echo "โœˆ๏ธ Skipping build (pass --build to rebuild)" -fi - -# 0๏ธโƒฃ Start OVN networking (required for VPC) -echo "" -echo "0๏ธโƒฃ Starting OVN networking..." - -if ! pidof systemd >/dev/null 2>&1; then - echo " โš ๏ธ No systemd (container?) โ€” skipping OVN lifecycle management" -elif ! command -v ovs-vsctl >/dev/null 2>&1; then - echo " โŒ OVS not installed" - echo " Run: make quickinstall && ./scripts/setup-ovn.sh --management" - exit 1 -else - # Start OVS + OVN services (may already be running) - sudo systemctl start openvswitch-switch - echo " โœ… openvswitch-switch started" - - # Start ovn-central if this node has it installed and enabled - if systemctl list-unit-files ovn-central.service >/dev/null 2>&1 && systemctl is-enabled ovn-central >/dev/null 2>&1; then - sudo systemctl start ovn-central - echo " โœ… ovn-central started" - fi - - sudo systemctl start ovn-controller - echo " โœ… ovn-controller started" - - # Verify br-int exists (created by setup-ovn.sh) - if ! sudo ovs-vsctl br-exists br-int 2>/dev/null; then - echo " โŒ br-int not found โ€” run ./scripts/setup-ovn.sh --management first" - exit 1 - fi - echo " โœ… br-int exists" -fi - -# 1๏ธโƒฃ Start NATS server -echo "" -if has_service "nats"; then - echo "1๏ธโƒฃ Starting NATS server..." - - if [ -f "$CONFIG_DIR/nats/nats.conf" ]; then - export SPINIFEX_CONFIG_PATH=$CONFIG_DIR/nats/nats.conf - echo " Using NATS config file: $CONFIG_DIR/nats/nats.conf" - else - echo " โš ๏ธ NATS config file not found at $CONFIG_DIR/nats/nats.conf, using defaults" - export SPINIFEX_NATS_HOST=0.0.0.0 - export SPINIFEX_NATS_PORT=4222 - export SPINIFEX_NATS_DATA_DIR=$DATA_DIR/nats/ - export SPINIFEX_NATS_JETSTREAM=false - fi - - # Extract NATS listen address from config (multi-node binds to specific IP, not 127.0.0.1) - NATS_CHECK_HOST="127.0.0.1" - if [ -f "$CONFIG_DIR/nats/nats.conf" ]; then - NATS_LISTEN_IP=$(grep -oP '^listen:\s*\K[^:]+' "$CONFIG_DIR/nats/nats.conf" 2>/dev/null || true) - if [ -n "$NATS_LISTEN_IP" ] && [ "$NATS_LISTEN_IP" != "0.0.0.0" ]; then - NATS_CHECK_HOST="$NATS_LISTEN_IP" - fi - fi - - NATS_CMD="./bin/spx service nats start" - start_service "nats" "$NATS_CMD" - set_oom_score "nats" "-500" - check_service "NATS" "$NATS_CHECK_HOST" "4222" - - # Wait for NATS JetStream to be ready before Predastore starts. - # Predastore lazily opens IAM KV buckets โ€” it starts with config-only auth - # and activates IAM auth once the spinifex daemon creates the KV buckets. - # Wait for NATS JetStream to be ready before Predastore starts. - # Predastore lazily opens IAM KV buckets โ€” it starts with config-only auth - # and activates IAM auth once the spinifex daemon creates the KV buckets. - # Use the monitoring HTTP endpoint (plaintext nc probe doesn't work with TLS). - # Monitoring is only on the primary node; secondary nodes skip this check. - NATS_HAS_MONITORING=false - if [ -f "$CONFIG_DIR/nats/nats.conf" ] && grep -q '^http:' "$CONFIG_DIR/nats/nats.conf"; then - NATS_HAS_MONITORING=true - fi - - if [ "$NATS_HAS_MONITORING" = "true" ]; then - echo "๐Ÿ” Waiting for NATS JetStream..." - NATS_JS_READY=false - for i in $(seq 1 15); do - if curl -sf "http://127.0.0.1:8222/jsz" > /dev/null 2>&1; then - echo " โœ… NATS JetStream is ready" - NATS_JS_READY=true - break - fi - sleep 1 - done - if [ "$NATS_JS_READY" = "false" ]; then - echo " โš ๏ธ NATS JetStream not confirmed ready โ€” Predastore may fall back to config-only auth" - fi - else - echo "๐Ÿ” NATS monitoring not configured โ€” skipping JetStream readiness check" - sleep 2 - fi -else - echo "1๏ธโƒฃ Skipping NATS (not a local service)" -fi - -# 2๏ธโƒฃ Start Predastore -echo "" -if has_service "predastore"; then - echo "2๏ธโƒฃ Starting Predastore..." - unset SPINIFEX_CONFIG_PATH - - export SPINIFEX_PREDASTORE_BASE_PATH=$DATA_DIR/predastore/ - export SPINIFEX_PREDASTORE_CONFIG_PATH=$CONFIG_DIR/predastore/predastore.toml - export SPINIFEX_PREDASTORE_TLS_CERT=$CONFIG_DIR/server.pem - export SPINIFEX_PREDASTORE_TLS_KEY=$CONFIG_DIR/server.key - - # Per-node predastore encryption key. Self-heal for devs who skipped - # `spx admin init`: generate with umask 0177 so the file is 0600 from - # creation (predastore rejects group/other-readable keys outright). - ENCRYPTION_KEY="$CONFIG_DIR/predastore/encryption.key" - if [ ! -f "$ENCRYPTION_KEY" ]; then - mkdir -p "$CONFIG_DIR/predastore" - ( umask 0177 && openssl rand -out "$ENCRYPTION_KEY" 32 ) - echo " Generated predastore encryption key: $ENCRYPTION_KEY" - fi - export SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE="$ENCRYPTION_KEY" - - # Auto-detect Predastore host:port from spinifex.toml [nodes..predastore] section - PREDASTORE_BIND="0.0.0.0:8443" - if [ -f "$CONFIG_DIR/spinifex.toml" ]; then - DETECTED_PREDASTORE_HOST=$(awk -F'"' '/\[nodes\..*\.predastore\]/{found=1} found && /^host/{print $2; exit}' "$CONFIG_DIR/spinifex.toml") - if [ -n "$DETECTED_PREDASTORE_HOST" ]; then - PREDASTORE_BIND="$DETECTED_PREDASTORE_HOST" - echo " Auto-detected Predastore bind=$PREDASTORE_BIND from spinifex.toml" - fi - fi - export SPINIFEX_PREDASTORE_HOST="${PREDASTORE_BIND%%:*}" - export SPINIFEX_PREDASTORE_PORT="${PREDASTORE_BIND##*:}" - - export SPINIFEX_PREDASTORE_BACKEND=distributed - - # Auto-detect Predastore NODE_ID from spinifex.toml if not already set - if [ -z "${SPINIFEX_PREDASTORE_NODE_ID:-}" ]; then - if [ -f "$CONFIG_DIR/spinifex.toml" ]; then - DETECTED_NODE_ID=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2); print $2; exit}' "$CONFIG_DIR/spinifex.toml") - if [ -n "$DETECTED_NODE_ID" ] && [ "$DETECTED_NODE_ID" != "0" ]; then - export SPINIFEX_PREDASTORE_NODE_ID="$DETECTED_NODE_ID" - echo " Auto-detected Predastore NODE_ID=$DETECTED_NODE_ID from spinifex.toml" - fi - fi - fi - export SPINIFEX_PREDASTORE_NODE_ID="${SPINIFEX_PREDASTORE_NODE_ID:-}" - - PREDASTORE_CMD="./bin/spx service predastore start" - start_service "predastore" "$PREDASTORE_CMD" - set_oom_score "predastore" "-500" - if is_multinode; then - echo " โญ๏ธ Skipping Predastore connectivity check (multi-node: needs quorum from peer nodes)" - else - check_service "Predastore" "$SPINIFEX_PREDASTORE_HOST" "$SPINIFEX_PREDASTORE_PORT" - fi -else - echo "2๏ธโƒฃ Skipping Predastore (not a local service)" -fi - - -# 3๏ธโƒฃ Start Viperblock -echo "" -if has_service "viperblock"; then - echo "3๏ธโƒฃ Starting Viperblock..." - - # Determine base directory for Viperblock data, dev uses /mnt/ramdisk if available - if [ -d "/mnt/ramdisk" ] && [ -w "/mnt/ramdisk" ]; then - VB_BASE_DIR="/mnt/ramdisk/" - else - VB_BASE_DIR="$DATA_DIR/viperblock/" - fi - - # Check if NBD plugin exists - NBD_PLUGIN_PATH="$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" - - if [ ! -f "$NBD_PLUGIN_PATH" ]; then - echo " โš ๏ธ NBD plugin not found at $NBD_PLUGIN_PATH" - echo " Building Viperblock NBD plugin..." - cd "$MULGA_ROOT/viperblock" && make build - cd "$PROJECT_ROOT" - fi - - export SPINIFEX_CONFIG_PATH=$CONFIG_DIR/spinifex.toml - export SPINIFEX_VIPERBLOCK_PLUGIN_PATH=$NBD_PLUGIN_PATH - export SPINIFEX_BASE_DIR=$VB_BASE_DIR - - VIPERBLOCK_CMD="./bin/spx service viperblock start" - start_service "viperblock" "$VIPERBLOCK_CMD" - set_oom_score "viperblock" "-500" -else - echo "3๏ธโƒฃ Skipping Viperblock (not a local service)" -fi - -# 4๏ธโƒฃ Start Spinifex Gateway/Daemon -echo "" -if has_service "daemon"; then - echo "4๏ธโƒฃ. Starting Spinifex Gateway..." - - export SPINIFEX_CONFIG_PATH=$CONFIG_DIR/spinifex.toml - export SPINIFEX_BASE_DIR=$DATA_DIR/spinifex/ - export SPINIFEX_WAL_DIR=$WAL_DIR - - SPINIFEX_CMD="./bin/spx service spinifex start" - start_service "spinifex" "$SPINIFEX_CMD" - set_oom_score "spinifex" "-500" -else - echo "4๏ธโƒฃ. Skipping Spinifex Gateway (not a local service)" -fi - - -# 5๏ธโƒฃ Start AWS Gateway -echo "" -if has_service "awsgw"; then - echo "5๏ธโƒฃ. Starting AWS Gateway..." - - unset SPINIFEX_NATS_HOST - unset SPINIFEX_PREDASTORE_HOST - export SPINIFEX_BASE_DIR=$DATA_DIR - export SPINIFEX_CONFIG_PATH=$CONFIG_DIR/spinifex.toml - export SPINIFEX_AWSGW_TLS_CERT=$CONFIG_DIR/server.pem - export SPINIFEX_AWSGW_TLS_KEY=$CONFIG_DIR/server.key - - AWSGW_CMD="./bin/spx service awsgw start" - start_service "awsgw" "$AWSGW_CMD" - set_oom_score "awsgw" "-500" -else - echo "5๏ธโƒฃ. Skipping AWS Gateway (not a local service)" -fi - - -# 6๏ธโƒฃ Start vpcd (VPC daemon) -echo "" -if has_service "vpcd"; then - echo "6๏ธโƒฃ. Starting vpcd (VPC daemon)..." - - export SPINIFEX_CONFIG_PATH=$CONFIG_DIR/spinifex.toml - - VPCD_CMD="./bin/spx service vpcd start" - start_service "vpcd" "$VPCD_CMD" - set_oom_score "vpcd" "-500" -else - echo "6๏ธโƒฃ. Skipping vpcd (not a local service)" -fi - -# 7๏ธโƒฃ Start Spinifex UI (skip with UI=false or if not a local service) -if [ "${UI:-}" != "false" ] && has_service "ui"; then - echo "" - echo "7๏ธโƒฃ. Starting Spinifex UI..." - - SPXUI_CMD="./bin/spx service spinifex-ui start" - start_service "spinifex-ui" "$SPXUI_CMD" - set_oom_score "spinifex-ui" "-500" -else - echo "" - echo "7๏ธโƒฃ. Skipping Spinifex UI" -fi - - -echo "" -echo "๐Ÿ”— Service endpoints will be:" -if [ "${UI:-}" != "false" ]; then - echo " - Spinifex UI: https://localhost:3000" -fi -echo " - NATS: nats://localhost:4222" -echo " - Predastore: https://localhost:8443" -echo " - AWS Gateway: https://localhost:9999" -echo "" -echo "๐Ÿ“Š Monitor background service logs:" -echo " tail -f $LOGS_DIR/*.log" -echo "" -echo "๐Ÿงช Test with AWS CLI (once daemon is running):" -echo " export AWS_PROFILE=spinifex" -echo " aws ec2 describe-instances" -echo "" - -# For multi-node clusters, check peer daemon health (best-effort) -if is_multinode; then - echo "Checking cluster peer health..." - # Extract peer daemon hosts from config (host = "ip:port" under [nodes.X.daemon]) - peer_hosts=$(awk ' - /^\[nodes\./ { node=1; daemon=0 } - node && /^\[.*\.daemon\]/ { daemon=1 } - daemon && /^host/ { gsub(/[" ]/, "", $3); print $3; daemon=0 } - ' "$CONFIG_DIR/spinifex.toml") - - for peer in $peer_hosts; do - attempts=0 - max_attempts=5 - while [ $attempts -lt $max_attempts ]; do - if curl -sk --connect-timeout 3 "https://$peer/health" > /dev/null 2>&1; then - echo " $peer: healthy" - break - fi - attempts=$((attempts + 1)) - if [ $attempts -lt $max_attempts ]; then - sleep 2 - fi - done - if [ $attempts -ge $max_attempts ]; then - echo " $peer: not responding (may still be starting)" - fi - done - echo "" -fi - -# This will only be reached if air/daemon exits normally -#echo "" -#echo "๐Ÿ›‘ Spinifex development environment stopped" diff --git a/scripts/stop-dev.sh b/scripts/stop-dev.sh deleted file mode 100755 index 21d654bca..000000000 --- a/scripts/stop-dev.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash - -# Stop Spinifex development environment -# This script stops all services started by start-dev.sh -# Usage: ./scripts/stop-dev.sh -# Note: Services are stopped using PID files, so data-dir is not required - -set -uo pipefail - -# Accept optional data directory argument -DATA_DIR="${1:-$HOME/spinifex}" - -# Always use the logs directory for PID files โ€” start-dev.sh writes shell PIDs -# there, and Go services write PIDs to their own data dirs. This ensures -# consistent stop behavior for both single-node and multi-node setups. -PID_DIR="$DATA_DIR/logs" - - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -LOGS_DIR="$PROJECT_ROOT/data/logs" - -# Check if /mnt/ramdisk is mounted -#if mountpoint -q /mnt/ramdisk; then - #echo "๐Ÿ›‘ Stopping Spinifex development environment..." - #sudo umount /mnt/ramdisk -#fi - -echo "Stopping Spinifex development environment..." - -# Parse services from spinifex.toml โ€” defaults to all if not set -CONFIG_DIR="${CONFIG_DIR:-$DATA_DIR/config}" -parse_services() { - local config_file="$CONFIG_DIR/spinifex.toml" - if [ -f "$config_file" ]; then - local svc_line=$(grep -m1 '^services' "$config_file" | sed 's/.*\[//;s/\].*//;s/"//g;s/,/ /g') - if [ -n "$svc_line" ]; then - echo "$svc_line" - return - fi - fi - echo "nats predastore viperblock daemon awsgw vpcd ui" -} - -SERVICES=$(parse_services) -has_service() { - local svc="$1" - case "$svc" in - spinifex) svc="daemon" ;; - spinifex-ui) svc="ui" ;; - esac - echo "$SERVICES" | grep -qw "$svc" -} - -# Detect multi-node cluster from config -is_multinode() { - local config_file="$CONFIG_DIR/spinifex.toml" - if [ -f "$config_file" ]; then - # Count only top-level node sections [nodes.X], not subsections [nodes.X.Y] - local node_count=$(grep -cE '^\[nodes\.[^.]+\]' "$config_file") - [ "$node_count" -gt 1 ] - else - return 1 - fi -} - -# For multi-node clusters, delegate to coordinated shutdown via NATS. -# Only when called without arguments (default path). When a data-dir is -# explicitly provided (e.g., E2E per-node cleanup), use per-service stop. -if is_multinode && [ -z "${SPINIFEX_FORCE_LOCAL_STOP:-}" ] && [ -z "${1:-}" ]; then - echo "Multi-node cluster detected. Using coordinated shutdown..." - echo " (Set SPINIFEX_FORCE_LOCAL_STOP=1 to force per-service stop)" - exec $PROJECT_ROOT/bin/spx admin cluster shutdown -fi - -# Function to stop a service. Sends SIGTERM and waits for the process to exit -# so resources (ports, DB locks) are fully released before returning. -stop_service() { - local name="$1" - local pidpath="$2" - - echo "๐Ÿ›‘ Stopping $name..." - - local pid - pid=$(cat "$pidpath/$name.pid" 2>/dev/null) || { echo "โš ๏ธ No PID file for $name"; return 1; } - - kill -SIGTERM "$pid" 2>/dev/null || { rm -f "$pidpath/$name.pid"; echo "โœ… $name already stopped"; echo ""; return 0; } - - # Wait for process to fully exit (up to 15s) so ports/locks are released - local checks=0 - while kill -0 "$pid" 2>/dev/null; do - sleep 1 - checks=$((checks + 1)) - if [ "$checks" -ge 15 ]; then - echo " Force killing $name..." - kill -9 "$pid" 2>/dev/null || true - sleep 1 - break - fi - done - - rm -f "$pidpath/$name.pid" - echo "โœ… $name stopped" - echo "" -} - -# Stop services in reverse order -echo "Stopping services..." -echo "" -# Stop Spinifex UI first (last to start) -has_service "spinifex-ui" && stop_service "spinifex-ui" "$PID_DIR" - -# Stop Spinifex daemon/gateway (it will terminate running instances, unmount nbd devices) -has_service "spinifex" && stop_service "spinifex" "$PID_DIR" - -# Wait for all QEMU instances to exit before stopping infrastructure services. -# The daemon sends system_powerdown to VMs during shutdown, but may be killed -# before VMs fully exit. We must wait here so viperblock/predastore aren't -# pulled out from under running VMs. -if pgrep -x qemu-system-x86_64 > /dev/null 2>&1; then - echo "โณ Waiting for QEMU instances to exit..." - timeout=120 - elapsed=0 - while pgrep -x qemu-system-x86_64 > /dev/null 2>&1; do - if [ $elapsed -ge $timeout ]; then - echo "โš ๏ธ Timeout waiting for QEMU processes, force killing..." - pkill -9 -x qemu-system-x86_64 2>/dev/null || true - sleep 1 - break - fi - sleep 1 - ((elapsed++)) - done - echo "โœ… All QEMU instances exited" -fi - -# Stop vpcd (VPC daemon) -has_service "vpcd" && stop_service "vpcd" "$PID_DIR" - -# Stop AWSGW -has_service "awsgw" && stop_service "awsgw" "$PID_DIR" - -# Stop Viperblock -has_service "viperblock" && stop_service "viperblock" "$PID_DIR" - -# Stop Predastore -has_service "predastore" && stop_service "predastore" "$PID_DIR" - -# Stop NATS -has_service "nats" && stop_service "nats" "$PID_DIR" - -# Stop OVN networking (prevents idle CPU burn when spinifex isn't running) -# CRITICAL: Do NOT stop openvswitch-switch if OVS carries WAN traffic. -# This can happen when an OVS bridge has a WAN IP (direct bridge mode) or -# when a veth pair links an OVS bridge to a Linux WAN bridge (veth mode). -# Stopping OVS would destroy the bridge and break WAN connectivity. -# OVS is infrastructure โ€” only stop ovn-controller and ovn-central. -if pidof systemd >/dev/null 2>&1; then - echo "๐Ÿ›‘ Stopping OVN networking..." - sudo systemctl stop ovn-controller 2>/dev/null && echo "โœ… ovn-controller stopped" || true - if systemctl is-active --quiet ovn-central 2>/dev/null; then - sudo systemctl stop ovn-central 2>/dev/null && echo "โœ… ovn-central stopped" || true - fi - - # Only stop OVS if tearing it down won't kill WAN connectivity. - # Two cases where OVS is load-bearing for WAN: - # 1. An OVS bridge has a WAN IP directly (direct bridge mode) - # 2. A veth pair links a Linux bridge to an OVS bridge (veth mode) โ€” - # stopping OVS destroys the OVS bridge end, breaking the link - WAN_BRIDGE_HAS_IP=false - VETH_LINKS_WAN=false - for br in $(sudo ovs-vsctl list-br 2>/dev/null); do - if [ "$br" = "br-int" ]; then continue; fi - if ip -4 addr show "$br" 2>/dev/null | grep -q "inet "; then - WAN_BRIDGE_HAS_IP=true - break - fi - done - # Check if a veth pair links an OVS bridge to a Linux bridge with a WAN IP - if ip link show veth-wan-br >/dev/null 2>&1; then - VETH_LINKS_WAN=true - fi - if [ "$WAN_BRIDGE_HAS_IP" = true ]; then - echo "โš ๏ธ Skipping openvswitch-switch stop ($br has WAN IP โ€” stopping would kill connectivity)" - elif [ "$VETH_LINKS_WAN" = true ]; then - echo "โš ๏ธ Skipping openvswitch-switch stop (veth pair links OVS to WAN bridge)" - else - sudo systemctl stop openvswitch-switch 2>/dev/null && echo "โœ… openvswitch-switch stopped" || true - fi -fi - -echo "" -echo "โœ… Spinifex development environment stopped" - -# Show any remaining spinifex-related processes -remaining=$(pgrep -af '(bin/spx|spinifex-ui|nats-server|predastore|viperblock|vpcd|qemu-system)' | grep -v "stop-dev.sh" || true) -if [[ -n "$remaining" ]]; then - echo "" - echo "โš ๏ธ Some related processes may still be running:" - echo "$remaining" -fi \ No newline at end of file