diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4d0614639..c4937d5e7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -180,9 +180,13 @@ jobs: sparse-checkout: scripts/tofu-cluster path: mulga - - name: Clone and build + - name: Build Install Artifacts working-directory: mulga/scripts/tofu-cluster - run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar" + run: | + ./build-install-artifacts.sh "${GIT_BRANCH}" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-tests.tar" - name: Provision VMs working-directory: mulga/scripts/tofu-cluster @@ -194,13 +198,14 @@ jobs: ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 - - name: Bootstrap Cluster + - name: Bootstrap via Binary Install working-directory: mulga/scripts/tofu-cluster run: | - SPX_PREBUILT=true \ - SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \ SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \ - ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1 + ./bootstrap-install.sh env2 \ + "${RUNNER_TEMP}/setup.sh" \ + "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ + --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster @@ -218,7 +223,7 @@ jobs: done ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \ - "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}" + "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS}" - name: Teardown if: always() diff --git a/Makefile b/Makefile index 425da1917..66b16f37c 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,17 @@ run: $(MAKE) go_build $(MAKE) go_run +# Fast iteration: build + install binary + restart all services +deploy: build + sudo install -m 755 bin/spx /usr/local/bin/spx + sudo systemctl daemon-reload + sudo systemctl restart spinifex.target + +# Re-run setup.sh after changing systemd units, helper scripts, or logrotate config. +# Not needed for code-only changes — use deploy for those. +reinstall: + scripts/dev-install.sh + clean: rm -f ./bin/$(GO_PROJECT_NAME) rm -rf spinifex/services/spinifexui/frontend/dist @@ -207,7 +218,8 @@ distro-arm64: distro-clean: rm -rf dist/ -.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run clean \ +.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \ + deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ distro distro-amd64 distro-arm64 distro-clean diff --git a/build/systemd/spinifex-awsgw.service b/build/systemd/spinifex-awsgw.service index 1626cebdd..42f6df37b 100644 --- a/build/systemd/spinifex-awsgw.service +++ b/build/systemd/spinifex-awsgw.service @@ -6,7 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple -User=spinifex +User=spinifex-gw Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service awsgw start @@ -18,5 +18,27 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/ Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key +# Filesystem access +NoNewPrivileges=yes +ProtectSystem=strict +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key +ReadWritePaths=/var/lib/spinifex /var/log/spinifex /etc/spinifex/bootstrap.json /tmp + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 0722bd50d..1bc5b9f9f 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -6,7 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple -User=spinifex +User=spinifex-daemon Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service spinifex start @@ -17,5 +17,29 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/ Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/ +# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd) +ProtectSystem=full +PrivateDevices=no +DevicePolicy=closed +DeviceAllow=/dev/kvm rw +DeviceAllow=/dev/net/tun rw +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key +ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-nats.service b/build/systemd/spinifex-nats.service index 42797b81d..eaf0e89b7 100644 --- a/build/systemd/spinifex-nats.service +++ b/build/systemd/spinifex-nats.service @@ -5,7 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic [Service] Type=simple -User=spinifex +User=spinifex-nats Group=spinifex ExecStart=/usr/local/bin/spx service nats start Restart=on-failure @@ -13,5 +13,27 @@ RestartSec=5 OOMScoreAdjust=-500 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf +# Filesystem access +NoNewPrivileges=yes +ProtectSystem=strict +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/nats +ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex /tmp + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-predastore.service b/build/systemd/spinifex-predastore.service index 712fab8cf..40407a227 100644 --- a/build/systemd/spinifex-predastore.service +++ b/build/systemd/spinifex-predastore.service @@ -6,7 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple -User=spinifex +User=spinifex-storage Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/var/lib/spinifex/predastore-start.sh @@ -18,5 +18,27 @@ Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/ Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key +# Filesystem access +NoNewPrivileges=yes +ProtectSystem=strict +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key +ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex /tmp + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-ui.service b/build/systemd/spinifex-ui.service index 5354fc1c0..e2a6f762f 100644 --- a/build/systemd/spinifex-ui.service +++ b/build/systemd/spinifex-ui.service @@ -5,7 +5,7 @@ After=spinifex-awsgw.service [Service] Type=simple -User=spinifex +User=spinifex-ui Group=spinifex ExecStart=/usr/local/bin/spx service spinifex-ui start Restart=on-failure @@ -14,5 +14,27 @@ OOMScoreAdjust=-500 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key +# Filesystem access +NoNewPrivileges=yes +ProtectSystem=strict +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key +ReadWritePaths=/var/log/spinifex /tmp + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-viperblock.service b/build/systemd/spinifex-viperblock.service index 9592ddda2..1e17ac3c6 100644 --- a/build/systemd/spinifex-viperblock.service +++ b/build/systemd/spinifex-viperblock.service @@ -6,7 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple -User=spinifex +User=spinifex-viperblock Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service viperblock start @@ -17,5 +17,27 @@ EnvironmentFile=/etc/spinifex/systemd.env Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/ +# Filesystem access +NoNewPrivileges=yes +ProtectSystem=strict +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env +ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /tmp + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native + [Install] WantedBy=spinifex.target diff --git a/build/systemd/spinifex-vpcd.service b/build/systemd/spinifex-vpcd.service index df643cf77..3fea74e97 100644 --- a/build/systemd/spinifex-vpcd.service +++ b/build/systemd/spinifex-vpcd.service @@ -6,7 +6,7 @@ Wants=spinifex-nats.service [Service] Type=simple -User=spinifex +User=spinifex-vpcd Group=spinifex ExecStartPre=/var/lib/spinifex/wait-for-nats.sh ExecStart=/usr/local/bin/spx service vpcd start @@ -14,6 +14,28 @@ Restart=on-failure RestartSec=5 OOMScoreAdjust=-500 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml +Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/ + +# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl) +ProtectSystem=full +PrivateDevices=yes +ReadOnlyPaths=/etc/spinifex/spinifex.toml +ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex + +# Hardening +ProtectHome=yes +PrivateTmp=no +RestrictSUIDSGID=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +ProtectClock=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectProc=invisible +SystemCallArchitectures=native [Install] WantedBy=spinifex.target diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 3491aa349..a18ba253e 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -933,6 +933,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { Region: region, NatsToken: natsToken, DataDir: spxRoot, + LogDir: LogDirFor(spxRoot), ConfigDir: configDir, Node: node, @@ -1014,14 +1015,10 @@ func runAdminInit(cmd *cobra.Command, args []string) { admin.CreateServiceDirectories(spxRoot) - // In production layout (running as root), fix ownership so the service user - // can read config and write data. SUDO_USER identifies the operator account. + // In production layout (running as root), set per-service ownership on + // directories and shared config files (root:spinifex with correct modes). if os.Getuid() == 0 { - sudoUser := os.Getenv("SUDO_USER") - if sudoUser != "" { - admin.ChownRecursive(configDir, sudoUser) - admin.ChownRecursive(spxRoot, sudoUser) - } + admin.SetServiceOwnership() } // Print success message @@ -1171,6 +1168,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, Region: region, NatsToken: natsToken, DataDir: spxRoot, + LogDir: LogDirFor(spxRoot), ConfigDir: configDir, Node: node, @@ -1246,14 +1244,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, admin.CreateServiceDirectories(spxRoot) - // In production layout (running as root), fix ownership so the service user - // can read config and write data. SUDO_USER identifies the operator account. + // In production layout (running as root), set per-service ownership on + // directories and shared config files (root:spinifex with correct modes). if os.Getuid() == 0 { - sudoUser := os.Getenv("SUDO_USER") - if sudoUser != "" { - admin.ChownRecursive(configDir, sudoUser) - admin.ChownRecursive(spxRoot, sudoUser) - } + admin.SetServiceOwnership() } // Keep formation server running briefly so joining nodes can fetch complete status @@ -1569,6 +1563,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) { Region: creds.Region, NatsToken: creds.NatsToken, DataDir: dataDir, + LogDir: LogDirFor(dataDir), ConfigDir: configDir, Node: node, @@ -1648,14 +1643,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) { admin.CreateServiceDirectories(dataDir) - // In production layout (running as root), fix ownership so the service user - // can read config and write data. SUDO_USER identifies the operator account. + // In production layout (running as root), set per-service ownership on + // directories and shared config files (root:spinifex with correct modes). if os.Getuid() == 0 { - sudoUser := os.Getenv("SUDO_USER") - if sudoUser != "" { - admin.ChownRecursive(configDir, sudoUser) - admin.ChownRecursive(dataDir, sudoUser) - } + admin.SetServiceOwnership() } // Print cluster summary diff --git a/cmd/spinifex/cmd/paths.go b/cmd/spinifex/cmd/paths.go index 54d5e0b86..51c201613 100644 --- a/cmd/spinifex/cmd/paths.go +++ b/cmd/spinifex/cmd/paths.go @@ -26,6 +26,16 @@ func DefaultDataDir() string { return filepath.Join(realUserHomeDir(), "spinifex") } +// LogDirFor returns the log directory for a given data directory. +// Production: /var/log/spinifex (matches systemd ReadWritePaths) +// Development: /logs (supports custom per-node data dirs) +func LogDirFor(dataDir string) string { + if isProductionLayout() { + return "/var/log/spinifex" + } + return filepath.Join(dataDir, "logs") +} + // realUserHomeDir returns the home directory of the real (non-sudo) user. // When running under sudo, SUDO_USER is set to the invoking user — resolve // their home directory so config/data land in the right place. diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 14fa282df..26d8a5d34 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -733,6 +733,12 @@ var vpcdStartCmd = &cobra.Command{ } } + baseDir := viper.GetString("base-dir") + if baseDir != "" { + fmt.Println("Overwriting vpcd base-dir to:", baseDir) + nodeConfig.BaseDir = baseDir + } + svc, err := service.New("vpcd", &vpcd.Config{ NatsHost: nodeConfig.NATS.Host, NatsToken: nodeConfig.NATS.ACL.Token, diff --git a/cmd/spinifex/cmd/templates/nats.conf b/cmd/spinifex/cmd/templates/nats.conf index be94dfeaa..82216c6f8 100644 --- a/cmd/spinifex/cmd/templates/nats.conf +++ b/cmd/spinifex/cmd/templates/nats.conf @@ -40,7 +40,7 @@ jetstream { debug: false trace: false logtime: true -log_file: "{{.DataDir}}/logs/nats.log" +log_file: "{{.LogDir}}/nats.log" # Limits max_connections: 1000 diff --git a/scripts/dev-install.sh b/scripts/dev-install.sh new file mode 100755 index 000000000..f8d3dc746 --- /dev/null +++ b/scripts/dev-install.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# dev-install.sh — Full local development setup via production installer. +# Builds from source, assembles a tarball, runs setup.sh for scaffolding +# (user creation, directories, systemd units), initializes the cluster, +# and starts services via systemd. +# +# Usage: ./scripts/dev-install.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" + +echo "=== Building ===" +cd "$PROJECT_ROOT" && make build +cd "$MULGA_ROOT/viperblock" && make go_build_nbd + +echo "=== Assembling tarball for setup.sh ===" +STAGING=$(mktemp -d) +trap 'rm -rf "$STAGING"' EXIT +cp "$PROJECT_ROOT/bin/spx" "$STAGING/spx" +cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/" +cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/" +mkdir -p "$STAGING/systemd" +cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/" +mkdir -p "$STAGING/scripts" +cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/" +cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex" +tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" . + +echo "=== Cleaning stale state from previous installs ===" +# Stop any running services before modifying files +sudo systemctl stop spinifex.target 2>/dev/null || true +sudo systemctl reset-failed 2>/dev/null || true + +# Remove stale files owned by the dev user in production paths. +# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave +# files owned by tf-user that service users (spinifex-nats, etc.) can't read +# under systemd's ProtectSystem=strict sandboxing. +for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do + if [ -d "$dir" ]; then + # Remove PID files, stale logs, and the legacy ~/spinifex/config symlink + sudo find "$dir" -name '*.pid' -delete 2>/dev/null || true + sudo find "$dir" -name '*.log' -delete 2>/dev/null || true + sudo find "$dir" -name '*.log.*' -delete 2>/dev/null || true + fi +done +# Remove legacy data dir contents that conflict with production layout +if [ -d /var/lib/spinifex/config ]; then + sudo rm -rf /var/lib/spinifex/config +fi + +echo "=== Running setup.sh (creates users, dirs, systemd units) ===" +sudo INSTALL_SPINIFEX_TARBALL=/tmp/spinifex-local.tar.gz bash "$PROJECT_ROOT/scripts/setup.sh" +rm -f /tmp/spinifex-local.tar.gz + +echo "=== Setting up OVN ===" +sudo /usr/local/share/spinifex/setup-ovn.sh --management + +echo "=== Initializing ===" +sudo spx admin init --force --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1 + +echo "=== Installing CA certificate ===" +sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt +sudo update-ca-certificates + +echo "=== Starting services ===" +sudo systemctl start spinifex.target + +echo "=== Done ===" +echo "Services: sudo systemctl status spinifex.target" +echo "Logs: journalctl -u 'spinifex-*' -f" +echo "Test: AWS_PROFILE=spinifex aws ec2 describe-instances" +echo "Iterate: make deploy (rebuild + restart all)" + +# Check if user needs to re-login for spinifex group membership +if ! id -Gn 2>/dev/null | grep -qw spinifex; then + echo "" + echo "NOTE: You were added to the 'spinifex' group but your current" + echo "session doesn't reflect it yet. Log out and back in, or run:" + echo " newgrp spinifex" +fi diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index 4009ceab7..3412a2984 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -165,24 +165,16 @@ else fi echo "" -echo "🎉 Development environment setup complete!" +echo "Development environment setup complete!" echo "" -echo "📋 Next steps:" +echo "Recommended: Use scripts/dev-install.sh for a production-parity setup" +echo " (systemd, privilege separation, production paths)." echo "" -echo " 1. Setup OVN networking:" -echo " ./scripts/setup-ovn.sh --management" +echo " scripts/dev-install.sh # One-time: build, install, init, start via systemd" +echo " make deploy # Daily: rebuild + copy binary + restart all" +echo " make reinstall # Changed a .service file? Re-run full setup" echo "" -echo " 2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):" -echo " sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1" -echo "" -echo " 3. Start services:" -echo " ./scripts/start-dev.sh" -echo "" -echo "🔧 Development endpoints:" -echo " - Spinifex Gateway: https://localhost:$SPINIFEX_GATEWAY_PORT" -echo " - Predastore S3: https://localhost:$PREDASTORE_PORT" -echo " - NATS: nats://localhost:$NATS_PORT" -echo "" -echo "🧪 Test with AWS CLI:" -echo " export AWS_PROFILE=spinifex" -echo " aws ec2 describe-instances" +echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):" +echo " ./scripts/setup-ovn.sh --management" +echo " sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1" +echo " ./scripts/start-dev.sh" diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index 006f5bb5b..3f25e0079 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -565,21 +565,13 @@ else echo " systemd override: already exists" fi -# Create sudoers rule for network commands that always need root -# (ip tuntap, ip link set — NET_ADMIN operations) +# Sudoers rules for spinifex-daemon and spinifex-vpcd are managed by setup.sh +# (install_sudoers). Skip writing here to avoid conflicts. SUDOERS_FILE="/etc/sudoers.d/spinifex-network" -if [ ! -f "$SUDOERS_FILE" ]; then - CURRENT_USER=$(whoami) - sudo tee "$SUDOERS_FILE" >/dev/null < /dev/null 2>&1; then + $SUDO groupadd --system "$SPINIFEX_GROUP" fi - info "Services will run as: $SPINIFEX_USER:$SPINIFEX_GROUP" + # Create per-service users with correct home directories + declare -A SERVICE_HOMES=( + [nats]="/var/lib/spinifex/nats" + [gw]="/var/lib/spinifex" + [daemon]="/var/lib/spinifex/spinifex" + [storage]="/var/lib/spinifex/predastore" + [viperblock]="/var/lib/spinifex/viperblock" + [vpcd]="/var/lib/spinifex" + [ui]="/var/lib/spinifex" + ) + for svc in nats gw daemon storage viperblock vpcd ui; do + local user="spinifex-${svc}" + if ! id "$user" > /dev/null 2>&1; then + $SUDO useradd --system --no-create-home \ + --home-dir "${SERVICE_HOMES[$svc]}" \ + --gid "$SPINIFEX_GROUP" \ + --shell /usr/sbin/nologin \ + "$user" + fi + done - # Allow the service user to manage OVN/OVS and tap devices - $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS -# Spinifex VPC networking: allow service user to manage tap devices, OVS, and OVN -${SPINIFEX_USER} ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip -${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl -${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl + # Add invoking user to spinifex group for admin CLI access + ADMIN_USER="${SUDO_USER:-$(whoami)}" + if [ "$ADMIN_USER" != "root" ]; then + $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER" + fi + + # KVM access for daemon + if getent group kvm > /dev/null 2>&1; then + $SUDO usermod -aG kvm spinifex-daemon + fi + + info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})" +} + +# --- Install scoped sudoers rules --- +install_sudoers() { + $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS' +# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs +spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip +spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl +spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd + +# Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check +spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl +spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl +spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller SUDOERS $SUDO chmod 0440 /etc/sudoers.d/spinifex-network - info "Sudoers rules installed for $SPINIFEX_USER" + $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network" + info "Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd" } # --- Install apt dependencies --- @@ -241,12 +280,15 @@ install_files() { create_directories() { info "Creating directories..." + # Top-level directories (root-owned, group-readable by spinifex) $SUDO mkdir -p /etc/spinifex - $SUDO chmod 0700 /etc/spinifex - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex + $SUDO chmod 0750 /etc/spinifex + $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex $SUDO mkdir -p /var/lib/spinifex - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex + # TODO: tighten to 0750 once awsgw gets its own data dir (currently writes awsgw.pid here) + $SUDO chmod 0770 /var/lib/spinifex + $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/ if [ ! -e /var/lib/spinifex/config ]; then @@ -259,26 +301,57 @@ create_directories() { fi $SUDO mkdir -p /var/log/spinifex - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/log/spinifex + $SUDO chmod 0775 /var/log/spinifex + $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex $SUDO mkdir -p /run/spinifex - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /run/spinifex + $SUDO chmod 0775 /run/spinifex + $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex + + # Per-service config directories + $SUDO mkdir -p /etc/spinifex/nats + $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats + $SUDO chmod 0750 /etc/spinifex/nats + + $SUDO mkdir -p /etc/spinifex/predastore + $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore + $SUDO chmod 0750 /etc/spinifex/predastore + + # Per-service data directories + $SUDO mkdir -p /var/lib/spinifex/nats + $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats + $SUDO chmod 0700 /var/lib/spinifex/nats + + $SUDO mkdir -p /var/lib/spinifex/spinifex + $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex + $SUDO chmod 0700 /var/lib/spinifex/spinifex + + $SUDO mkdir -p /var/lib/spinifex/predastore + $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore + $SUDO chmod 0700 /var/lib/spinifex/predastore $SUDO mkdir -p /var/lib/spinifex/viperblock - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock + $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock + $SUDO chmod 0700 /var/lib/spinifex/viperblock + $SUDO mkdir -p /var/lib/spinifex/vpcd + $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd + $SUDO chmod 0700 /var/lib/spinifex/vpcd + + # Top-level config files (root-owned, group-readable) # Generate environment file with install-specific values (e.g. arch-dependent paths) $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF # Generated by setup.sh — install-specific environment variables SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so EOF - $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env + $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env + $SUDO chmod 0640 /etc/spinifex/systemd.env info "Generated /etc/spinifex/systemd.env" - # Service helper scripts + # Service helper scripts (root-owned, group-executable by all service users) if [ -d "$EXTRACT_DIR/scripts" ]; then for script in "$EXTRACT_DIR"/scripts/*.sh; do - $SUDO install -o "$SPINIFEX_USER" -g "$SPINIFEX_GROUP" -m 0755 \ + $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \ "$script" "/var/lib/spinifex/$(basename "$script")" info " /var/lib/spinifex/$(basename "$script")" done @@ -294,15 +367,12 @@ install_systemd() { fi for unit in "$EXTRACT_DIR"/systemd/*; do - # Substitute User= and Group= with the detected service user - sed "s/^User=spinifex$/User=$SPINIFEX_USER/;s/^Group=spinifex$/Group=$SPINIFEX_GROUP/" \ - "$unit" | $SUDO tee "/etc/systemd/system/$(basename "$unit")" > /dev/null - $SUDO chmod 0644 "/etc/systemd/system/$(basename "$unit")" + $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")" info " /etc/systemd/system/$(basename "$unit")" done $SUDO systemctl daemon-reload - info "Systemd units installed (running as $SPINIFEX_USER:$SPINIFEX_GROUP)" + info "Systemd units installed (per-service users)" } # --- Install logrotate --- @@ -343,7 +413,7 @@ print_summary() { echo "" echo " Version: $INSTALLED_VERSION" echo " Architecture: $ARCH" - echo " Service user: $SPINIFEX_USER" + echo " Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}" echo " Binary: /usr/local/bin/spx" echo " Config: /etc/spinifex/" echo " Data: /var/lib/spinifex/" @@ -384,7 +454,8 @@ main() { handle_upgrade install_apt_deps install_aws_cli - detect_service_user + create_service_users + install_sudoers download_spinifex install_files create_directories diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index b8da4a7f9..46a75b9d8 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -11,6 +11,11 @@ set -euo pipefail +echo "WARNING: start-dev.sh is deprecated for local development." >&2 +echo " Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2 +echo " start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2 +echo "" >&2 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" @@ -33,6 +38,8 @@ echo "Using configuration directory: $CONFIG_DIR" LOGS_DIR="$DATA_DIR/logs" WAL_DIR="$DATA_DIR/spinifex" +echo "⚠️ Dev mode: all services run as $(whoami) without privilege separation" +echo "" echo "🚀 Starting Spinifex development environment..." echo "Project root: $PROJECT_ROOT" echo "Data directory: $DATA_DIR" diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 2185a9359..4d36be803 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -40,6 +40,7 @@ type ConfigSettings struct { Region string NatsToken string DataDir string + LogDir string ConfigDir string // Add more fields as needed @@ -212,6 +213,7 @@ func CreateServiceDirectories(spxRoot string) { filepath.Join(spxRoot, "nats"), filepath.Join(spxRoot, "predastore"), filepath.Join(spxRoot, "viperblock"), + filepath.Join(spxRoot, "vpcd"), filepath.Join(spxRoot, "spinifex"), } @@ -241,18 +243,30 @@ func FileExists(path string) bool { func ChownRecursive(path, username string) { u, err := user.Lookup(username) if err != nil { + slog.Warn("ChownRecursive: user lookup failed, skipping", "user", username, "path", path, "err", err) + return + } + uid, err := strconv.Atoi(u.Uid) + if err != nil { + slog.Warn("ChownRecursive: invalid UID, skipping", "user", username, "uid", u.Uid, "err", err) + return + } + gid, err := strconv.Atoi(u.Gid) + if err != nil { + slog.Warn("ChownRecursive: invalid GID, skipping", "user", username, "gid", u.Gid, "err", err) return } - uid, _ := strconv.Atoi(u.Uid) - gid, _ := strconv.Atoi(u.Gid) // Use os.Root to scope filesystem operations and avoid symlink TOCTOU races. // Falls back to direct chown if os.Root is not available. root, rootErr := os.OpenRoot(path) if rootErr != nil { - // Fallback: direct chown on the top-level path only + // Fallback: direct chown on the top-level path only — subdirectory contents + // will retain their original ownership. + slog.Warn("ChownRecursive: OpenRoot not available, only top-level directory ownership changed", + "path", path, "err", rootErr) if chownErr := os.Chown(path, uid, gid); chownErr != nil { // #nosec G122 - slog.Debug("chown failed", "path", path, "err", chownErr) + slog.Warn("chown failed", "path", path, "err", chownErr) } return } @@ -271,6 +285,78 @@ func ChownRecursive(path, username string) { }) } +// SetServiceOwnership sets per-service ownership on data/config directories +// and shared config files to root:spinifex with correct modes. +// Keep in sync with setup.sh create_directories() and plan doc section 2. +func SetServiceOwnership() { + grp, err := user.LookupGroup("spinifex") + if err != nil { + slog.Error("SetServiceOwnership: spinifex group not found, skipping all ownership changes", "err", err) + fmt.Fprintln(os.Stderr, "WARNING: spinifex group not found — service ownership not set. Run setup.sh first or create the group manually.") + return + } + gid, err := strconv.Atoi(grp.Gid) + if err != nil { + slog.Error("SetServiceOwnership: invalid spinifex group GID", "gid", grp.Gid, "err", err) + fmt.Fprintln(os.Stderr, "WARNING: invalid spinifex group GID — service ownership not set.") + return + } + + // Per-service directory trees + for path, u := range map[string]string{ + "/etc/spinifex/nats": "spinifex-nats", + "/var/lib/spinifex/nats": "spinifex-nats", + "/etc/spinifex/predastore": "spinifex-storage", + "/var/lib/spinifex/predastore": "spinifex-storage", + "/var/lib/spinifex/spinifex": "spinifex-daemon", + "/var/lib/spinifex/viperblock": "spinifex-viperblock", + "/var/lib/spinifex/vpcd": "spinifex-vpcd", + } { + if _, err := os.Stat(path); err != nil { + continue + } + ChownRecursive(path, u) + } + + // Shared data directories — root:spinifex 0770 so daemon + admin CLI can write + for _, dir := range []string{ + "/var/lib/spinifex/images", + "/var/lib/spinifex/amis", + "/var/lib/spinifex/volumes", + "/var/lib/spinifex/state", + } { + if _, err := os.Stat(dir); err != nil { + continue + } + if err := os.Lchown(dir, 0, gid); err != nil { + slog.Warn("SetServiceOwnership: chown failed", "path", dir, "err", err) + } + if err := os.Chmod(dir, 0770); err != nil { //nolint:gosec // directories need group-write for daemon + admin CLI + slog.Warn("SetServiceOwnership: chmod failed", "path", dir, "err", err) + } + } + + // Shared config files — root:spinifex, ca.key stays root:root 0600 + for path, mode := range map[string]os.FileMode{ + "/etc/spinifex/spinifex.toml": 0640, + "/etc/spinifex/master.key": 0640, + "/etc/spinifex/bootstrap.json": 0660, // group-write so awsgw can consume and truncate + "/etc/spinifex/server.pem": 0644, + "/etc/spinifex/server.key": 0640, + "/etc/spinifex/ca.pem": 0644, + } { + if _, err := os.Stat(path); err != nil { + continue + } + if err := os.Lchown(path, 0, gid); err != nil { + slog.Warn("SetServiceOwnership: chown failed", "path", path, "err", err) + } + if err := os.Chmod(path, mode); err != nil { + slog.Warn("SetServiceOwnership: chmod failed", "path", path, "err", err) + } + } +} + // updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs func UpdateAWSINIFile(path, section string, values map[string]string) error { var cfg *ini.File diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index c5eb6d99b..2c0fe7955 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -601,6 +601,14 @@ func launchService(cfg *Config) (err error) { slog.Info("NBDKit started successfully and is running") } + // NBDKit creates the socket with its own umask (typically 0755). + // The daemon (different user, same group) needs write access to connect. + if nbdSocket != "" { + if err := os.Chmod(nbdSocket, 0770); err != nil { //nolint:gosec // socket needs group-write for cross-service access + slog.Warn("Failed to chmod NBD socket", "socket", nbdSocket, "err", err) + } + } + ebsResponse.Mounted = true ebsResponse.URI = nbdURI diff --git a/tests/e2e/run-multinode-e2e.sh b/tests/e2e/run-multinode-e2e.sh index c9addb44d..39ef854f6 100755 --- a/tests/e2e/run-multinode-e2e.sh +++ b/tests/e2e/run-multinode-e2e.sh @@ -11,9 +11,6 @@ set -euo pipefail # Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc) export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" -# Ensure we are in the spinifex project root -cd "$(dirname "$0")/../.." - # ========================================================================== # Argument parsing # ========================================================================== @@ -36,14 +33,12 @@ NATS_MONITOR_PORT=8222 PREDASTORE_PORT=8443 AWSGW_PORT=9999 SSH_KEY_PATH="$HOME/.ssh/tf-user-ap-southeast-2" -SPINIFEX_DATA_DIR="$HOME/spinifex" -SPINIFEX_CONFIG="$SPINIFEX_DATA_DIR/config/spinifex.toml" -SPINIFEX_BIN="./bin/spx" +SPINIFEX_BIN="spx" # Use Spinifex profile for AWS CLI export AWS_PROFILE=spinifex # Trust Spinifex CA for AWS CLI v2 (bundles its own Python/certifi, ignores system CA store) -export AWS_CA_BUNDLE="$SPINIFEX_DATA_DIR/config/ca.pem" +export AWS_CA_BUNDLE="/etc/spinifex/ca.pem" # Track test results TESTS_PASSED=0 @@ -106,19 +101,15 @@ dump_all_node_logs() { local ip="${NODE_IPS[$i]}" echo "" echo "=== Node $((i+1)) ($ip) ===" - if [ "$ip" = "$LOCAL_IP" ]; then - for f in "$SPINIFEX_DATA_DIR/logs/"*.log; do - [ -f "$f" ] || continue - echo "--- $(basename "$f") (last 50 lines) ---" - tail -50 "$f" 2>/dev/null || echo "(not found)" - done - else - peer_ssh "$ip" 'for f in ~/spinifex/logs/*.log; do - [ -f "$f" ] || continue - echo "--- $(basename "$f") (last 50 lines) ---" - tail -50 "$f" 2>/dev/null || echo "(not found)" - done' || echo "(node unreachable)" - fi + for svc in spinifex-nats spinifex-predastore spinifex-viperblock \ + spinifex-daemon spinifex-awsgw spinifex-vpcd; do + echo "--- ${svc} (last 50 lines) ---" + if [ "$ip" = "$LOCAL_IP" ]; then + sudo journalctl -u "$svc" --no-pager -n 50 2>/dev/null || echo "(not found)" + else + peer_ssh "$ip" "sudo journalctl -u $svc --no-pager -n 50" 2>/dev/null || echo "(node unreachable)" + fi + done done echo "" echo "==========================================" @@ -403,7 +394,7 @@ pass_test "Daemon readiness" # Spinifex CLI: get nodes echo "Checking spx get nodes..." -GET_NODES_OUTPUT=$($SPINIFEX_BIN get nodes --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null) +GET_NODES_OUTPUT=$($SPINIFEX_BIN get nodes --timeout 5s 2>/dev/null) echo "$GET_NODES_OUTPUT" READY_COUNT=$(echo "$GET_NODES_OUTPUT" | grep -c "Ready" || true) if [ "$READY_COUNT" -ge 3 ]; then @@ -415,7 +406,7 @@ fi # Spinifex CLI: get vms (should be empty) echo "Checking spx get vms (empty)..." -GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null) +GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null) echo "$GET_VMS_OUTPUT" pass_test "spx get vms (empty)" @@ -500,7 +491,7 @@ fi # Verify spx get vms shows all instances echo "" echo "Verifying spx get vms..." -GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null) +GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null) echo "$GET_VMS_OUTPUT" for instance_id in "${INSTANCE_IDS[@]}"; do if ! echo "$GET_VMS_OUTPUT" | grep -q "$instance_id"; then @@ -787,8 +778,8 @@ echo "Stopping services on node2 ($NODE2_IP) to simulate node failure..." # Stop services on node2 only — SPINIFEX_FORCE_LOCAL_STOP prevents coordinated # cluster shutdown which would kill all nodes via NATS -peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && SPINIFEX_FORCE_LOCAL_STOP=1 ./scripts/stop-dev.sh" || { - echo " WARNING: stop-dev.sh returned non-zero (may be expected)" +peer_ssh "$NODE2_IP" "sudo systemctl stop spinifex.target" || { + echo " WARNING: systemctl stop returned non-zero (may be expected)" } # Wait for NATS cluster to detect the failure and reform @@ -848,7 +839,7 @@ echo "Phase 9: Node Recovery" echo "========================================" echo "Restarting services on node2 ($NODE2_IP)..." -peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && ./scripts/start-dev.sh" || { +peer_ssh "$NODE2_IP" "sudo systemctl start spinifex.target" || { echo " ERROR: Failed to restart services on node2" fail_test "Node2 restart" } @@ -899,7 +890,7 @@ fi # Verify spx get nodes shows 3 Ready again echo " Checking spx get nodes after recovery..." -GET_NODES_RECOVER=$($SPINIFEX_BIN get nodes --config "$SPINIFEX_CONFIG" --timeout 10s 2>/dev/null || echo "") +GET_NODES_RECOVER=$($SPINIFEX_BIN get nodes --timeout 10s 2>/dev/null || echo "") echo "$GET_NODES_RECOVER" READY_RECOVER=$(echo "$GET_NODES_RECOVER" | grep -c "Ready" || true) if [ "$READY_RECOVER" -ge 3 ]; then @@ -1252,7 +1243,7 @@ else echo "Step 4: Validating spread placement across nodes..." # Use spx get vms to check node assignment - SPX_VMS=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null) + SPX_VMS=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null) echo "$SPX_VMS" SPREAD_NODES=()