mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
feat: privilege separation (#114)
This commit is contained in:
Julian Sommer
@@ -180,9 +180,13 @@ jobs:
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- name: Clone and build
|
||||
- name: Build Install Artifacts
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
|
||||
run: |
|
||||
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
||||
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
||||
"${RUNNER_TEMP}/setup.sh" \
|
||||
"${RUNNER_TEMP}/spinifex-tests.tar"
|
||||
|
||||
- name: Provision VMs
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
@@ -194,13 +198,14 @@ jobs:
|
||||
./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
|
||||
tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
||||
|
||||
- name: Bootstrap Cluster
|
||||
- name: Bootstrap via Binary Install
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
SPX_PREBUILT=true \
|
||||
SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
|
||||
SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
|
||||
./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
|
||||
./bootstrap-install.sh env2 \
|
||||
"${RUNNER_TEMP}/setup.sh" \
|
||||
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
||||
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
||||
|
||||
- name: Run E2E Tests
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
@@ -218,7 +223,7 @@ jobs:
|
||||
done
|
||||
|
||||
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
|
||||
"cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}"
|
||||
"cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS}"
|
||||
|
||||
- name: Teardown
|
||||
if: always()
|
||||
|
||||
@@ -116,6 +116,17 @@ run:
|
||||
$(MAKE) go_build
|
||||
$(MAKE) go_run
|
||||
|
||||
# Fast iteration: build + install binary + restart all services
|
||||
deploy: build
|
||||
sudo install -m 755 bin/spx /usr/local/bin/spx
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart spinifex.target
|
||||
|
||||
# Re-run setup.sh after changing systemd units, helper scripts, or logrotate config.
|
||||
# Not needed for code-only changes — use deploy for those.
|
||||
reinstall:
|
||||
scripts/dev-install.sh
|
||||
|
||||
clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/services/spinifexui/frontend/dist
|
||||
@@ -207,7 +218,8 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
|
||||
.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck \
|
||||
distro distro-amd64 distro-arm64 distro-clean
|
||||
|
||||
@@ -6,7 +6,7 @@ Wants=spinifex-nats.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-gw
|
||||
Group=spinifex
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service awsgw start
|
||||
@@ -18,5 +18,27 @@ Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/
|
||||
Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem
|
||||
Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key
|
||||
|
||||
# Filesystem access
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
|
||||
ReadWritePaths=/var/lib/spinifex /var/log/spinifex /etc/spinifex/bootstrap.json /tmp
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -6,7 +6,7 @@ Wants=spinifex-nats.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
@@ -17,5 +17,29 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
|
||||
Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
|
||||
Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
|
||||
|
||||
# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
|
||||
ProtectSystem=full
|
||||
PrivateDevices=no
|
||||
DevicePolicy=closed
|
||||
DeviceAllow=/dev/kvm rw
|
||||
DeviceAllow=/dev/net/tun rw
|
||||
ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key
|
||||
ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -5,7 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-nats
|
||||
Group=spinifex
|
||||
ExecStart=/usr/local/bin/spx service nats start
|
||||
Restart=on-failure
|
||||
@@ -13,5 +13,27 @@ RestartSec=5
|
||||
OOMScoreAdjust=-500
|
||||
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
|
||||
|
||||
# Filesystem access
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/nats
|
||||
ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex /tmp
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -6,7 +6,7 @@ Wants=spinifex-nats.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-storage
|
||||
Group=spinifex
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/var/lib/spinifex/predastore-start.sh
|
||||
@@ -18,5 +18,27 @@ Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
|
||||
Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
|
||||
Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
|
||||
|
||||
# Filesystem access
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
|
||||
ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex /tmp
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -5,7 +5,7 @@ After=spinifex-awsgw.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-ui
|
||||
Group=spinifex
|
||||
ExecStart=/usr/local/bin/spx service spinifex-ui start
|
||||
Restart=on-failure
|
||||
@@ -14,5 +14,27 @@ OOMScoreAdjust=-500
|
||||
Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
|
||||
Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
|
||||
|
||||
# Filesystem access
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
|
||||
ReadWritePaths=/var/log/spinifex /tmp
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -6,7 +6,7 @@ Wants=spinifex-nats.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-viperblock
|
||||
Group=spinifex
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service viperblock start
|
||||
@@ -17,5 +17,27 @@ EnvironmentFile=/etc/spinifex/systemd.env
|
||||
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
|
||||
Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
|
||||
|
||||
# Filesystem access
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env
|
||||
ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /tmp
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
@@ -6,7 +6,7 @@ Wants=spinifex-nats.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=spinifex
|
||||
User=spinifex-vpcd
|
||||
Group=spinifex
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service vpcd start
|
||||
@@ -14,6 +14,28 @@ Restart=on-failure
|
||||
RestartSec=5
|
||||
OOMScoreAdjust=-500
|
||||
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
|
||||
Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
|
||||
|
||||
# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
|
||||
ProtectSystem=full
|
||||
PrivateDevices=yes
|
||||
ReadOnlyPaths=/etc/spinifex/spinifex.toml
|
||||
ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
|
||||
|
||||
# Hardening
|
||||
ProtectHome=yes
|
||||
PrivateTmp=no
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
ProtectClock=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectProc=invisible
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target
|
||||
|
||||
+12
-21
@@ -933,6 +933,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
Region: region,
|
||||
NatsToken: natsToken,
|
||||
DataDir: spxRoot,
|
||||
LogDir: LogDirFor(spxRoot),
|
||||
ConfigDir: configDir,
|
||||
|
||||
Node: node,
|
||||
@@ -1014,14 +1015,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
|
||||
admin.CreateServiceDirectories(spxRoot)
|
||||
|
||||
// In production layout (running as root), fix ownership so the service user
|
||||
// can read config and write data. SUDO_USER identifies the operator account.
|
||||
// In production layout (running as root), set per-service ownership on
|
||||
// directories and shared config files (root:spinifex with correct modes).
|
||||
if os.Getuid() == 0 {
|
||||
sudoUser := os.Getenv("SUDO_USER")
|
||||
if sudoUser != "" {
|
||||
admin.ChownRecursive(configDir, sudoUser)
|
||||
admin.ChownRecursive(spxRoot, sudoUser)
|
||||
}
|
||||
admin.SetServiceOwnership()
|
||||
}
|
||||
|
||||
// Print success message
|
||||
@@ -1171,6 +1168,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
|
||||
Region: region,
|
||||
NatsToken: natsToken,
|
||||
DataDir: spxRoot,
|
||||
LogDir: LogDirFor(spxRoot),
|
||||
ConfigDir: configDir,
|
||||
|
||||
Node: node,
|
||||
@@ -1246,14 +1244,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
|
||||
|
||||
admin.CreateServiceDirectories(spxRoot)
|
||||
|
||||
// In production layout (running as root), fix ownership so the service user
|
||||
// can read config and write data. SUDO_USER identifies the operator account.
|
||||
// In production layout (running as root), set per-service ownership on
|
||||
// directories and shared config files (root:spinifex with correct modes).
|
||||
if os.Getuid() == 0 {
|
||||
sudoUser := os.Getenv("SUDO_USER")
|
||||
if sudoUser != "" {
|
||||
admin.ChownRecursive(configDir, sudoUser)
|
||||
admin.ChownRecursive(spxRoot, sudoUser)
|
||||
}
|
||||
admin.SetServiceOwnership()
|
||||
}
|
||||
|
||||
// Keep formation server running briefly so joining nodes can fetch complete status
|
||||
@@ -1569,6 +1563,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
Region: creds.Region,
|
||||
NatsToken: creds.NatsToken,
|
||||
DataDir: dataDir,
|
||||
LogDir: LogDirFor(dataDir),
|
||||
ConfigDir: configDir,
|
||||
|
||||
Node: node,
|
||||
@@ -1648,14 +1643,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
|
||||
admin.CreateServiceDirectories(dataDir)
|
||||
|
||||
// In production layout (running as root), fix ownership so the service user
|
||||
// can read config and write data. SUDO_USER identifies the operator account.
|
||||
// In production layout (running as root), set per-service ownership on
|
||||
// directories and shared config files (root:spinifex with correct modes).
|
||||
if os.Getuid() == 0 {
|
||||
sudoUser := os.Getenv("SUDO_USER")
|
||||
if sudoUser != "" {
|
||||
admin.ChownRecursive(configDir, sudoUser)
|
||||
admin.ChownRecursive(dataDir, sudoUser)
|
||||
}
|
||||
admin.SetServiceOwnership()
|
||||
}
|
||||
|
||||
// Print cluster summary
|
||||
|
||||
@@ -26,6 +26,16 @@ func DefaultDataDir() string {
|
||||
return filepath.Join(realUserHomeDir(), "spinifex")
|
||||
}
|
||||
|
||||
// LogDirFor returns the log directory for a given data directory.
|
||||
// Production: /var/log/spinifex (matches systemd ReadWritePaths)
|
||||
// Development: <dataDir>/logs (supports custom per-node data dirs)
|
||||
func LogDirFor(dataDir string) string {
|
||||
if isProductionLayout() {
|
||||
return "/var/log/spinifex"
|
||||
}
|
||||
return filepath.Join(dataDir, "logs")
|
||||
}
|
||||
|
||||
// realUserHomeDir returns the home directory of the real (non-sudo) user.
|
||||
// When running under sudo, SUDO_USER is set to the invoking user — resolve
|
||||
// their home directory so config/data land in the right place.
|
||||
|
||||
@@ -733,6 +733,12 @@ var vpcdStartCmd = &cobra.Command{
|
||||
}
|
||||
}
|
||||
|
||||
baseDir := viper.GetString("base-dir")
|
||||
if baseDir != "" {
|
||||
fmt.Println("Overwriting vpcd base-dir to:", baseDir)
|
||||
nodeConfig.BaseDir = baseDir
|
||||
}
|
||||
|
||||
svc, err := service.New("vpcd", &vpcd.Config{
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
NatsToken: nodeConfig.NATS.ACL.Token,
|
||||
|
||||
@@ -40,7 +40,7 @@ jetstream {
|
||||
debug: false
|
||||
trace: false
|
||||
logtime: true
|
||||
log_file: "{{.DataDir}}/logs/nats.log"
|
||||
log_file: "{{.LogDir}}/nats.log"
|
||||
|
||||
# Limits
|
||||
max_connections: 1000
|
||||
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# dev-install.sh — Full local development setup via production installer.
|
||||
# Builds from source, assembles a tarball, runs setup.sh for scaffolding
|
||||
# (user creation, directories, systemd units), initializes the cluster,
|
||||
# and starts services via systemd.
|
||||
#
|
||||
# Usage: ./scripts/dev-install.sh
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
|
||||
|
||||
echo "=== Building ==="
|
||||
cd "$PROJECT_ROOT" && make build
|
||||
cd "$MULGA_ROOT/viperblock" && make go_build_nbd
|
||||
|
||||
echo "=== Assembling tarball for setup.sh ==="
|
||||
STAGING=$(mktemp -d)
|
||||
trap 'rm -rf "$STAGING"' EXIT
|
||||
cp "$PROJECT_ROOT/bin/spx" "$STAGING/spx"
|
||||
cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
|
||||
mkdir -p "$STAGING/systemd"
|
||||
cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
|
||||
mkdir -p "$STAGING/scripts"
|
||||
cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
|
||||
cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
|
||||
tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
|
||||
|
||||
echo "=== Cleaning stale state from previous installs ==="
|
||||
# Stop any running services before modifying files
|
||||
sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
sudo systemctl reset-failed 2>/dev/null || true
|
||||
|
||||
# Remove stale files owned by the dev user in production paths.
|
||||
# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave
|
||||
# files owned by tf-user that service users (spinifex-nats, etc.) can't read
|
||||
# under systemd's ProtectSystem=strict sandboxing.
|
||||
for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do
|
||||
if [ -d "$dir" ]; then
|
||||
# Remove PID files, stale logs, and the legacy ~/spinifex/config symlink
|
||||
sudo find "$dir" -name '*.pid' -delete 2>/dev/null || true
|
||||
sudo find "$dir" -name '*.log' -delete 2>/dev/null || true
|
||||
sudo find "$dir" -name '*.log.*' -delete 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# Remove legacy data dir contents that conflict with production layout
|
||||
if [ -d /var/lib/spinifex/config ]; then
|
||||
sudo rm -rf /var/lib/spinifex/config
|
||||
fi
|
||||
|
||||
echo "=== Running setup.sh (creates users, dirs, systemd units) ==="
|
||||
sudo INSTALL_SPINIFEX_TARBALL=/tmp/spinifex-local.tar.gz bash "$PROJECT_ROOT/scripts/setup.sh"
|
||||
rm -f /tmp/spinifex-local.tar.gz
|
||||
|
||||
echo "=== Setting up OVN ==="
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management
|
||||
|
||||
echo "=== Initializing ==="
|
||||
sudo spx admin init --force --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
|
||||
|
||||
echo "=== Installing CA certificate ==="
|
||||
sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
|
||||
sudo update-ca-certificates
|
||||
|
||||
echo "=== Starting services ==="
|
||||
sudo systemctl start spinifex.target
|
||||
|
||||
echo "=== Done ==="
|
||||
echo "Services: sudo systemctl status spinifex.target"
|
||||
echo "Logs: journalctl -u 'spinifex-*' -f"
|
||||
echo "Test: AWS_PROFILE=spinifex aws ec2 describe-instances"
|
||||
echo "Iterate: make deploy (rebuild + restart all)"
|
||||
|
||||
# Check if user needs to re-login for spinifex group membership
|
||||
if ! id -Gn 2>/dev/null | grep -qw spinifex; then
|
||||
echo ""
|
||||
echo "NOTE: You were added to the 'spinifex' group but your current"
|
||||
echo "session doesn't reflect it yet. Log out and back in, or run:"
|
||||
echo " newgrp spinifex"
|
||||
fi
|
||||
+10
-18
@@ -165,24 +165,16 @@ else
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🎉 Development environment setup complete!"
|
||||
echo "Development environment setup complete!"
|
||||
echo ""
|
||||
echo "📋 Next steps:"
|
||||
echo "Recommended: Use scripts/dev-install.sh for a production-parity setup"
|
||||
echo " (systemd, privilege separation, production paths)."
|
||||
echo ""
|
||||
echo " 1. Setup OVN networking:"
|
||||
echo " ./scripts/setup-ovn.sh --management"
|
||||
echo " scripts/dev-install.sh # One-time: build, install, init, start via systemd"
|
||||
echo " make deploy # Daily: rebuild + copy binary + restart all"
|
||||
echo " make reinstall # Changed a .service file? Re-run full setup"
|
||||
echo ""
|
||||
echo " 2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):"
|
||||
echo " sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
|
||||
echo ""
|
||||
echo " 3. Start services:"
|
||||
echo " ./scripts/start-dev.sh"
|
||||
echo ""
|
||||
echo "🔧 Development endpoints:"
|
||||
echo " - Spinifex Gateway: https://localhost:$SPINIFEX_GATEWAY_PORT"
|
||||
echo " - Predastore S3: https://localhost:$PREDASTORE_PORT"
|
||||
echo " - NATS: nats://localhost:$NATS_PORT"
|
||||
echo ""
|
||||
echo "🧪 Test with AWS CLI:"
|
||||
echo " export AWS_PROFILE=spinifex"
|
||||
echo " aws ec2 describe-instances"
|
||||
echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):"
|
||||
echo " ./scripts/setup-ovn.sh --management"
|
||||
echo " sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
|
||||
echo " ./scripts/start-dev.sh"
|
||||
|
||||
+5
-13
@@ -565,21 +565,13 @@ else
|
||||
echo " systemd override: already exists"
|
||||
fi
|
||||
|
||||
# Create sudoers rule for network commands that always need root
|
||||
# (ip tuntap, ip link set — NET_ADMIN operations)
|
||||
# Sudoers rules for spinifex-daemon and spinifex-vpcd are managed by setup.sh
|
||||
# (install_sudoers). Skip writing here to avoid conflicts.
|
||||
SUDOERS_FILE="/etc/sudoers.d/spinifex-network"
|
||||
if [ ! -f "$SUDOERS_FILE" ]; then
|
||||
CURRENT_USER=$(whoami)
|
||||
sudo tee "$SUDOERS_FILE" >/dev/null <<EOF
|
||||
# Spinifex VPC networking: allow non-root daemon to manage tap devices and OVS
|
||||
$CURRENT_USER ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
|
||||
$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
|
||||
EOF
|
||||
sudo chmod 0440 "$SUDOERS_FILE"
|
||||
echo " sudoers rule: created ($SUDOERS_FILE)"
|
||||
if [ -f "$SUDOERS_FILE" ]; then
|
||||
echo " sudoers rule: already exists ($SUDOERS_FILE, managed by setup.sh)"
|
||||
else
|
||||
echo " sudoers rule: already exists"
|
||||
echo " sudoers rule: not found — run setup.sh first, or install manually"
|
||||
fi
|
||||
|
||||
# --- Step 9: Configure OVN log rotation ---
|
||||
|
||||
+102
-31
@@ -89,27 +89,66 @@ detect_arch() {
|
||||
info "Detected architecture: $MACHINE ($ARCH)"
|
||||
}
|
||||
|
||||
# --- Detect service user ---
|
||||
detect_service_user() {
|
||||
# Use the real user — either the current user, or SUDO_USER if running via sudo
|
||||
SPINIFEX_USER="${SUDO_USER:-$(whoami)}"
|
||||
SPINIFEX_GROUP="$(id -gn "$SPINIFEX_USER")"
|
||||
# --- Create per-service system users ---
|
||||
create_service_users() {
|
||||
SPINIFEX_GROUP="spinifex"
|
||||
|
||||
if [ "$SPINIFEX_USER" = "root" ]; then
|
||||
fatal "Cannot determine service user. Do not run as root directly — use: curl ... | bash (as a normal user)"
|
||||
# Create shared group
|
||||
if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
|
||||
$SUDO groupadd --system "$SPINIFEX_GROUP"
|
||||
fi
|
||||
|
||||
info "Services will run as: $SPINIFEX_USER:$SPINIFEX_GROUP"
|
||||
# Create per-service users with correct home directories
|
||||
declare -A SERVICE_HOMES=(
|
||||
[nats]="/var/lib/spinifex/nats"
|
||||
[gw]="/var/lib/spinifex"
|
||||
[daemon]="/var/lib/spinifex/spinifex"
|
||||
[storage]="/var/lib/spinifex/predastore"
|
||||
[viperblock]="/var/lib/spinifex/viperblock"
|
||||
[vpcd]="/var/lib/spinifex"
|
||||
[ui]="/var/lib/spinifex"
|
||||
)
|
||||
for svc in nats gw daemon storage viperblock vpcd ui; do
|
||||
local user="spinifex-${svc}"
|
||||
if ! id "$user" > /dev/null 2>&1; then
|
||||
$SUDO useradd --system --no-create-home \
|
||||
--home-dir "${SERVICE_HOMES[$svc]}" \
|
||||
--gid "$SPINIFEX_GROUP" \
|
||||
--shell /usr/sbin/nologin \
|
||||
"$user"
|
||||
fi
|
||||
done
|
||||
|
||||
# Allow the service user to manage OVN/OVS and tap devices
|
||||
$SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
|
||||
# Spinifex VPC networking: allow service user to manage tap devices, OVS, and OVN
|
||||
${SPINIFEX_USER} ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
|
||||
${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
|
||||
# Add invoking user to spinifex group for admin CLI access
|
||||
ADMIN_USER="${SUDO_USER:-$(whoami)}"
|
||||
if [ "$ADMIN_USER" != "root" ]; then
|
||||
$SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
|
||||
fi
|
||||
|
||||
# KVM access for daemon
|
||||
if getent group kvm > /dev/null 2>&1; then
|
||||
$SUDO usermod -aG kvm spinifex-daemon
|
||||
fi
|
||||
|
||||
info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})"
|
||||
}
|
||||
|
||||
# --- Install scoped sudoers rules ---
|
||||
install_sudoers() {
|
||||
$SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
|
||||
# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs
|
||||
spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
|
||||
spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
|
||||
|
||||
# Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check
|
||||
spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
|
||||
spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
|
||||
SUDOERS
|
||||
$SUDO chmod 0440 /etc/sudoers.d/spinifex-network
|
||||
info "Sudoers rules installed for $SPINIFEX_USER"
|
||||
$SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
|
||||
info "Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd"
|
||||
}
|
||||
|
||||
# --- Install apt dependencies ---
|
||||
@@ -241,12 +280,15 @@ install_files() {
|
||||
create_directories() {
|
||||
info "Creating directories..."
|
||||
|
||||
# Top-level directories (root-owned, group-readable by spinifex)
|
||||
$SUDO mkdir -p /etc/spinifex
|
||||
$SUDO chmod 0700 /etc/spinifex
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex
|
||||
$SUDO chmod 0750 /etc/spinifex
|
||||
$SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex
|
||||
|
||||
$SUDO mkdir -p /var/lib/spinifex
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex
|
||||
# TODO: tighten to 0750 once awsgw gets its own data dir (currently writes awsgw.pid here)
|
||||
$SUDO chmod 0770 /var/lib/spinifex
|
||||
$SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex
|
||||
|
||||
# Symlink so services that expect BaseDir/config/ can find /etc/spinifex/
|
||||
if [ ! -e /var/lib/spinifex/config ]; then
|
||||
@@ -259,26 +301,57 @@ create_directories() {
|
||||
fi
|
||||
|
||||
$SUDO mkdir -p /var/log/spinifex
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/log/spinifex
|
||||
$SUDO chmod 0775 /var/log/spinifex
|
||||
$SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex
|
||||
|
||||
$SUDO mkdir -p /run/spinifex
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /run/spinifex
|
||||
$SUDO chmod 0775 /run/spinifex
|
||||
$SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex
|
||||
|
||||
# Per-service config directories
|
||||
$SUDO mkdir -p /etc/spinifex/nats
|
||||
$SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
|
||||
$SUDO chmod 0750 /etc/spinifex/nats
|
||||
|
||||
$SUDO mkdir -p /etc/spinifex/predastore
|
||||
$SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
|
||||
$SUDO chmod 0750 /etc/spinifex/predastore
|
||||
|
||||
# Per-service data directories
|
||||
$SUDO mkdir -p /var/lib/spinifex/nats
|
||||
$SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
|
||||
$SUDO chmod 0700 /var/lib/spinifex/nats
|
||||
|
||||
$SUDO mkdir -p /var/lib/spinifex/spinifex
|
||||
$SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
|
||||
$SUDO chmod 0700 /var/lib/spinifex/spinifex
|
||||
|
||||
$SUDO mkdir -p /var/lib/spinifex/predastore
|
||||
$SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
|
||||
$SUDO chmod 0700 /var/lib/spinifex/predastore
|
||||
|
||||
$SUDO mkdir -p /var/lib/spinifex/viperblock
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
|
||||
$SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
|
||||
$SUDO chmod 0700 /var/lib/spinifex/viperblock
|
||||
|
||||
$SUDO mkdir -p /var/lib/spinifex/vpcd
|
||||
$SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
|
||||
$SUDO chmod 0700 /var/lib/spinifex/vpcd
|
||||
|
||||
# Top-level config files (root-owned, group-readable)
|
||||
# Generate environment file with install-specific values (e.g. arch-dependent paths)
|
||||
$SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
|
||||
# Generated by setup.sh — install-specific environment variables
|
||||
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
|
||||
EOF
|
||||
$SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
|
||||
$SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
|
||||
$SUDO chmod 0640 /etc/spinifex/systemd.env
|
||||
info "Generated /etc/spinifex/systemd.env"
|
||||
|
||||
# Service helper scripts
|
||||
# Service helper scripts (root-owned, group-executable by all service users)
|
||||
if [ -d "$EXTRACT_DIR/scripts" ]; then
|
||||
for script in "$EXTRACT_DIR"/scripts/*.sh; do
|
||||
$SUDO install -o "$SPINIFEX_USER" -g "$SPINIFEX_GROUP" -m 0755 \
|
||||
$SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
|
||||
"$script" "/var/lib/spinifex/$(basename "$script")"
|
||||
info " /var/lib/spinifex/$(basename "$script")"
|
||||
done
|
||||
@@ -294,15 +367,12 @@ install_systemd() {
|
||||
fi
|
||||
|
||||
for unit in "$EXTRACT_DIR"/systemd/*; do
|
||||
# Substitute User= and Group= with the detected service user
|
||||
sed "s/^User=spinifex$/User=$SPINIFEX_USER/;s/^Group=spinifex$/Group=$SPINIFEX_GROUP/" \
|
||||
"$unit" | $SUDO tee "/etc/systemd/system/$(basename "$unit")" > /dev/null
|
||||
$SUDO chmod 0644 "/etc/systemd/system/$(basename "$unit")"
|
||||
$SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
|
||||
info " /etc/systemd/system/$(basename "$unit")"
|
||||
done
|
||||
|
||||
$SUDO systemctl daemon-reload
|
||||
info "Systemd units installed (running as $SPINIFEX_USER:$SPINIFEX_GROUP)"
|
||||
info "Systemd units installed (per-service users)"
|
||||
}
|
||||
|
||||
# --- Install logrotate ---
|
||||
@@ -343,7 +413,7 @@ print_summary() {
|
||||
echo ""
|
||||
echo " Version: $INSTALLED_VERSION"
|
||||
echo " Architecture: $ARCH"
|
||||
echo " Service user: $SPINIFEX_USER"
|
||||
echo " Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
|
||||
echo " Binary: /usr/local/bin/spx"
|
||||
echo " Config: /etc/spinifex/"
|
||||
echo " Data: /var/lib/spinifex/"
|
||||
@@ -384,7 +454,8 @@ main() {
|
||||
handle_upgrade
|
||||
install_apt_deps
|
||||
install_aws_cli
|
||||
detect_service_user
|
||||
create_service_users
|
||||
install_sudoers
|
||||
download_spinifex
|
||||
install_files
|
||||
create_directories
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "WARNING: start-dev.sh is deprecated for local development." >&2
|
||||
echo " Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2
|
||||
echo " start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2
|
||||
echo "" >&2
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
|
||||
@@ -33,6 +38,8 @@ echo "Using configuration directory: $CONFIG_DIR"
|
||||
LOGS_DIR="$DATA_DIR/logs"
|
||||
WAL_DIR="$DATA_DIR/spinifex"
|
||||
|
||||
echo "⚠️ Dev mode: all services run as $(whoami) without privilege separation"
|
||||
echo ""
|
||||
echo "🚀 Starting Spinifex development environment..."
|
||||
echo "Project root: $PROJECT_ROOT"
|
||||
echo "Data directory: $DATA_DIR"
|
||||
|
||||
+90
-4
@@ -40,6 +40,7 @@ type ConfigSettings struct {
|
||||
Region string
|
||||
NatsToken string
|
||||
DataDir string
|
||||
LogDir string
|
||||
ConfigDir string
|
||||
|
||||
// Add more fields as needed
|
||||
@@ -212,6 +213,7 @@ func CreateServiceDirectories(spxRoot string) {
|
||||
filepath.Join(spxRoot, "nats"),
|
||||
filepath.Join(spxRoot, "predastore"),
|
||||
filepath.Join(spxRoot, "viperblock"),
|
||||
filepath.Join(spxRoot, "vpcd"),
|
||||
filepath.Join(spxRoot, "spinifex"),
|
||||
}
|
||||
|
||||
@@ -241,18 +243,30 @@ func FileExists(path string) bool {
|
||||
func ChownRecursive(path, username string) {
|
||||
u, err := user.Lookup(username)
|
||||
if err != nil {
|
||||
slog.Warn("ChownRecursive: user lookup failed, skipping", "user", username, "path", path, "err", err)
|
||||
return
|
||||
}
|
||||
uid, err := strconv.Atoi(u.Uid)
|
||||
if err != nil {
|
||||
slog.Warn("ChownRecursive: invalid UID, skipping", "user", username, "uid", u.Uid, "err", err)
|
||||
return
|
||||
}
|
||||
gid, err := strconv.Atoi(u.Gid)
|
||||
if err != nil {
|
||||
slog.Warn("ChownRecursive: invalid GID, skipping", "user", username, "gid", u.Gid, "err", err)
|
||||
return
|
||||
}
|
||||
uid, _ := strconv.Atoi(u.Uid)
|
||||
gid, _ := strconv.Atoi(u.Gid)
|
||||
|
||||
// Use os.Root to scope filesystem operations and avoid symlink TOCTOU races.
|
||||
// Falls back to direct chown if os.Root is not available.
|
||||
root, rootErr := os.OpenRoot(path)
|
||||
if rootErr != nil {
|
||||
// Fallback: direct chown on the top-level path only
|
||||
// Fallback: direct chown on the top-level path only — subdirectory contents
|
||||
// will retain their original ownership.
|
||||
slog.Warn("ChownRecursive: OpenRoot not available, only top-level directory ownership changed",
|
||||
"path", path, "err", rootErr)
|
||||
if chownErr := os.Chown(path, uid, gid); chownErr != nil { // #nosec G122
|
||||
slog.Debug("chown failed", "path", path, "err", chownErr)
|
||||
slog.Warn("chown failed", "path", path, "err", chownErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -271,6 +285,78 @@ func ChownRecursive(path, username string) {
|
||||
})
|
||||
}
|
||||
|
||||
// SetServiceOwnership sets per-service ownership on data/config directories
|
||||
// and shared config files to root:spinifex with correct modes.
|
||||
// Keep in sync with setup.sh create_directories() and plan doc section 2.
|
||||
func SetServiceOwnership() {
|
||||
grp, err := user.LookupGroup("spinifex")
|
||||
if err != nil {
|
||||
slog.Error("SetServiceOwnership: spinifex group not found, skipping all ownership changes", "err", err)
|
||||
fmt.Fprintln(os.Stderr, "WARNING: spinifex group not found — service ownership not set. Run setup.sh first or create the group manually.")
|
||||
return
|
||||
}
|
||||
gid, err := strconv.Atoi(grp.Gid)
|
||||
if err != nil {
|
||||
slog.Error("SetServiceOwnership: invalid spinifex group GID", "gid", grp.Gid, "err", err)
|
||||
fmt.Fprintln(os.Stderr, "WARNING: invalid spinifex group GID — service ownership not set.")
|
||||
return
|
||||
}
|
||||
|
||||
// Per-service directory trees
|
||||
for path, u := range map[string]string{
|
||||
"/etc/spinifex/nats": "spinifex-nats",
|
||||
"/var/lib/spinifex/nats": "spinifex-nats",
|
||||
"/etc/spinifex/predastore": "spinifex-storage",
|
||||
"/var/lib/spinifex/predastore": "spinifex-storage",
|
||||
"/var/lib/spinifex/spinifex": "spinifex-daemon",
|
||||
"/var/lib/spinifex/viperblock": "spinifex-viperblock",
|
||||
"/var/lib/spinifex/vpcd": "spinifex-vpcd",
|
||||
} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
continue
|
||||
}
|
||||
ChownRecursive(path, u)
|
||||
}
|
||||
|
||||
// Shared data directories — root:spinifex 0770 so daemon + admin CLI can write
|
||||
for _, dir := range []string{
|
||||
"/var/lib/spinifex/images",
|
||||
"/var/lib/spinifex/amis",
|
||||
"/var/lib/spinifex/volumes",
|
||||
"/var/lib/spinifex/state",
|
||||
} {
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := os.Lchown(dir, 0, gid); err != nil {
|
||||
slog.Warn("SetServiceOwnership: chown failed", "path", dir, "err", err)
|
||||
}
|
||||
if err := os.Chmod(dir, 0770); err != nil { //nolint:gosec // directories need group-write for daemon + admin CLI
|
||||
slog.Warn("SetServiceOwnership: chmod failed", "path", dir, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shared config files — root:spinifex, ca.key stays root:root 0600
|
||||
for path, mode := range map[string]os.FileMode{
|
||||
"/etc/spinifex/spinifex.toml": 0640,
|
||||
"/etc/spinifex/master.key": 0640,
|
||||
"/etc/spinifex/bootstrap.json": 0660, // group-write so awsgw can consume and truncate
|
||||
"/etc/spinifex/server.pem": 0644,
|
||||
"/etc/spinifex/server.key": 0640,
|
||||
"/etc/spinifex/ca.pem": 0644,
|
||||
} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := os.Lchown(path, 0, gid); err != nil {
|
||||
slog.Warn("SetServiceOwnership: chown failed", "path", path, "err", err)
|
||||
}
|
||||
if err := os.Chmod(path, mode); err != nil {
|
||||
slog.Warn("SetServiceOwnership: chmod failed", "path", path, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
|
||||
func UpdateAWSINIFile(path, section string, values map[string]string) error {
|
||||
var cfg *ini.File
|
||||
|
||||
@@ -601,6 +601,14 @@ func launchService(cfg *Config) (err error) {
|
||||
slog.Info("NBDKit started successfully and is running")
|
||||
}
|
||||
|
||||
// NBDKit creates the socket with its own umask (typically 0755).
|
||||
// The daemon (different user, same group) needs write access to connect.
|
||||
if nbdSocket != "" {
|
||||
if err := os.Chmod(nbdSocket, 0770); err != nil { //nolint:gosec // socket needs group-write for cross-service access
|
||||
slog.Warn("Failed to chmod NBD socket", "socket", nbdSocket, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
ebsResponse.Mounted = true
|
||||
ebsResponse.URI = nbdURI
|
||||
|
||||
|
||||
@@ -11,9 +11,6 @@ set -euo pipefail
|
||||
# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc)
|
||||
export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
|
||||
|
||||
# Ensure we are in the spinifex project root
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
# ==========================================================================
|
||||
# Argument parsing
|
||||
# ==========================================================================
|
||||
@@ -36,14 +33,12 @@ NATS_MONITOR_PORT=8222
|
||||
PREDASTORE_PORT=8443
|
||||
AWSGW_PORT=9999
|
||||
SSH_KEY_PATH="$HOME/.ssh/tf-user-ap-southeast-2"
|
||||
SPINIFEX_DATA_DIR="$HOME/spinifex"
|
||||
SPINIFEX_CONFIG="$SPINIFEX_DATA_DIR/config/spinifex.toml"
|
||||
SPINIFEX_BIN="./bin/spx"
|
||||
SPINIFEX_BIN="spx"
|
||||
|
||||
# Use Spinifex profile for AWS CLI
|
||||
export AWS_PROFILE=spinifex
|
||||
# Trust Spinifex CA for AWS CLI v2 (bundles its own Python/certifi, ignores system CA store)
|
||||
export AWS_CA_BUNDLE="$SPINIFEX_DATA_DIR/config/ca.pem"
|
||||
export AWS_CA_BUNDLE="/etc/spinifex/ca.pem"
|
||||
|
||||
# Track test results
|
||||
TESTS_PASSED=0
|
||||
@@ -106,19 +101,15 @@ dump_all_node_logs() {
|
||||
local ip="${NODE_IPS[$i]}"
|
||||
echo ""
|
||||
echo "=== Node $((i+1)) ($ip) ==="
|
||||
if [ "$ip" = "$LOCAL_IP" ]; then
|
||||
for f in "$SPINIFEX_DATA_DIR/logs/"*.log; do
|
||||
[ -f "$f" ] || continue
|
||||
echo "--- $(basename "$f") (last 50 lines) ---"
|
||||
tail -50 "$f" 2>/dev/null || echo "(not found)"
|
||||
done
|
||||
else
|
||||
peer_ssh "$ip" 'for f in ~/spinifex/logs/*.log; do
|
||||
[ -f "$f" ] || continue
|
||||
echo "--- $(basename "$f") (last 50 lines) ---"
|
||||
tail -50 "$f" 2>/dev/null || echo "(not found)"
|
||||
done' || echo "(node unreachable)"
|
||||
fi
|
||||
for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
|
||||
spinifex-daemon spinifex-awsgw spinifex-vpcd; do
|
||||
echo "--- ${svc} (last 50 lines) ---"
|
||||
if [ "$ip" = "$LOCAL_IP" ]; then
|
||||
sudo journalctl -u "$svc" --no-pager -n 50 2>/dev/null || echo "(not found)"
|
||||
else
|
||||
peer_ssh "$ip" "sudo journalctl -u $svc --no-pager -n 50" 2>/dev/null || echo "(node unreachable)"
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
@@ -403,7 +394,7 @@ pass_test "Daemon readiness"
|
||||
|
||||
# Spinifex CLI: get nodes
|
||||
echo "Checking spx get nodes..."
|
||||
GET_NODES_OUTPUT=$($SPINIFEX_BIN get nodes --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null)
|
||||
GET_NODES_OUTPUT=$($SPINIFEX_BIN get nodes --timeout 5s 2>/dev/null)
|
||||
echo "$GET_NODES_OUTPUT"
|
||||
READY_COUNT=$(echo "$GET_NODES_OUTPUT" | grep -c "Ready" || true)
|
||||
if [ "$READY_COUNT" -ge 3 ]; then
|
||||
@@ -415,7 +406,7 @@ fi
|
||||
|
||||
# Spinifex CLI: get vms (should be empty)
|
||||
echo "Checking spx get vms (empty)..."
|
||||
GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null)
|
||||
GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null)
|
||||
echo "$GET_VMS_OUTPUT"
|
||||
pass_test "spx get vms (empty)"
|
||||
|
||||
@@ -500,7 +491,7 @@ fi
|
||||
# Verify spx get vms shows all instances
|
||||
echo ""
|
||||
echo "Verifying spx get vms..."
|
||||
GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null)
|
||||
GET_VMS_OUTPUT=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null)
|
||||
echo "$GET_VMS_OUTPUT"
|
||||
for instance_id in "${INSTANCE_IDS[@]}"; do
|
||||
if ! echo "$GET_VMS_OUTPUT" | grep -q "$instance_id"; then
|
||||
@@ -787,8 +778,8 @@ echo "Stopping services on node2 ($NODE2_IP) to simulate node failure..."
|
||||
|
||||
# Stop services on node2 only — SPINIFEX_FORCE_LOCAL_STOP prevents coordinated
|
||||
# cluster shutdown which would kill all nodes via NATS
|
||||
peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && SPINIFEX_FORCE_LOCAL_STOP=1 ./scripts/stop-dev.sh" || {
|
||||
echo " WARNING: stop-dev.sh returned non-zero (may be expected)"
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl stop spinifex.target" || {
|
||||
echo " WARNING: systemctl stop returned non-zero (may be expected)"
|
||||
}
|
||||
|
||||
# Wait for NATS cluster to detect the failure and reform
|
||||
@@ -848,7 +839,7 @@ echo "Phase 9: Node Recovery"
|
||||
echo "========================================"
|
||||
echo "Restarting services on node2 ($NODE2_IP)..."
|
||||
|
||||
peer_ssh "$NODE2_IP" "cd ~/Development/mulga/spinifex && ./scripts/start-dev.sh" || {
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl start spinifex.target" || {
|
||||
echo " ERROR: Failed to restart services on node2"
|
||||
fail_test "Node2 restart"
|
||||
}
|
||||
@@ -899,7 +890,7 @@ fi
|
||||
|
||||
# Verify spx get nodes shows 3 Ready again
|
||||
echo " Checking spx get nodes after recovery..."
|
||||
GET_NODES_RECOVER=$($SPINIFEX_BIN get nodes --config "$SPINIFEX_CONFIG" --timeout 10s 2>/dev/null || echo "")
|
||||
GET_NODES_RECOVER=$($SPINIFEX_BIN get nodes --timeout 10s 2>/dev/null || echo "")
|
||||
echo "$GET_NODES_RECOVER"
|
||||
READY_RECOVER=$(echo "$GET_NODES_RECOVER" | grep -c "Ready" || true)
|
||||
if [ "$READY_RECOVER" -ge 3 ]; then
|
||||
@@ -1252,7 +1243,7 @@ else
|
||||
echo "Step 4: Validating spread placement across nodes..."
|
||||
|
||||
# Use spx get vms to check node assignment
|
||||
SPX_VMS=$($SPINIFEX_BIN get vms --config "$SPINIFEX_CONFIG" --timeout 5s 2>/dev/null)
|
||||
SPX_VMS=$($SPINIFEX_BIN get vms --timeout 5s 2>/dev/null)
|
||||
echo "$SPX_VMS"
|
||||
|
||||
SPREAD_NODES=()
|
||||
|
||||
Reference in New Issue
Block a user