fix: use binary install for single node e2e (#111)

This commit is contained in:
Julian Sommer
2026-04-02 15:22:41 +10:00
committed by GitHub
parent e5b6204875
commit dda4575888
7 changed files with 93 additions and 152 deletions
+12 -7
View File
@@ -48,9 +48,13 @@ jobs:
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Clone and build
- name: Build install artifacts
working-directory: mulga/scripts/tofu-cluster
run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
@@ -62,13 +66,14 @@ jobs:
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap Cluster
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_PREBUILT=true \
SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
./bootstrap-install.sh env3 \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
@@ -78,7 +83,7 @@ jobs:
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh && ./tests/e2e/run-cert-e2e.sh'
'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh'
- name: Teardown
if: always()
-15
View File
@@ -161,21 +161,6 @@ jobs:
TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
- name: Run Smoke Test
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-i "$SSH_KEY" \
smoke-test-install.sh \
"tf-user@${WAN_IP}:/tmp/smoke-test.sh"
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-i "$SSH_KEY" "tf-user@${WAN_IP}" \
'bash /tmp/smoke-test.sh'
- name: Promote Release
if: success()
env:
+8 -2
View File
@@ -76,9 +76,15 @@ func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
}
// Backfill BaseDir from config file path when not set in the config.
// Config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" {
nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
if isProductionLayout() {
// Production: BaseDir is /var/lib/spinifex (data dir).
// /var/lib/spinifex/config is a symlink to /etc/spinifex.
nodeConfig.BaseDir = DefaultDataDir()
} else {
// Dev: config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
}
cfg.Nodes[cfg.Node] = nodeConfig
}
+6 -7
View File
@@ -7,7 +7,7 @@ import (
)
// DefaultConfigDir returns the default configuration directory.
// Production: /etc/spinifex (when running as root AND /etc/spinifex exists)
// Production: /etc/spinifex (when /etc/spinifex exists)
// Development: ~/spinifex/config
func DefaultConfigDir() string {
if isProductionLayout() {
@@ -45,13 +45,12 @@ func DefaultConfigFile() string {
}
// isProductionLayout returns true when running in a production install.
// Detected by: running as root AND /etc/spinifex directory already exists
// (created by setup.sh during binary install).
// Detected by: /etc/spinifex directory exists (created by setup.sh during
// binary install). No root check — allows non-root users (e.g. tf-user) to
// run CLI commands like `spx get nodes` without sudo or --config flags.
func isProductionLayout() bool {
if os.Getuid() == 0 {
if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
return true
}
if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
return true
}
return false
}
+11 -11
View File
@@ -7,9 +7,9 @@ import (
)
func TestDefaultConfigDir_Dev(t *testing.T) {
// Non-root user without /etc/spinifex should get dev paths
if os.Getuid() == 0 {
t.Skip("test requires non-root user")
// Without /etc/spinifex should get dev paths
if _, err := os.Stat("/etc/spinifex"); err == nil {
t.Skip("test requires /etc/spinifex to not exist")
}
homeDir, _ := os.UserHomeDir()
@@ -21,8 +21,8 @@ func TestDefaultConfigDir_Dev(t *testing.T) {
}
func TestDefaultDataDir_Dev(t *testing.T) {
if os.Getuid() == 0 {
t.Skip("test requires non-root user")
if _, err := os.Stat("/etc/spinifex"); err == nil {
t.Skip("test requires /etc/spinifex to not exist")
}
homeDir, _ := os.UserHomeDir()
@@ -34,8 +34,8 @@ func TestDefaultDataDir_Dev(t *testing.T) {
}
func TestDefaultConfigFile(t *testing.T) {
if os.Getuid() == 0 {
t.Skip("test requires non-root user")
if _, err := os.Stat("/etc/spinifex"); err == nil {
t.Skip("test requires /etc/spinifex to not exist")
}
got := DefaultConfigFile()
@@ -46,12 +46,12 @@ func TestDefaultConfigFile(t *testing.T) {
}
}
func TestIsProductionLayout_NonRoot(t *testing.T) {
if os.Getuid() == 0 {
t.Skip("test requires non-root user")
func TestIsProductionLayout_NoEtcSpinifex(t *testing.T) {
if _, err := os.Stat("/etc/spinifex"); err == nil {
t.Skip("test requires /etc/spinifex to not exist")
}
if isProductionLayout() {
t.Error("isProductionLayout() = true for non-root user, want false")
t.Error("isProductionLayout() = true without /etc/spinifex, want false")
}
}
+17 -5
View File
@@ -56,8 +56,12 @@ case "${1:-}" in
done < <(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+')
# SERVICE_IPS: only IPs where awsgw actually listens (bind IP from config).
# awsgw binds to the host specified in spinifex.toml, not all interfaces.
if [ -f "$HOME/spinifex/config/spinifex.toml" ]; then
AWSGW_BIND=$(grep -A5 '\.awsgw\]' "$HOME/spinifex/config/spinifex.toml" | grep 'host' | head -1 | sed 's/.*= *"\([^:]*\).*/\1/')
CONFIG_FILE="/etc/spinifex/spinifex.toml"
if [ ! -f "$CONFIG_FILE" ]; then
CONFIG_FILE="$HOME/spinifex/config/spinifex.toml"
fi
if [ -f "$CONFIG_FILE" ]; then
AWSGW_BIND=$(grep -A5 '\.awsgw\]' "$CONFIG_FILE" | grep 'host' | head -1 | sed 's/.*= *"\([^:]*\).*/\1/')
fi
if [ -n "${AWSGW_BIND:-}" ] && [ "$AWSGW_BIND" != "0.0.0.0" ]; then
SERVICE_IPS=("$AWSGW_BIND")
@@ -89,6 +93,7 @@ fail() {
# Resolve the Spinifex config CA cert path (not the system copy).
resolve_ca_cert() {
for candidate in \
"/etc/spinifex/ca.pem" \
"$HOME/spinifex/config/ca.pem" \
"$HOME/node1/config/ca.pem"; do
if [ -f "$candidate" ]; then
@@ -109,6 +114,13 @@ fi
SYSTEM_CA_PATH="/usr/local/share/ca-certificates/spinifex-ca.crt"
# Resolve single-node config dir (production or dev)
if [ -d /etc/spinifex ]; then
SINGLE_CONFIG_DIR="/etc/spinifex"
else
SINGLE_CONFIG_DIR="$HOME/spinifex/config"
fi
echo "Using CA cert: $CA_CERT"
echo "Mode: $MODE"
echo "All IPs (SAN check): ${NODE_IPS[*]}"
@@ -123,7 +135,7 @@ for ip in "${NODE_IPS[@]}"; do
# Determine which node's cert to check.
case "$MODE" in
single)
CERT_PATH="$HOME/spinifex/config/server.pem"
CERT_PATH="$SINGLE_CONFIG_DIR/server.pem"
;;
pseudo)
# Each pseudo node has its own config dir.
@@ -166,7 +178,7 @@ HOSTNAME=$(hostname)
if [ "$HOSTNAME" != "localhost" ] && [ -n "$HOSTNAME" ]; then
case "$MODE" in
single)
CERT_PATH="$HOME/spinifex/config/server.pem"
CERT_PATH="$SINGLE_CONFIG_DIR/server.pem"
;;
pseudo)
CERT_PATH="$HOME/node1/config/server.pem"
@@ -298,7 +310,7 @@ echo ""
echo "=== Test 7: Certificate metadata ==="
case "$MODE" in
single) CERT_PATH="$HOME/spinifex/config/server.pem" ;;
single) CERT_PATH="$SINGLE_CONFIG_DIR/server.pem" ;;
pseudo) CERT_PATH="$HOME/node1/config/server.pem" ;;
*) CERT_PATH="" ;;
esac
+39 -105
View File
@@ -1,53 +1,25 @@
#!/bin/bash
# run-e2e.sh — Single-node E2E test suite (production binary install).
#
# Assumes services are already running via systemd (bootstrap-install.sh).
# Uses production paths: /etc/spinifex, /var/log/spinifex, spx from PATH.
set -e
# Ensure Go is on PATH (SSH non-interactive shells don't source .bashrc)
export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
# Ensure we are in the project root
cd "$(dirname "$0")/../.."
# Run from the directory containing this script (~/e2e-tests/)
cd "$(dirname "$0")"
# Source helper functions
source ./tests/e2e/lib/multinode-helpers.sh
source ./lib/multinode-helpers.sh
# Ensure services are stopped on exit and print logs on failure
# Dump journald logs on failure
cleanup() {
EXIT_CODE=$?
if [ $EXIT_CODE -ne 0 ]; then
echo ""
echo "=== NATS Service Log ==="
if [ -f ~/spinifex/logs/nats.log ]; then
tail -100 ~/spinifex/logs/nats.log 2>/dev/null
fi
echo ""
echo "=== Predastore Service Log ==="
if [ -f ~/spinifex/logs/predastore.log ]; then
tail -100 ~/spinifex/logs/predastore.log 2>/dev/null
fi
echo ""
echo "=== Viperblock Service Log ==="
if [ -f ~/spinifex/logs/viperblock.log ]; then
tail -100 ~/spinifex/logs/viperblock.log 2>/dev/null
fi
echo ""
echo "=== Spinifex Daemon Log ==="
if [ -f ~/spinifex/logs/spinifex.log ]; then
tail -200 ~/spinifex/logs/spinifex.log 2>/dev/null
fi
echo ""
echo "=== AWS Gateway Log ==="
if [ -f ~/spinifex/logs/awsgw.log ]; then
tail -200 ~/spinifex/logs/awsgw.log 2>/dev/null
fi
echo ""
echo "=== vpcd Log ==="
if [ -f ~/spinifex/logs/vpcd.log ]; then
tail -200 ~/spinifex/logs/vpcd.log 2>/dev/null
fi
fi
# Only stop services if we started them (not when bootstrapped)
if [ "$BOOTSTRAPPED" != "true" ]; then
./scripts/stop-dev.sh
for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
spinifex-daemon spinifex-awsgw spinifex-vpcd; do
echo "=== ${svc} ==="
sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true
done
fi
exit $EXIT_CODE
}
@@ -56,30 +28,22 @@ trap cleanup EXIT
# Use Spinifex profile (spx admin init sets endpoint_url, ca_bundle, region in ~/.aws/config)
export AWS_PROFILE=spinifex
# Detect bootstrapped environment from spinifex.toml config
BOOTSTRAPPED="false"
# Resolve gateway and predastore hosts from config
GATEWAY_HOST="localhost"
PREDASTORE_HOST="localhost"
if [ -f ~/spinifex/config/spinifex.toml ]; then
# Resolve gateway host from AWS profile endpoint_url (set by spx admin init)
ENDPOINT_URL=$(aws configure get endpoint_url 2>/dev/null || true)
if [ -n "$ENDPOINT_URL" ]; then
DETECTED_GW_HOST=$(echo "$ENDPOINT_URL" | sed 's|https\?://||;s|:.*||')
if [ -n "$DETECTED_GW_HOST" ]; then
GATEWAY_HOST="$DETECTED_GW_HOST"
fi
ENDPOINT_URL=$(aws configure get endpoint_url 2>/dev/null || true)
if [ -n "$ENDPOINT_URL" ]; then
DETECTED_GW_HOST=$(echo "$ENDPOINT_URL" | sed 's|https\?://||;s|:.*||')
if [ -n "$DETECTED_GW_HOST" ]; then
GATEWAY_HOST="$DETECTED_GW_HOST"
fi
# Resolve predastore host from spinifex.toml
DETECTED_PS_HOST=$(awk -F'"' '/\[nodes\..*\.predastore\]/{found=1} found && /^host/{print $2; exit}' ~/spinifex/config/spinifex.toml)
if [ -n "$DETECTED_PS_HOST" ]; then
PREDASTORE_HOST="${DETECTED_PS_HOST%%:*}"
fi
# Check if gateway is actually responding
if curl -sk "https://${GATEWAY_HOST}:9999" > /dev/null 2>&1; then
BOOTSTRAPPED="true"
echo "Detected bootstrapped environment — skipping cluster setup"
echo " Gateway host: $GATEWAY_HOST"
echo " Predastore host: $PREDASTORE_HOST"
fi
DETECTED_PS_HOST=$(awk -F'"' '/\[nodes\..*\.predastore\]/{found=1} found && /^host/{print $2; exit}' /etc/spinifex/spinifex.toml)
if [ -n "$DETECTED_PS_HOST" ]; then
PS_IP="${DETECTED_PS_HOST%%:*}"
# 0.0.0.0 means "all interfaces" — use localhost for S3 client calls
if [ "$PS_IP" != "0.0.0.0" ]; then
PREDASTORE_HOST="$PS_IP"
fi
fi
@@ -96,7 +60,7 @@ setup_test_profile() {
# Phase 1: Environment Setup
echo "Phase 1: Environment Setup"
# Check for KVM support inside the container
# Check for KVM support
echo "Checking for KVM support..."
if [ -e /dev/kvm ]; then
echo "✅ /dev/kvm exists"
@@ -107,28 +71,10 @@ if [ -e /dev/kvm ]; then
exit 1
fi
else
echo "❌ /dev/kvm does NOT exist. Ensure --privileged and -v /dev/kvm:/dev/kvm are used."
echo "❌ /dev/kvm does NOT exist."
exit 1
fi
if [ "$BOOTSTRAPPED" != "true" ]; then
INIT_FLAGS="--region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
if [ -n "${SPX_EXTERNAL_POOL:-}" ]; then
INIT_FLAGS="$INIT_FLAGS --external-mode=pool --external-pool=$SPX_EXTERNAL_POOL"
fi
./bin/spx admin init $INIT_FLAGS
# Trust the Spinifex CA certificate for AWS CLI SSL verification
echo "Adding Spinifex CA certificate to system trust store..."
sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
sudo update-ca-certificates
# Start all services
# Ensure logs directory exists for start-dev.sh
mkdir -p ~/spinifex/logs
./scripts/start-dev.sh
fi
# Wait for health checks on AWS Gateway
echo "Waiting for AWS Gateway..."
MAX_RETRIES=30
@@ -160,7 +106,7 @@ echo "Phase 1b: Cluster Stats CLI"
# Test spx get nodes — should show node1 as Ready
echo "Testing spx get nodes..."
GET_NODES_OUTPUT=$(./bin/spx get nodes --config ~/spinifex/config/spinifex.toml --timeout 5s 2>/dev/null)
GET_NODES_OUTPUT=$(spx get nodes --timeout 5s 2>/dev/null)
echo "$GET_NODES_OUTPUT"
if ! echo "$GET_NODES_OUTPUT" | grep -q "Ready"; then
echo "spx get nodes did not show any Ready nodes"
@@ -170,7 +116,7 @@ echo "spx get nodes passed"
# Test spx top nodes — should show CPU/MEM stats
echo "Testing spx top nodes..."
TOP_NODES_OUTPUT=$(./bin/spx top nodes --config ~/spinifex/config/spinifex.toml --timeout 5s 2>/dev/null)
TOP_NODES_OUTPUT=$(spx top nodes --timeout 5s 2>/dev/null)
echo "$TOP_NODES_OUTPUT"
if ! echo "$TOP_NODES_OUTPUT" | grep -q "0/"; then
echo "spx top nodes did not show resource stats"
@@ -180,7 +126,7 @@ echo "spx top nodes passed"
# Test spx get vms — should show no VMs yet
echo "Testing spx get vms (empty)..."
GET_VMS_OUTPUT=$(./bin/spx get vms --config ~/spinifex/config/spinifex.toml --timeout 5s 2>/dev/null)
GET_VMS_OUTPUT=$(spx get vms --timeout 5s 2>/dev/null)
echo "$GET_VMS_OUTPUT"
if ! echo "$GET_VMS_OUTPUT" | grep -q "No VMs found"; then
echo "spx get vms should show 'No VMs found' before any launches"
@@ -303,21 +249,9 @@ else
fi
echo "Using image: $IMAGE_NAME"
# Import the pre-downloaded Ubuntu image using file-based import
# When bootstrapped, the image is already imported — discover the existing AMI
if [ "$BOOTSTRAPPED" = "true" ]; then
echo "Discovering existing AMI (imported by bootstrap)..."
AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
else
echo "Importing pre-cached Ubuntu image..."
IMPORT_LOG=$(./bin/spx admin images import \
--file ~/images/ubuntu-24.04.img \
--arch "$ARCH" \
--distro ubuntu \
--version 24.04 \
--force 2>/dev/null)
AMI_ID=$(echo "$IMPORT_LOG" | grep -o 'ami-[a-z0-9]\+')
fi
# AMI already imported by bootstrap-install.sh — discover the existing AMI
echo "Discovering existing AMI (imported by bootstrap)..."
AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
echo "Failed to capture AMI ID"
@@ -395,7 +329,7 @@ fi
# Phase 5a-pre: Verify spx get vms shows running instance
echo "Phase 5a-pre: Cluster Stats CLI (with running VM)"
GET_VMS_OUTPUT=$(./bin/spx get vms --config ~/spinifex/config/spinifex.toml --timeout 5s 2>/dev/null)
GET_VMS_OUTPUT=$(spx get vms --timeout 5s 2>/dev/null)
echo "$GET_VMS_OUTPUT"
if ! echo "$GET_VMS_OUTPUT" | grep -q "$INSTANCE_ID"; then
echo "spx get vms did not show running instance $INSTANCE_ID"
@@ -1901,7 +1835,7 @@ echo "Phase 8 Step 1: Account Setup"
echo "----------------------------------------"
echo " Creating Team Alpha account..."
ALPHA_OUTPUT=$(./bin/spx admin account create --name "Team Alpha" 2>&1)
ALPHA_OUTPUT=$(spx admin account create --name "Team Alpha" 2>&1)
echo "$ALPHA_OUTPUT"
ALPHA_ACCOUNT=$(echo "$ALPHA_OUTPUT" | grep "Account ID:" | awk '{print $NF}')
ALPHA_KEY_ID=$(echo "$ALPHA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}')
@@ -1915,7 +1849,7 @@ echo " Team Alpha: account=$ALPHA_ACCOUNT key=$ALPHA_KEY_ID"
setup_test_profile spx-team-alpha "$ALPHA_KEY_ID" "$ALPHA_SECRET"
echo " Creating Team Beta account..."
BETA_OUTPUT=$(./bin/spx admin account create --name "Team Beta" 2>&1)
BETA_OUTPUT=$(spx admin account create --name "Team Beta" 2>&1)
echo "$BETA_OUTPUT"
BETA_ACCOUNT=$(echo "$BETA_OUTPUT" | grep "Account ID:" | awk '{print $NF}')
BETA_KEY_ID=$(echo "$BETA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}')
@@ -2450,7 +2384,7 @@ echo "----------------------------------------"
# Empty account (Gamma)
echo " Creating empty Gamma account..."
GAMMA_OUTPUT=$(./bin/spx admin account create --name "Team Gamma" 2>&1)
GAMMA_OUTPUT=$(spx admin account create --name "Team Gamma" 2>&1)
GAMMA_KEY_ID=$(echo "$GAMMA_OUTPUT" | grep "Access Key ID:" | awk '{print $NF}')
GAMMA_SECRET=$(echo "$GAMMA_OUTPUT" | grep "Secret Access Key:" | awk '{print $NF}')
setup_test_profile spx-team-gamma "$GAMMA_KEY_ID" "$GAMMA_SECRET"
@@ -2723,7 +2657,7 @@ echo "========================================"
# Check if external networking is configured
HAS_EXTERNAL=false
if grep -q 'external_mode = "pool"' ~/spinifex/config/spinifex.toml 2>/dev/null; then
if grep -q 'external_mode = "pool"' /etc/spinifex/spinifex.toml 2>/dev/null; then
HAS_EXTERNAL=true
fi