mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
chore: cleanup stale scripts
This commit is contained in:
@@ -38,14 +38,12 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
|---|---|
|
||||
| `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. |
|
||||
| `verify-network.sh` | Asserts a node's datapath matches its declared planes — EIP reachability, which plane carries Geneve, and which interfaces services bind. Catches what `smoke-test.sh` cannot, since none of its traffic leaves the box. |
|
||||
| `test-ddil-phase1.sh` | systemd-level smoke test for daemon-local-autonomy tier 1. Destructive on the local node: stops and starts `spinifex-nats` and restarts `spinifex-daemon`, restoring both via a cleanup trap. |
|
||||
| `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. |
|
||||
| `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). |
|
||||
| `sync-aws-models.sh` | Replaces the embedded AWS Smithy models under `internal/awsmodel/models/` with the ten service files from `aws/api-models-aws` at the given commit, and records that commit in `internal/awsmodel/model_source.go`. Takes a full 40-character SHA. |
|
||||
| `run-bench.sh` | Launches a benchmark against running Ubuntu instances, auto-detecting the AMI for the host architecture. Tracks nbdkit and perf alongside the run. |
|
||||
| `disk-performance.sh` | Runs a `fio` random 70/30 read-write benchmark swept across 4k/16k/128k/1M. Writes JSON results to `/tmp/spinifex-disk-bench`; override with `BENCH_DIR`, `OUT_DIR`, `SIZE`, `JOBS`, `BLOCK_SIZES`. |
|
||||
| `network-performance.sh` | iperf throughput from several clients to one server, driven over SSH, writing a `summary.txt` of Gbit/s per client. `--server-ip` sets the address clients dial, so pointing it at instance private addresses measures the VPC overlay rather than the external pool. |
|
||||
| `workload-performance.sh` | In-guest memory, disk, CPU and network workload test. Installs Go and runs the Badger test suite with jemalloc disabled and reduced parallelism. |
|
||||
|
||||
## Guest images
|
||||
|
||||
@@ -93,13 +91,6 @@ Installed into the EKS node AMI and run at boot or on a timer. All host communic
|
||||
| `mulga-mgmt-net.sh` | Boot oneshot bringing a multi-NIC system VM's interfaces up by MAC from the QEMU fw_cfg netcfg blob, before cloud-init's network stage. cloud-init cannot reliably pick the right NIC out of two. |
|
||||
| `mulga-vpc-mtu.sh` | Boot oneshot pinning the VPC data NIC MTU so flannel sizes its VXLAN overlay to fit the OVN geneve underlay. Otherwise large frames are silently dropped while handshakes pass. |
|
||||
|
||||
## Launch helpers
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `launch-gpu.sh` | Launches a GPU instance from the AMD or NVIDIA GPU AMI, selecting the AMI from the instance type's GPU brand. `--amd` / `--nvidia` override the detection. |
|
||||
| `ecs-node-bringup.sh` | Launches an ECS container-instance VM. Renders the cloud-init user-data seeding the agent's control-plane config, IAM credentials and gateway CA — there is no ECS-specific launch API, by design. |
|
||||
|
||||
## `demo/` — GPU demo
|
||||
|
||||
| Script | What it does |
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ecs-node-bringup.sh — launch a Spinifex ECS container-instance VM.
|
||||
#
|
||||
# ECS container instances are plain EC2 instances booted from the spinifex-ecs-node
|
||||
# AMI (Alpine + containerd + ecs-agent), AWS-faithful: there is no ECS-specific
|
||||
# launch API. This script renders the cloud-init user-data that seeds the agent's
|
||||
# control-plane config + IAM credentials into /etc/spinifex-ecs/agent.env and the
|
||||
# gateway TLS CA into /etc/spinifex-ecs/gateway-ca.pem, then runs the instance.
|
||||
#
|
||||
# The agent then registers, heartbeats, polls assignments, and reports task state
|
||||
# entirely through the AWS gateway over TLS+SigV4 — it never touches NATS.
|
||||
#
|
||||
# Usage:
|
||||
# AWS_PROFILE=spinifex scripts/ecs-node-bringup.sh <cluster-name>
|
||||
#
|
||||
# Env overrides:
|
||||
# GATEWAY_IP gateway IP the guest dials (default: host WAN-bridge IP)
|
||||
# GATEWAY_CA host path to the gateway CA PEM (default /etc/spinifex/ca.pem)
|
||||
# AMI_ID ECS-node AMI (default: latest spinifex-ecs-node)
|
||||
# SUBNET_ID / SG_ID / KEY_NAME / INSTANCE_TYPE (default: first available / t3.small)
|
||||
set -euo pipefail
|
||||
|
||||
CLUSTER="${1:?usage: ecs-node-bringup.sh <cluster-name>}"
|
||||
PROFILE="${AWS_PROFILE:-spinifex}"
|
||||
REGION="$(aws configure get region --profile "$PROFILE")"
|
||||
|
||||
# VPC guests NAT out through the WAN and reach host-run services on the host's
|
||||
# WAN-bridge IP; the mgmt bridge (10.15.8.x) is not routable from a guest netns.
|
||||
# Default to the br-wan address, falling back to the mgmt IP if it can't be read.
|
||||
default_gateway_ip() {
|
||||
ip -4 addr show br-wan 2>/dev/null | awk '/inet /{print $2; exit}' | cut -d/ -f1
|
||||
}
|
||||
GATEWAY_IP="${GATEWAY_IP:-$(default_gateway_ip)}"
|
||||
GATEWAY_IP="${GATEWAY_IP:-10.15.8.1}"
|
||||
GATEWAY_CA="${GATEWAY_CA:-/etc/spinifex/ca.pem}"
|
||||
INSTANCE_TYPE="${INSTANCE_TYPE:-t3.small}"
|
||||
|
||||
awscli() { aws --profile "$PROFILE" "$@"; }
|
||||
|
||||
AMI_ID="${AMI_ID:-$(awscli ec2 describe-images \
|
||||
--filters 'Name=name,Values=spinifex-ecs-node' \
|
||||
--query 'sort_by(Images,&CreationDate)[-1].ImageId' --output text)}"
|
||||
SUBNET_ID="${SUBNET_ID:-$(awscli ec2 describe-subnets \
|
||||
--query 'Subnets[0].SubnetId' --output text)}"
|
||||
SG_ID="${SG_ID:-$(awscli ec2 describe-security-groups \
|
||||
--query 'SecurityGroups[0].GroupId' --output text)}"
|
||||
KEY_NAME="${KEY_NAME:-$(awscli ec2 describe-key-pairs \
|
||||
--query 'KeyPairs[0].KeyName' --output text)}"
|
||||
|
||||
ACCESS_KEY="$(aws configure get aws_access_key_id --profile "$PROFILE")"
|
||||
SECRET_KEY="$(aws configure get aws_secret_access_key --profile "$PROFILE")"
|
||||
|
||||
echo "cluster=$CLUSTER ami=$AMI_ID subnet=$SUBNET_ID sg=$SG_ID key=$KEY_NAME gw=https://$GATEWAY_IP:9999" >&2
|
||||
|
||||
# Indent a file's content by six spaces for a cloud-init "content: |" block.
|
||||
indent6() { sed 's/^/ /' "$1"; }
|
||||
|
||||
USERDATA="$(cat <<EOF
|
||||
#cloud-config
|
||||
bootcmd:
|
||||
- rm -f /etc/resolv.conf
|
||||
- printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > /etc/resolv.conf
|
||||
write_files:
|
||||
- path: /etc/spinifex-ecs/agent.env
|
||||
permissions: '0600'
|
||||
content: |
|
||||
ECS_GATEWAY_URL=https://$GATEWAY_IP:9999
|
||||
ECS_GATEWAY_CA=/etc/spinifex-ecs/gateway-ca.pem
|
||||
ECS_REGION=$REGION
|
||||
ECS_CLUSTER=$CLUSTER
|
||||
ECS_ACCESS_KEY=$ACCESS_KEY
|
||||
ECS_SECRET_KEY=$SECRET_KEY
|
||||
- path: /etc/spinifex-ecs/gateway-ca.pem
|
||||
permissions: '0644'
|
||||
content: |
|
||||
$(indent6 "$GATEWAY_CA")
|
||||
EOF
|
||||
)"
|
||||
|
||||
INSTANCE_ID="$(awscli ec2 run-instances \
|
||||
--image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" \
|
||||
--key-name "$KEY_NAME" --subnet-id "$SUBNET_ID" --security-group-ids "$SG_ID" \
|
||||
--user-data "$USERDATA" \
|
||||
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=ecs-node-$CLUSTER}]" \
|
||||
--query 'Instances[0].InstanceId' --output text)"
|
||||
|
||||
echo "$INSTANCE_ID"
|
||||
@@ -1,248 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export AWS_PROFILE=spinifex
|
||||
|
||||
# Resolve the actual user's home — $HOME is /root when run under sudo.
|
||||
_USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6)
|
||||
SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}"
|
||||
SSH_USER="${SSH_USER:-ubuntu}"
|
||||
SSH_TIMEOUT="${SSH_TIMEOUT:-300}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
launch-gpu.sh — Launch a GPU instance from the Spinifex AMD or NVIDIA GPU AMI.
|
||||
|
||||
Detects the GPU brand from the specified instance type and selects the matching
|
||||
pre-built AMI automatically. Use --amd or --nvidia to override.
|
||||
|
||||
Usage:
|
||||
scripts/launch-gpu.sh --type <instance-type> [--disk GB] [--amd|--nvidia]
|
||||
|
||||
Options:
|
||||
--type TYPE EC2 instance type to launch (required)
|
||||
--disk GB Root disk size in GB (default: 50)
|
||||
--ami NAME Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-26.04-nvidia-gpu-x86_64)
|
||||
--amd Force AMD GPU AMI (ubuntu-amd-gpu)
|
||||
--nvidia Force NVIDIA GPU AMI (ubuntu-26.04-nvidia-gpu-x86_64)
|
||||
|
||||
Env overrides:
|
||||
SSH_KEY Path to SSH private key (default: ~/.ssh/spinifex-key)
|
||||
SSH_USER SSH user inside guest (default: ubuntu)
|
||||
SSH_TIMEOUT Seconds to wait for initial SSH (default: 300)
|
||||
EOF
|
||||
}
|
||||
|
||||
INSTANCE_TYPE=""
|
||||
DISK_GB=50
|
||||
GPU_BRAND=""
|
||||
AMI_OVERRIDE=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--type) INSTANCE_TYPE="$2"; shift 2 ;;
|
||||
--disk) DISK_GB="$2"; shift 2 ;;
|
||||
--ami) AMI_OVERRIDE="$2"; shift 2 ;;
|
||||
--amd) GPU_BRAND="amd"; shift ;;
|
||||
--nvidia) GPU_BRAND="nvidia"; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1"; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
list_gpu_types() {
|
||||
aws ec2 describe-instance-types \
|
||||
--query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]' \
|
||||
--output table 2>/dev/null || true
|
||||
}
|
||||
|
||||
if [ -z "$INSTANCE_TYPE" ]; then
|
||||
echo "Error: --type is required."
|
||||
echo ""
|
||||
echo "Available GPU instance types on this node:"
|
||||
list_gpu_types
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Validate instance type and fetch GPU info in one call ---
|
||||
echo "==> Checking instance type: ${INSTANCE_TYPE}"
|
||||
INSTANCE_INFO=$(aws ec2 describe-instance-types \
|
||||
--query "InstanceTypes[?InstanceType=='${INSTANCE_TYPE}' && GpuInfo].[GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]" \
|
||||
--output text 2>/dev/null || true)
|
||||
|
||||
if [ -z "$INSTANCE_INFO" ] || [ "$INSTANCE_INFO" = "None" ]; then
|
||||
echo "'${INSTANCE_TYPE}' is not available or is not a GPU type on this node."
|
||||
echo ""
|
||||
echo "Available GPU instance types:"
|
||||
list_gpu_types
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IFS=$'\t' read -r GPU_COUNT GPU_MANUFACTURER GPU_NAME <<< "$INSTANCE_INFO"
|
||||
echo " ${GPU_COUNT}× ${GPU_NAME} (${GPU_MANUFACTURER})"
|
||||
|
||||
# --- Resolve GPU brand ---
|
||||
if [ -z "$GPU_BRAND" ]; then
|
||||
case "$GPU_MANUFACTURER" in
|
||||
AMD|amd) GPU_BRAND="amd" ;;
|
||||
NVIDIA|nvidia) GPU_BRAND="nvidia" ;;
|
||||
*)
|
||||
echo "Cannot auto-detect GPU brand from manufacturer '${GPU_MANUFACTURER}'."
|
||||
echo "Use --amd or --nvidia to specify explicitly."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "$GPU_BRAND" in
|
||||
amd) AMI_NAME="ubuntu-amd-gpu" ;;
|
||||
nvidia) AMI_NAME="ubuntu-26.04-nvidia-gpu-x86_64" ;;
|
||||
esac
|
||||
|
||||
# --- Resolve AMI ---
|
||||
AMI_LOOKUP="${AMI_OVERRIDE:-${AMI_NAME}}"
|
||||
echo "==> Resolving AMI: ${AMI_LOOKUP}"
|
||||
AMI_ID=$(aws ec2 describe-images \
|
||||
--query "Images[?Name=='${AMI_LOOKUP}'].ImageId | [0]" --output text 2>/dev/null || true)
|
||||
if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
|
||||
echo "No AMI found with name '${AMI_LOOKUP}'."
|
||||
echo "Build and import it first, or specify the AMI name with --ami <name>"
|
||||
echo " scripts/build-system-image.sh scripts/images/ubuntu-gpu-${GPU_BRAND}.conf --import"
|
||||
exit 1
|
||||
fi
|
||||
echo " $AMI_ID"
|
||||
|
||||
# --- Resolve subnet ---
|
||||
echo "==> Resolving subnet"
|
||||
SUBNET_ID=$(aws ec2 describe-subnets \
|
||||
--query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text 2>/dev/null || true)
|
||||
if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
|
||||
echo "No public subnet found"
|
||||
exit 1
|
||||
fi
|
||||
echo " $SUBNET_ID"
|
||||
|
||||
VPC_ID=$(aws ec2 describe-subnets --subnet-ids "$SUBNET_ID" \
|
||||
--query 'Subnets[0].VpcId' --output text 2>/dev/null || true)
|
||||
if [ -z "$VPC_ID" ] || [ "$VPC_ID" = "None" ]; then
|
||||
echo "Could not resolve VPC for subnet $SUBNET_ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Ensure SSH security group ---
|
||||
# The default VPC SG only allows intra-SG traffic; we need a group that
|
||||
# permits port 22 from anywhere so the SSH wait below can connect.
|
||||
SSH_SG_NAME="spinifex-gpu-ssh"
|
||||
echo "==> Ensuring SSH security group (${SSH_SG_NAME})"
|
||||
SG_ID=$(aws ec2 describe-security-groups \
|
||||
--filters "Name=group-name,Values=${SSH_SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" \
|
||||
--query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)
|
||||
if [ -z "$SG_ID" ] || [ "$SG_ID" = "None" ]; then
|
||||
SG_ID=$(aws ec2 create-security-group \
|
||||
--group-name "$SSH_SG_NAME" \
|
||||
--description "Allow SSH access to GPU instances" \
|
||||
--vpc-id "$VPC_ID" \
|
||||
--query 'GroupId' --output text)
|
||||
aws ec2 authorize-security-group-ingress \
|
||||
--group-id "$SG_ID" \
|
||||
--protocol tcp \
|
||||
--port 22 \
|
||||
--cidr 0.0.0.0/0 >/dev/null
|
||||
echo " Created $SG_ID"
|
||||
else
|
||||
echo " Reusing $SG_ID"
|
||||
fi
|
||||
|
||||
# --- Ensure SSH key ---
|
||||
if [ ! -f "$SSH_KEY" ]; then
|
||||
echo "==> Generating SSH key"
|
||||
sudo -u "${SUDO_USER:-$(id -un)}" mkdir -p "$(dirname "$SSH_KEY")"
|
||||
sudo -u "${SUDO_USER:-$(id -un)}" ssh-keygen -t ed25519 -f "$SSH_KEY" -N ""
|
||||
fi
|
||||
# Delete before import — stale key pair material causes import to fail
|
||||
# silently, leaving the instance unreachable via SSH.
|
||||
echo "==> Importing SSH key"
|
||||
aws ec2 delete-key-pair --key-name spinifex-key >/dev/null 2>&1 || true
|
||||
aws ec2 import-key-pair --key-name spinifex-key \
|
||||
--public-key-material "fileb://${SSH_KEY}.pub" >/dev/null 2>&1 || true
|
||||
|
||||
# --- Launch ---
|
||||
echo "==> Launching ${INSTANCE_TYPE} (${DISK_GB} GB)..."
|
||||
ID=$(aws ec2 run-instances \
|
||||
--image-id "$AMI_ID" \
|
||||
--instance-type "$INSTANCE_TYPE" \
|
||||
--key-name spinifex-key \
|
||||
--subnet-id "$SUBNET_ID" \
|
||||
--security-group-ids "$SG_ID" \
|
||||
--count 1 \
|
||||
--block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":${DISK_GB},\"VolumeType\":\"gp3\",\"DeleteOnTermination\":true}}]" \
|
||||
--query 'Instances[0].InstanceId' --output text)
|
||||
if [ -z "$ID" ] || [ "$ID" = "None" ] || [ "$ID" = "null" ]; then
|
||||
echo "run-instances returned no InstanceId"
|
||||
exit 1
|
||||
fi
|
||||
echo " $ID"
|
||||
|
||||
# --- Wait for running ---
|
||||
echo "==> Waiting for running state..."
|
||||
COUNT=0; STATE="unknown"
|
||||
while [ $COUNT -lt 120 ]; do
|
||||
STATE=$(aws ec2 describe-instances --instance-ids "$ID" \
|
||||
--query "Reservations[0].Instances[0].State.Name" --output text 2>/dev/null || echo "unknown")
|
||||
[ "$STATE" = "running" ] && break
|
||||
[ "$STATE" = "terminated" ] && { echo "Instance terminated unexpectedly"; exit 1; }
|
||||
sleep 2; COUNT=$((COUNT + 2))
|
||||
done
|
||||
[ "$STATE" = "running" ] || { echo "Instance did not reach running state (last: $STATE)"; exit 1; }
|
||||
|
||||
# --- Wait for public IP ---
|
||||
echo "==> Waiting for public IP..."
|
||||
IP=""
|
||||
for _i in $(seq 1 120); do
|
||||
IP=$(aws ec2 describe-instances --instance-ids "$ID" \
|
||||
--query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true)
|
||||
[ -n "$IP" ] && [ "$IP" != "None" ] && [ "$IP" != "null" ] && break
|
||||
sleep 1
|
||||
done
|
||||
[ -n "$IP" ] && [ "$IP" != "None" ] || { echo "No public IP assigned after 120s"; exit 1; }
|
||||
echo " $IP"
|
||||
|
||||
# Clear stale known_hosts entry — instances reuse IPs across runs.
|
||||
KNOWN_HOSTS="$HOME/.ssh/known_hosts"
|
||||
[ -f "$KNOWN_HOSTS" ] && ssh-keygen -f "$KNOWN_HOSTS" -R "$IP" >/dev/null 2>&1 || true
|
||||
|
||||
# --- Wait for SSH ---
|
||||
echo "==> Waiting for SSH..."
|
||||
_ssh() {
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -o BatchMode=yes \
|
||||
-i "$SSH_KEY" "${SSH_USER}@${IP}" "$@"
|
||||
}
|
||||
|
||||
elapsed=0
|
||||
while [ "$elapsed" -lt "$SSH_TIMEOUT" ]; do
|
||||
_ssh 'echo ready' >/dev/null 2>&1 && break
|
||||
sleep 5; elapsed=$((elapsed + 5))
|
||||
done
|
||||
[ "$elapsed" -lt "$SSH_TIMEOUT" ] || { echo "SSH timeout after ${SSH_TIMEOUT}s"; exit 1; }
|
||||
|
||||
# --- GPU visibility check ---
|
||||
echo "==> Checking GPU visibility..."
|
||||
lspci_out=$(_ssh 'lspci -nn 2>/dev/null' || true)
|
||||
if printf '%s\n' "$lspci_out" | grep -qiE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA"; then
|
||||
printf '%s\n' "$lspci_out" \
|
||||
| grep -iE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA" \
|
||||
| sed 's/^/ ✓ /'
|
||||
else
|
||||
echo " WARNING: GPU not visible in lspci — check host-side passthrough assignment"
|
||||
printf '%s\n' "$lspci_out" | head -20 | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# --- Done ---
|
||||
echo ""
|
||||
echo "✅ ${INSTANCE_TYPE} ready — ${ID} (${IP})"
|
||||
echo ""
|
||||
echo " ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USER}@${IP}"
|
||||
echo ""
|
||||
echo " Terminate when done:"
|
||||
echo " AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids ${ID}"
|
||||
@@ -1,121 +0,0 @@
|
||||
#!/bin/bash
|
||||
# test-ddil-phase1.sh — systemd-level smoke test for daemon-local-autonomy
|
||||
# Tier 1 (1a / 1b / 1c / 1d / 1d-strict). Run after `make deploy` on a single
|
||||
# node where spinifex-daemon and spinifex-nats are managed by systemd.
|
||||
#
|
||||
# Usage: scripts/test-ddil-phase1.sh
|
||||
# Requires: sudo, curl, jq
|
||||
#
|
||||
# Destructive on the local node — stops/starts spinifex-nats and restarts
|
||||
# spinifex-daemon. Do NOT run against a host carrying workloads. The cleanup
|
||||
# trap restores both services on exit.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DAEMON_URL="${DAEMON_URL:-https://127.0.0.1:4432}"
|
||||
DAEMON_SVC=spinifex-daemon
|
||||
NATS_SVC=spinifex-nats
|
||||
OVERRIDE=/etc/systemd/system/${DAEMON_SVC}.service.d/99-test-ddil.conf
|
||||
NATS_BLOCK=/run/spinifex-nats-blocked
|
||||
NATS_BLOCK_DROPIN=/run/systemd/system/${NATS_SVC}.service.d/99-test-block.conf
|
||||
CURL=(curl -fsS -k --max-time 5)
|
||||
|
||||
ok() { printf '\033[32mPASS\033[0m %s\n' "$1"; }
|
||||
fail() { printf '\033[31mFAIL\033[0m %s\n' "$1"; exit 1; }
|
||||
info() { printf '\033[36m==>\033[0m %s\n' "$1"; }
|
||||
|
||||
cleanup() {
|
||||
info "cleanup: removing nats block, override, restoring services"
|
||||
sudo rm -f "$NATS_BLOCK" "$NATS_BLOCK_DROPIN" "$OVERRIDE"
|
||||
sudo rmdir "$(dirname "$NATS_BLOCK_DROPIN")" 2>/dev/null || true
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl start "$NATS_SVC" 2>/dev/null || true
|
||||
sudo systemctl restart "$DAEMON_SVC" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
field() { "${CURL[@]}" "${DAEMON_URL}/local/status" | jq -r ".$1 // empty"; }
|
||||
|
||||
wait_field() {
|
||||
local f=$1 want=$2 timeout=${3:-30} elapsed=0
|
||||
while [ "$elapsed" -lt "$timeout" ]; do
|
||||
[ "$(field "$f" 2>/dev/null || true)" = "$want" ] && return 0
|
||||
sleep 1; elapsed=$((elapsed + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 0. baseline
|
||||
info "baseline: daemon active + cluster mode"
|
||||
sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon not running — run \`make deploy\` first"
|
||||
sudo systemctl start "$NATS_SVC"
|
||||
wait_field mode cluster 30 || fail "never reached mode=cluster (got: $(field mode))"
|
||||
[ "$(field nats)" = connected ] || fail "nats not connected (got: $(field nats))"
|
||||
ok "mode=cluster nats=connected"
|
||||
|
||||
# 1. kill NATS → standalone, daemon survives
|
||||
info "1a/1c: kill NATS"
|
||||
retry_before=$(field nats_retry_count)
|
||||
sudo systemctl stop "$NATS_SVC"
|
||||
wait_field mode standalone 30 || fail "never went standalone (got: $(field mode))"
|
||||
wait_field nats disconnected 5 || fail "nats not disconnected (got: $(field nats))"
|
||||
sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon exited — should stay up"
|
||||
ok "standalone, nats=disconnected, daemon alive"
|
||||
|
||||
# 2. /local/instances reachable during partition
|
||||
info "1b: /local/instances during partition"
|
||||
"${CURL[@]}" "${DAEMON_URL}/local/instances" | jq -e . >/dev/null \
|
||||
|| fail "/local/instances did not return JSON"
|
||||
ok "/local/instances served while NATS down"
|
||||
|
||||
# 3. restart NATS → cluster mode + retry count advances + KV push observed
|
||||
# nats_retry_count counts disconnect→reconnect *cycles* (bumped from
|
||||
# onNATSReconnect, daemon.go:321). It stays flat during disconnect and
|
||||
# advances by 1 here when the reconnect callback fires.
|
||||
info "1c reconnect: restart NATS"
|
||||
before_sync=$(field last_kv_sync_at)
|
||||
sudo systemctl start "$NATS_SVC"
|
||||
wait_field mode cluster 60 || fail "did not return to cluster mode"
|
||||
wait_field nats connected 10 || fail "nats not reconnected"
|
||||
sleep 5
|
||||
retry_after=$(field nats_retry_count)
|
||||
[ "$retry_after" -gt "$retry_before" ] \
|
||||
|| fail "nats_retry_count did not advance on reconnect ($retry_before → $retry_after)"
|
||||
after_sync=$(field last_kv_sync_at)
|
||||
[ -n "$after_sync" ] && [ "$after_sync" != "$before_sync" ] \
|
||||
|| fail "last_kv_sync_at did not advance ($before_sync → $after_sync)"
|
||||
ok "cluster restored, retry $retry_before → $retry_after, KV resync at $after_sync"
|
||||
|
||||
# 4. SPINIFEX_REQUIRE_NATS=1 → daemon exits within ~30s when NATS down.
|
||||
# Block spinifex-nats via a runtime drop-in with ConditionPathExists=!. This
|
||||
# defeats the daemon unit's Wants=spinifex-nats.service auto-start, which
|
||||
# would otherwise pull NATS back up on `systemctl restart spinifex-daemon`
|
||||
# and let the daemon connect immediately. mask refuses (real unit lives in
|
||||
# /etc) and --runtime mask is shadowed by /etc — the condition drop-in is
|
||||
# the only mechanism that works against an /etc-installed unit.
|
||||
info "1d-strict: SPINIFEX_REQUIRE_NATS=1 exits when NATS down"
|
||||
sudo mkdir -p "$(dirname "$NATS_BLOCK_DROPIN")"
|
||||
sudo tee "$NATS_BLOCK_DROPIN" >/dev/null <<EOF
|
||||
[Unit]
|
||||
ConditionPathExists=!$NATS_BLOCK
|
||||
EOF
|
||||
sudo touch "$NATS_BLOCK"
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl stop "$NATS_SVC"
|
||||
sudo mkdir -p "$(dirname "$OVERRIDE")"
|
||||
sudo tee "$OVERRIDE" >/dev/null <<EOF
|
||||
[Service]
|
||||
Environment=SPINIFEX_REQUIRE_NATS=1
|
||||
Restart=no
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart "$DAEMON_SVC" || true
|
||||
start=$(date +%s)
|
||||
while sudo systemctl is-active --quiet "$DAEMON_SVC"; do
|
||||
[ $(( $(date +%s) - start )) -gt 45 ] && fail "did not exit within 45s"
|
||||
sleep 1
|
||||
done
|
||||
ok "exited after $(( $(date +%s) - start ))s"
|
||||
|
||||
echo
|
||||
echo "DDIL Phase 1 systemd integration: ALL PASS"
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Workload performance testing (memory, disk, cpu, network)
|
||||
|
||||
sudo apt update
|
||||
sudo apt install -y make gcc git
|
||||
|
||||
wget https://go.dev/dl/go1.27.2.linux-amd64.tar.gz
|
||||
sudo tar -C /usr/local -xzf go1.27.2.linux-amd64.tar.gz
|
||||
|
||||
echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
|
||||
source $HOME/.bashrc
|
||||
|
||||
git clone https://github.com/dgraph-io/badger.git
|
||||
cd badger
|
||||
|
||||
# disable jemalloc
|
||||
sed -i 's/-tags=jemalloc/-tags=nojemalloc/g' test.sh
|
||||
|
||||
# run 4 tests, otherwise OOM on <8GB RAM
|
||||
sed -i 's/-parallel 16/-parallel 4/g; s/-parallel=16/-parallel=4/g' test.sh
|
||||
|
||||
# run tests
|
||||
./test.sh
|
||||
Reference in New Issue
Block a user