diff --git a/scripts/README.md b/scripts/README.md index 215693f1f..99b673bb8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -38,14 +38,12 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o |---|---| | `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. | | `verify-network.sh` | Asserts a node's datapath matches its declared planes — EIP reachability, which plane carries Geneve, and which interfaces services bind. Catches what `smoke-test.sh` cannot, since none of its traffic leaves the box. | -| `test-ddil-phase1.sh` | systemd-level smoke test for daemon-local-autonomy tier 1. Destructive on the local node: stops and starts `spinifex-nats` and restarts `spinifex-daemon`, restoring both via a cleanup trap. | | `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. | | `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). | | `sync-aws-models.sh` | Replaces the embedded AWS Smithy models under `internal/awsmodel/models/` with the ten service files from `aws/api-models-aws` at the given commit, and records that commit in `internal/awsmodel/model_source.go`. Takes a full 40-character SHA. | | `run-bench.sh` | Launches a benchmark against running Ubuntu instances, auto-detecting the AMI for the host architecture. Tracks nbdkit and perf alongside the run. | | `disk-performance.sh` | Runs a `fio` random 70/30 read-write benchmark swept across 4k/16k/128k/1M. Writes JSON results to `/tmp/spinifex-disk-bench`; override with `BENCH_DIR`, `OUT_DIR`, `SIZE`, `JOBS`, `BLOCK_SIZES`. | | `network-performance.sh` | iperf throughput from several clients to one server, driven over SSH, writing a `summary.txt` of Gbit/s per client. `--server-ip` sets the address clients dial, so pointing it at instance private addresses measures the VPC overlay rather than the external pool. | -| `workload-performance.sh` | In-guest memory, disk, CPU and network workload test. Installs Go and runs the Badger test suite with jemalloc disabled and reduced parallelism. | ## Guest images @@ -93,13 +91,6 @@ Installed into the EKS node AMI and run at boot or on a timer. All host communic | `mulga-mgmt-net.sh` | Boot oneshot bringing a multi-NIC system VM's interfaces up by MAC from the QEMU fw_cfg netcfg blob, before cloud-init's network stage. cloud-init cannot reliably pick the right NIC out of two. | | `mulga-vpc-mtu.sh` | Boot oneshot pinning the VPC data NIC MTU so flannel sizes its VXLAN overlay to fit the OVN geneve underlay. Otherwise large frames are silently dropped while handshakes pass. | -## Launch helpers - -| Script | What it does | -|---|---| -| `launch-gpu.sh` | Launches a GPU instance from the AMD or NVIDIA GPU AMI, selecting the AMI from the instance type's GPU brand. `--amd` / `--nvidia` override the detection. | -| `ecs-node-bringup.sh` | Launches an ECS container-instance VM. Renders the cloud-init user-data seeding the agent's control-plane config, IAM credentials and gateway CA — there is no ECS-specific launch API, by design. | - ## `demo/` — GPU demo | Script | What it does | diff --git a/scripts/ecs-node-bringup.sh b/scripts/ecs-node-bringup.sh deleted file mode 100755 index 37017b2f1..000000000 --- a/scripts/ecs-node-bringup.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# ecs-node-bringup.sh — launch a Spinifex ECS container-instance VM. -# -# ECS container instances are plain EC2 instances booted from the spinifex-ecs-node -# AMI (Alpine + containerd + ecs-agent), AWS-faithful: there is no ECS-specific -# launch API. This script renders the cloud-init user-data that seeds the agent's -# control-plane config + IAM credentials into /etc/spinifex-ecs/agent.env and the -# gateway TLS CA into /etc/spinifex-ecs/gateway-ca.pem, then runs the instance. -# -# The agent then registers, heartbeats, polls assignments, and reports task state -# entirely through the AWS gateway over TLS+SigV4 — it never touches NATS. -# -# Usage: -# AWS_PROFILE=spinifex scripts/ecs-node-bringup.sh -# -# Env overrides: -# GATEWAY_IP gateway IP the guest dials (default: host WAN-bridge IP) -# GATEWAY_CA host path to the gateway CA PEM (default /etc/spinifex/ca.pem) -# AMI_ID ECS-node AMI (default: latest spinifex-ecs-node) -# SUBNET_ID / SG_ID / KEY_NAME / INSTANCE_TYPE (default: first available / t3.small) -set -euo pipefail - -CLUSTER="${1:?usage: ecs-node-bringup.sh }" -PROFILE="${AWS_PROFILE:-spinifex}" -REGION="$(aws configure get region --profile "$PROFILE")" - -# VPC guests NAT out through the WAN and reach host-run services on the host's -# WAN-bridge IP; the mgmt bridge (10.15.8.x) is not routable from a guest netns. -# Default to the br-wan address, falling back to the mgmt IP if it can't be read. -default_gateway_ip() { - ip -4 addr show br-wan 2>/dev/null | awk '/inet /{print $2; exit}' | cut -d/ -f1 -} -GATEWAY_IP="${GATEWAY_IP:-$(default_gateway_ip)}" -GATEWAY_IP="${GATEWAY_IP:-10.15.8.1}" -GATEWAY_CA="${GATEWAY_CA:-/etc/spinifex/ca.pem}" -INSTANCE_TYPE="${INSTANCE_TYPE:-t3.small}" - -awscli() { aws --profile "$PROFILE" "$@"; } - -AMI_ID="${AMI_ID:-$(awscli ec2 describe-images \ - --filters 'Name=name,Values=spinifex-ecs-node' \ - --query 'sort_by(Images,&CreationDate)[-1].ImageId' --output text)}" -SUBNET_ID="${SUBNET_ID:-$(awscli ec2 describe-subnets \ - --query 'Subnets[0].SubnetId' --output text)}" -SG_ID="${SG_ID:-$(awscli ec2 describe-security-groups \ - --query 'SecurityGroups[0].GroupId' --output text)}" -KEY_NAME="${KEY_NAME:-$(awscli ec2 describe-key-pairs \ - --query 'KeyPairs[0].KeyName' --output text)}" - -ACCESS_KEY="$(aws configure get aws_access_key_id --profile "$PROFILE")" -SECRET_KEY="$(aws configure get aws_secret_access_key --profile "$PROFILE")" - -echo "cluster=$CLUSTER ami=$AMI_ID subnet=$SUBNET_ID sg=$SG_ID key=$KEY_NAME gw=https://$GATEWAY_IP:9999" >&2 - -# Indent a file's content by six spaces for a cloud-init "content: |" block. -indent6() { sed 's/^/ /' "$1"; } - -USERDATA="$(cat < /etc/resolv.conf -write_files: - - path: /etc/spinifex-ecs/agent.env - permissions: '0600' - content: | - ECS_GATEWAY_URL=https://$GATEWAY_IP:9999 - ECS_GATEWAY_CA=/etc/spinifex-ecs/gateway-ca.pem - ECS_REGION=$REGION - ECS_CLUSTER=$CLUSTER - ECS_ACCESS_KEY=$ACCESS_KEY - ECS_SECRET_KEY=$SECRET_KEY - - path: /etc/spinifex-ecs/gateway-ca.pem - permissions: '0644' - content: | -$(indent6 "$GATEWAY_CA") -EOF -)" - -INSTANCE_ID="$(awscli ec2 run-instances \ - --image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" \ - --key-name "$KEY_NAME" --subnet-id "$SUBNET_ID" --security-group-ids "$SG_ID" \ - --user-data "$USERDATA" \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=ecs-node-$CLUSTER}]" \ - --query 'Instances[0].InstanceId' --output text)" - -echo "$INSTANCE_ID" diff --git a/scripts/launch-gpu.sh b/scripts/launch-gpu.sh deleted file mode 100755 index b5a1fca15..000000000 --- a/scripts/launch-gpu.sh +++ /dev/null @@ -1,248 +0,0 @@ -#!/bin/bash -set -euo pipefail - -export AWS_PROFILE=spinifex - -# Resolve the actual user's home — $HOME is /root when run under sudo. -_USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6) -SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}" -SSH_USER="${SSH_USER:-ubuntu}" -SSH_TIMEOUT="${SSH_TIMEOUT:-300}" - -usage() { - cat <<'EOF' -launch-gpu.sh — Launch a GPU instance from the Spinifex AMD or NVIDIA GPU AMI. - -Detects the GPU brand from the specified instance type and selects the matching -pre-built AMI automatically. Use --amd or --nvidia to override. - -Usage: - scripts/launch-gpu.sh --type [--disk GB] [--amd|--nvidia] - -Options: - --type TYPE EC2 instance type to launch (required) - --disk GB Root disk size in GB (default: 50) - --ami NAME Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-26.04-nvidia-gpu-x86_64) - --amd Force AMD GPU AMI (ubuntu-amd-gpu) - --nvidia Force NVIDIA GPU AMI (ubuntu-26.04-nvidia-gpu-x86_64) - -Env overrides: - SSH_KEY Path to SSH private key (default: ~/.ssh/spinifex-key) - SSH_USER SSH user inside guest (default: ubuntu) - SSH_TIMEOUT Seconds to wait for initial SSH (default: 300) -EOF -} - -INSTANCE_TYPE="" -DISK_GB=50 -GPU_BRAND="" -AMI_OVERRIDE="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --type) INSTANCE_TYPE="$2"; shift 2 ;; - --disk) DISK_GB="$2"; shift 2 ;; - --ami) AMI_OVERRIDE="$2"; shift 2 ;; - --amd) GPU_BRAND="amd"; shift ;; - --nvidia) GPU_BRAND="nvidia"; shift ;; - -h|--help) usage; exit 0 ;; - *) echo "Unknown option: $1"; usage; exit 1 ;; - esac -done - -list_gpu_types() { - aws ec2 describe-instance-types \ - --query 'InstanceTypes[?GpuInfo].[InstanceType,GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]' \ - --output table 2>/dev/null || true -} - -if [ -z "$INSTANCE_TYPE" ]; then - echo "Error: --type is required." - echo "" - echo "Available GPU instance types on this node:" - list_gpu_types - exit 1 -fi - -# --- Validate instance type and fetch GPU info in one call --- -echo "==> Checking instance type: ${INSTANCE_TYPE}" -INSTANCE_INFO=$(aws ec2 describe-instance-types \ - --query "InstanceTypes[?InstanceType=='${INSTANCE_TYPE}' && GpuInfo].[GpuInfo.Gpus[0].Count,GpuInfo.Gpus[0].Manufacturer,GpuInfo.Gpus[0].Name]" \ - --output text 2>/dev/null || true) - -if [ -z "$INSTANCE_INFO" ] || [ "$INSTANCE_INFO" = "None" ]; then - echo "'${INSTANCE_TYPE}' is not available or is not a GPU type on this node." - echo "" - echo "Available GPU instance types:" - list_gpu_types - exit 1 -fi - -IFS=$'\t' read -r GPU_COUNT GPU_MANUFACTURER GPU_NAME <<< "$INSTANCE_INFO" -echo " ${GPU_COUNT}× ${GPU_NAME} (${GPU_MANUFACTURER})" - -# --- Resolve GPU brand --- -if [ -z "$GPU_BRAND" ]; then - case "$GPU_MANUFACTURER" in - AMD|amd) GPU_BRAND="amd" ;; - NVIDIA|nvidia) GPU_BRAND="nvidia" ;; - *) - echo "Cannot auto-detect GPU brand from manufacturer '${GPU_MANUFACTURER}'." - echo "Use --amd or --nvidia to specify explicitly." - exit 1 - ;; - esac -fi - -case "$GPU_BRAND" in - amd) AMI_NAME="ubuntu-amd-gpu" ;; - nvidia) AMI_NAME="ubuntu-26.04-nvidia-gpu-x86_64" ;; -esac - -# --- Resolve AMI --- -AMI_LOOKUP="${AMI_OVERRIDE:-${AMI_NAME}}" -echo "==> Resolving AMI: ${AMI_LOOKUP}" -AMI_ID=$(aws ec2 describe-images \ - --query "Images[?Name=='${AMI_LOOKUP}'].ImageId | [0]" --output text 2>/dev/null || true) -if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then - echo "No AMI found with name '${AMI_LOOKUP}'." - echo "Build and import it first, or specify the AMI name with --ami " - echo " scripts/build-system-image.sh scripts/images/ubuntu-gpu-${GPU_BRAND}.conf --import" - exit 1 -fi -echo " $AMI_ID" - -# --- Resolve subnet --- -echo "==> Resolving subnet" -SUBNET_ID=$(aws ec2 describe-subnets \ - --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId | [0]' --output text 2>/dev/null || true) -if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then - echo "No public subnet found" - exit 1 -fi -echo " $SUBNET_ID" - -VPC_ID=$(aws ec2 describe-subnets --subnet-ids "$SUBNET_ID" \ - --query 'Subnets[0].VpcId' --output text 2>/dev/null || true) -if [ -z "$VPC_ID" ] || [ "$VPC_ID" = "None" ]; then - echo "Could not resolve VPC for subnet $SUBNET_ID" - exit 1 -fi - -# --- Ensure SSH security group --- -# The default VPC SG only allows intra-SG traffic; we need a group that -# permits port 22 from anywhere so the SSH wait below can connect. -SSH_SG_NAME="spinifex-gpu-ssh" -echo "==> Ensuring SSH security group (${SSH_SG_NAME})" -SG_ID=$(aws ec2 describe-security-groups \ - --filters "Name=group-name,Values=${SSH_SG_NAME}" "Name=vpc-id,Values=${VPC_ID}" \ - --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true) -if [ -z "$SG_ID" ] || [ "$SG_ID" = "None" ]; then - SG_ID=$(aws ec2 create-security-group \ - --group-name "$SSH_SG_NAME" \ - --description "Allow SSH access to GPU instances" \ - --vpc-id "$VPC_ID" \ - --query 'GroupId' --output text) - aws ec2 authorize-security-group-ingress \ - --group-id "$SG_ID" \ - --protocol tcp \ - --port 22 \ - --cidr 0.0.0.0/0 >/dev/null - echo " Created $SG_ID" -else - echo " Reusing $SG_ID" -fi - -# --- Ensure SSH key --- -if [ ! -f "$SSH_KEY" ]; then - echo "==> Generating SSH key" - sudo -u "${SUDO_USER:-$(id -un)}" mkdir -p "$(dirname "$SSH_KEY")" - sudo -u "${SUDO_USER:-$(id -un)}" ssh-keygen -t ed25519 -f "$SSH_KEY" -N "" -fi -# Delete before import — stale key pair material causes import to fail -# silently, leaving the instance unreachable via SSH. -echo "==> Importing SSH key" -aws ec2 delete-key-pair --key-name spinifex-key >/dev/null 2>&1 || true -aws ec2 import-key-pair --key-name spinifex-key \ - --public-key-material "fileb://${SSH_KEY}.pub" >/dev/null 2>&1 || true - -# --- Launch --- -echo "==> Launching ${INSTANCE_TYPE} (${DISK_GB} GB)..." -ID=$(aws ec2 run-instances \ - --image-id "$AMI_ID" \ - --instance-type "$INSTANCE_TYPE" \ - --key-name spinifex-key \ - --subnet-id "$SUBNET_ID" \ - --security-group-ids "$SG_ID" \ - --count 1 \ - --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":${DISK_GB},\"VolumeType\":\"gp3\",\"DeleteOnTermination\":true}}]" \ - --query 'Instances[0].InstanceId' --output text) -if [ -z "$ID" ] || [ "$ID" = "None" ] || [ "$ID" = "null" ]; then - echo "run-instances returned no InstanceId" - exit 1 -fi -echo " $ID" - -# --- Wait for running --- -echo "==> Waiting for running state..." -COUNT=0; STATE="unknown" -while [ $COUNT -lt 120 ]; do - STATE=$(aws ec2 describe-instances --instance-ids "$ID" \ - --query "Reservations[0].Instances[0].State.Name" --output text 2>/dev/null || echo "unknown") - [ "$STATE" = "running" ] && break - [ "$STATE" = "terminated" ] && { echo "Instance terminated unexpectedly"; exit 1; } - sleep 2; COUNT=$((COUNT + 2)) -done -[ "$STATE" = "running" ] || { echo "Instance did not reach running state (last: $STATE)"; exit 1; } - -# --- Wait for public IP --- -echo "==> Waiting for public IP..." -IP="" -for _i in $(seq 1 120); do - IP=$(aws ec2 describe-instances --instance-ids "$ID" \ - --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null || true) - [ -n "$IP" ] && [ "$IP" != "None" ] && [ "$IP" != "null" ] && break - sleep 1 -done -[ -n "$IP" ] && [ "$IP" != "None" ] || { echo "No public IP assigned after 120s"; exit 1; } -echo " $IP" - -# Clear stale known_hosts entry — instances reuse IPs across runs. -KNOWN_HOSTS="$HOME/.ssh/known_hosts" -[ -f "$KNOWN_HOSTS" ] && ssh-keygen -f "$KNOWN_HOSTS" -R "$IP" >/dev/null 2>&1 || true - -# --- Wait for SSH --- -echo "==> Waiting for SSH..." -_ssh() { - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=5 -o BatchMode=yes \ - -i "$SSH_KEY" "${SSH_USER}@${IP}" "$@" -} - -elapsed=0 -while [ "$elapsed" -lt "$SSH_TIMEOUT" ]; do - _ssh 'echo ready' >/dev/null 2>&1 && break - sleep 5; elapsed=$((elapsed + 5)) -done -[ "$elapsed" -lt "$SSH_TIMEOUT" ] || { echo "SSH timeout after ${SSH_TIMEOUT}s"; exit 1; } - -# --- GPU visibility check --- -echo "==> Checking GPU visibility..." -lspci_out=$(_ssh 'lspci -nn 2>/dev/null' || true) -if printf '%s\n' "$lspci_out" | grep -qiE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA"; then - printf '%s\n' "$lspci_out" \ - | grep -iE "1002:|10de:|Instinct|Aqua|Processing accelerator|NVIDIA" \ - | sed 's/^/ ✓ /' -else - echo " WARNING: GPU not visible in lspci — check host-side passthrough assignment" - printf '%s\n' "$lspci_out" | head -20 | sed 's/^/ /' -fi - -# --- Done --- -echo "" -echo "✅ ${INSTANCE_TYPE} ready — ${ID} (${IP})" -echo "" -echo " ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no ${SSH_USER}@${IP}" -echo "" -echo " Terminate when done:" -echo " AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids ${ID}" diff --git a/scripts/test-ddil-phase1.sh b/scripts/test-ddil-phase1.sh deleted file mode 100755 index 197c224cf..000000000 --- a/scripts/test-ddil-phase1.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash -# test-ddil-phase1.sh — systemd-level smoke test for daemon-local-autonomy -# Tier 1 (1a / 1b / 1c / 1d / 1d-strict). Run after `make deploy` on a single -# node where spinifex-daemon and spinifex-nats are managed by systemd. -# -# Usage: scripts/test-ddil-phase1.sh -# Requires: sudo, curl, jq -# -# Destructive on the local node — stops/starts spinifex-nats and restarts -# spinifex-daemon. Do NOT run against a host carrying workloads. The cleanup -# trap restores both services on exit. - -set -euo pipefail - -DAEMON_URL="${DAEMON_URL:-https://127.0.0.1:4432}" -DAEMON_SVC=spinifex-daemon -NATS_SVC=spinifex-nats -OVERRIDE=/etc/systemd/system/${DAEMON_SVC}.service.d/99-test-ddil.conf -NATS_BLOCK=/run/spinifex-nats-blocked -NATS_BLOCK_DROPIN=/run/systemd/system/${NATS_SVC}.service.d/99-test-block.conf -CURL=(curl -fsS -k --max-time 5) - -ok() { printf '\033[32mPASS\033[0m %s\n' "$1"; } -fail() { printf '\033[31mFAIL\033[0m %s\n' "$1"; exit 1; } -info() { printf '\033[36m==>\033[0m %s\n' "$1"; } - -cleanup() { - info "cleanup: removing nats block, override, restoring services" - sudo rm -f "$NATS_BLOCK" "$NATS_BLOCK_DROPIN" "$OVERRIDE" - sudo rmdir "$(dirname "$NATS_BLOCK_DROPIN")" 2>/dev/null || true - sudo systemctl daemon-reload - sudo systemctl start "$NATS_SVC" 2>/dev/null || true - sudo systemctl restart "$DAEMON_SVC" 2>/dev/null || true -} -trap cleanup EXIT - -field() { "${CURL[@]}" "${DAEMON_URL}/local/status" | jq -r ".$1 // empty"; } - -wait_field() { - local f=$1 want=$2 timeout=${3:-30} elapsed=0 - while [ "$elapsed" -lt "$timeout" ]; do - [ "$(field "$f" 2>/dev/null || true)" = "$want" ] && return 0 - sleep 1; elapsed=$((elapsed + 1)) - done - return 1 -} - -# 0. baseline -info "baseline: daemon active + cluster mode" -sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon not running — run \`make deploy\` first" -sudo systemctl start "$NATS_SVC" -wait_field mode cluster 30 || fail "never reached mode=cluster (got: $(field mode))" -[ "$(field nats)" = connected ] || fail "nats not connected (got: $(field nats))" -ok "mode=cluster nats=connected" - -# 1. kill NATS → standalone, daemon survives -info "1a/1c: kill NATS" -retry_before=$(field nats_retry_count) -sudo systemctl stop "$NATS_SVC" -wait_field mode standalone 30 || fail "never went standalone (got: $(field mode))" -wait_field nats disconnected 5 || fail "nats not disconnected (got: $(field nats))" -sudo systemctl is-active --quiet "$DAEMON_SVC" || fail "daemon exited — should stay up" -ok "standalone, nats=disconnected, daemon alive" - -# 2. /local/instances reachable during partition -info "1b: /local/instances during partition" -"${CURL[@]}" "${DAEMON_URL}/local/instances" | jq -e . >/dev/null \ - || fail "/local/instances did not return JSON" -ok "/local/instances served while NATS down" - -# 3. restart NATS → cluster mode + retry count advances + KV push observed -# nats_retry_count counts disconnect→reconnect *cycles* (bumped from -# onNATSReconnect, daemon.go:321). It stays flat during disconnect and -# advances by 1 here when the reconnect callback fires. -info "1c reconnect: restart NATS" -before_sync=$(field last_kv_sync_at) -sudo systemctl start "$NATS_SVC" -wait_field mode cluster 60 || fail "did not return to cluster mode" -wait_field nats connected 10 || fail "nats not reconnected" -sleep 5 -retry_after=$(field nats_retry_count) -[ "$retry_after" -gt "$retry_before" ] \ - || fail "nats_retry_count did not advance on reconnect ($retry_before → $retry_after)" -after_sync=$(field last_kv_sync_at) -[ -n "$after_sync" ] && [ "$after_sync" != "$before_sync" ] \ - || fail "last_kv_sync_at did not advance ($before_sync → $after_sync)" -ok "cluster restored, retry $retry_before → $retry_after, KV resync at $after_sync" - -# 4. SPINIFEX_REQUIRE_NATS=1 → daemon exits within ~30s when NATS down. -# Block spinifex-nats via a runtime drop-in with ConditionPathExists=!. This -# defeats the daemon unit's Wants=spinifex-nats.service auto-start, which -# would otherwise pull NATS back up on `systemctl restart spinifex-daemon` -# and let the daemon connect immediately. mask refuses (real unit lives in -# /etc) and --runtime mask is shadowed by /etc — the condition drop-in is -# the only mechanism that works against an /etc-installed unit. -info "1d-strict: SPINIFEX_REQUIRE_NATS=1 exits when NATS down" -sudo mkdir -p "$(dirname "$NATS_BLOCK_DROPIN")" -sudo tee "$NATS_BLOCK_DROPIN" >/dev/null </dev/null <> $HOME/.bashrc -source $HOME/.bashrc - -git clone https://github.com/dgraph-io/badger.git -cd badger - -# disable jemalloc -sed -i 's/-tags=jemalloc/-tags=nojemalloc/g' test.sh - -# run 4 tests, otherwise OOM on <8GB RAM -sed -i 's/-parallel 16/-parallel 4/g; s/-parallel=16/-parallel=4/g' test.sh - -# run tests -./test.sh