Fix service helper sequence

Access key and secret key values were printed to stdout when using
--access-key and --secret-key flags. These are captured by systemd
journal and log aggregation, exposing secrets. Log that keys were
overridden without printing the actual values.

Addresses H-5 from security audit.

.githooks/pre-commit

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Only enforce preflight on dev and main
+if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
+  exit 0
+fi
+
+# Only run preflight if Go files were staged
+if git diff --cached --name-only | grep -q '\.go$'; then
+  echo "Go files staged on $branch — running make preflight..."
+  make preflight
+else
+  echo "No Go files staged, skipping preflight."
+fi

.github/dependabot.yml

@@ -0,0 +1,52 @@
+version: 2
+updates:
+  - package-ecosystem: gomod
+    target-branch: main
+    directory: /
+    schedule:
+      interval: weekly
+      day: monday
+      timezone: Australia/Sydney
+    cooldown:
+      default-days: 1
+    groups:
+      go-dependencies:
+        patterns:
+          - "*"
+
+  - package-ecosystem: github-actions
+    target-branch: main
+    directory: /
+    schedule:
+      interval: weekly
+      day: monday
+      timezone: Australia/Sydney
+    cooldown:
+      default-days: 1
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - "version-update:semver-minor"
+          - "version-update:semver-patch"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+
+  - package-ecosystem: npm
+    target-branch: main
+    directory: /spinifex/services/spinifexui/frontend
+    schedule:
+      interval: weekly
+      day: monday
+      timezone: Australia/Sydney
+    cooldown:
+      default-days: 1
+    ignore:
+      - dependency-name: "@types/node"
+        update-types:
+          - "version-update:semver-major"
+    groups:
+      npm-dependencies:
+        patterns:
+          - "*"

.github/workflows/e2e.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
@@ -13,6 +12,13 @@ on:
       - tests/e2e/**
       - .github/workflows/e2e.yml
   workflow_dispatch:
+    inputs:
+      sha:
+        description: "PR head SHA to report status on"
+        required: false
+
+permissions:
+  statuses: write
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -25,7 +31,7 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -61,6 +67,7 @@ jobs:
         run: |
           SPX_PREBUILT=true \
           SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
+          SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
           ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
 
       - name: Run E2E Tests
@@ -71,7 +78,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh'
+            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh && ./tests/e2e/run-cert-e2e.sh'
 
       - name: Teardown
         if: always()
@@ -84,11 +91,11 @@ jobs:
   e2e_pseudo_multi:
     name: Pseudo Multi-Node E2E
     runs-on: [self-hosted, ci-pseudo]
-    timeout-minutes: 25
+    timeout-minutes: 30
     env:
       TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -134,7 +141,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-pseudo-multinode-e2e.sh'
+            'cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL="192.168.1.220-192.168.1.229" ./tests/e2e/run-pseudo-multinode-e2e.sh'
 
       - name: Teardown
         if: always()
@@ -151,7 +158,7 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -187,6 +194,7 @@ jobs:
         run: |
           SPX_PREBUILT=true \
           SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
+          SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
           ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
 
       - name: Run E2E Tests
@@ -205,7 +213,7 @@ jobs:
           done
 
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
+            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}"
 
       - name: Teardown
         if: always()
@@ -214,3 +222,29 @@ jobs:
           tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
+
+  e2e_status:
+    name: Report E2E Status
+    if: always() && inputs.sha
+    needs: [e2e_single, e2e_pseudo_multi, e2e_real_multi]
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 5
+    steps:
+      - name: Report status to PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ needs.e2e_single.result }}" == "success" && \
+                "${{ needs.e2e_pseudo_multi.result }}" == "success" && \
+                "${{ needs.e2e_real_multi.result }}" == "success" ]]; then
+            STATE="success"
+            DESC="All E2E suites passed"
+          else
+            STATE="failure"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}"
+          fi
+          gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
+            -f state="$STATE" \
+            -f context=e2e \
+            -f description="$DESC" \
+            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/release.yml

@@ -37,7 +37,7 @@ jobs:
         run: cd spinifex && make distro-amd64 VERSION="$VERSION"
 
       - name: Upload amd64 Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: distro-amd64
           path: |
@@ -74,7 +74,7 @@ jobs:
         run: cd spinifex && make distro-arm64 VERSION="$VERSION"
 
       - name: Upload arm64 Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: distro-arm64
           path: |
@@ -86,13 +86,13 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Download amd64 Artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: distro-amd64
           path: artifacts
 
       - name: Download arm64 Artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: distro-arm64
           path: artifacts
@@ -108,7 +108,7 @@ jobs:
           generate_release_notes: true
 
       - name: Upload Tarball for Verify Job
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: release-assets
           path: artifacts/
@@ -140,7 +140,7 @@ jobs:
           path: mulga
 
       - name: Download Release Assets
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: release-assets
           path: release-assets

.github/workflows/spinifex.yml

@@ -4,13 +4,15 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
       - go.sum
       - Makefile
       - .github/workflows/spinifex.yml
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
 
 jobs:
@@ -121,7 +123,7 @@ jobs:
         run: go work init ./spinifex ./viperblock ./predastore
 
       - name: Lint
-        uses: golangci/golangci-lint-action@v7
+        uses: golangci/golangci-lint-action@v9
         with:
           version: v2.11.3
           working-directory: spinifex

.github/workflows/ui.yml

@@ -4,7 +4,12 @@ on:
   push:
     branches:
       - main
-      - dev
+    paths:
+      - spinifex/services/spinifexui/frontend/**
+      - .github/workflows/ui.yml
+  pull_request:
+    branches:
+      - main
     paths:
       - spinifex/services/spinifexui/frontend/**
       - .github/workflows/ui.yml

.gitignore

@@ -14,6 +14,7 @@ go.work.sum
 # Build outputs
 bin/
 dist/
+!spinifex/services/spinifexui/frontend/dist/
 tmp/
 *.test
 *.out
@@ -39,3 +40,4 @@ docs/PLAN.md
 docs/TODO.md
 docs/development/
 docs/internal/
+scripts/iac/aws/spinifex-test.pem

Makefile

@@ -63,6 +63,19 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+build-alb-agent:
+	@echo -e "\n....Building alb-agent (static)"
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/alb-agent cmd/alb-agent/main.go
+
+build-system-image: ## Build a system image from manifest (use IMAGE=alb)
+ifndef IMAGE
+	$(error IMAGE is required. Usage: make build-system-image IMAGE=alb)
+endif
+	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
+
+build-alb-image: ## Build ALB Alpine image
+	$(MAKE) build-system-image IMAGE=alb
+
 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
 	$(GOPATH)/bin/$(GO_PROJECT_NAME)
@@ -116,7 +129,7 @@ install-system:
 		nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
-		ovn-central ovn-host openvswitch-switch
+		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
 	@echo -e "\n....Installing Go 1.26.1 for $(ARCH) ($(GO_ARCH))...."
@@ -194,7 +207,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
+.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -50,25 +50,26 @@ AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, o
 - Designed for Control – Run on your own terms, whether that’s in a datacenter, remote site, or sensitive environment.
 - Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
 - Works Offline – No reliance on centralized cloud services or external networks.
-- Open Source – Licensed under Apache 2.0. Fork it, extend it, or deploy it as-is.
+- Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
 
 ## Installation
 
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [https://docs.mulgadc.com](https://docs.mulgadc.com/) for maintaining and installing Spinifex.
+Installation requires an Ubuntu / Debian system. See the detailed documentation at [https://docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
 
-### Single node install
+### Single Node Install
 
 The installation is straightforward to set up and running on a single node for testing purposes.
 
 ```bash
+curl https://install.mulgadc.com | bash
+
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+
+sudo spx admin init
+
+sudo systemctl start spinifex.target
+
 export AWS_PROFILE=spinifex
-export AWS_REGION=ap-southeast-2-sc
-```
-
-```bash
-curl https://install.mulgadc.com/ | bash
-
-spx admin init --region $AWS_REGION
 
 aws ec2 describe-instance-types
 ```
@@ -121,4 +122,4 @@ This hybrid approach ensures Spinifex benefits from both proven engineering expe
 
 ## License
 
-Spinifex is open source under the Apache 2.0 License. You're free to use, modify, and deploy it—anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it—anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/Dockerfile.distro

@@ -42,6 +42,7 @@ RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-s
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
+RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
 RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
 
 FROM scratch AS export

build/scripts/predastore-start.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Helper script for spinifex-predastore.service
+# Detects host, port, backend, and node-id from spinifex.toml
+# then execs spx. Falls back to single-node defaults if no config found.
+
+CONF=/etc/spinifex/spinifex.toml
+BIND="0.0.0.0:8443"
+BACKEND="filesystem"
+NODE_ID="0"
+
+if [ -f "$CONF" ]; then
+    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$H" ]; then
+        BIND="$H"
+        BACKEND="distributed"
+    fi
+
+    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
+    if [ -n "$N" ] && [ "$N" != "0" ]; then
+        NODE_ID="$N"
+    fi
+fi
+
+export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
+export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
+export SPINIFEX_PREDASTORE_BACKEND="$BACKEND"
+export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
+
+exec /usr/local/bin/spx service predastore start

build/scripts/wait-for-nats.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Wait for NATS to be reachable before starting a dependent service.
+# Reads the NATS address from spinifex.toml [nodes.*.nats] section,
+# falls back to nats.conf, then to 127.0.0.1:4222.
+
+CONF=/etc/spinifex/spinifex.toml
+NATS_CONF=/etc/spinifex/nats/nats.conf
+NATS_HOST=""
+NATS_PORT="4222"
+TIMEOUT=30
+
+# Try spinifex.toml first
+if [ -f "$CONF" ]; then
+    NATS_BIND=$(awk -F'"' '/\[nodes\..*\.nats\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$NATS_BIND" ]; then
+        NATS_HOST="${NATS_BIND%%:*}"
+        NATS_PORT="${NATS_BIND##*:}"
+    fi
+fi
+
+# Fall back to nats.conf
+if [ -z "$NATS_HOST" ] && [ -f "$NATS_CONF" ]; then
+    NATS_HOST=$(grep "^listen:" "$NATS_CONF" | sed 's/listen: *//;s/:.*//')
+fi
+
+# Final fallback
+NATS_HOST="${NATS_HOST:-127.0.0.1}"
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "Timed out waiting for NATS at $NATS_HOST:$NATS_PORT" >&2
+exit 1

build/systemd/spinifex-awsgw.service

@@ -2,13 +2,13 @@
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-daemon.service

@@ -2,13 +2,13 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-predastore.service

@@ -2,14 +2,14 @@
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
-ExecStart=/usr/local/bin/spx service predastore start
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/var/lib/spinifex/predastore-start.sh
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500

build/systemd/spinifex-ui.service

@@ -11,6 +11,8 @@ ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
+Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -2,13 +2,13 @@
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-predastore.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-vpcd.service

@@ -2,13 +2,13 @@
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5

cmd/alb-agent/main.go

@@ -0,0 +1,62 @@
+// alb-agent runs inside ALB VMs and manages HAProxy configuration.
+//
+// It connects to NATS, subscribes to config updates for its load balancer,
+// writes HAProxy configs to disk, and reloads HAProxy on changes. It also
+// responds to health pings so the daemon can verify the agent is alive.
+//
+// Usage:
+//
+//	alb-agent --lb-id=lb-xxxxx --nats=nats://10.0.0.1:4222
+//
+// The LB ID defaults to the hostname (set by cloud-init to the LB ID).
+// The NATS URL defaults to nats://127.0.0.1:4222.
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mulgadc/spinifex/spinifex/albagent"
+)
+
+func main() {
+	var (
+		lbID    string
+		natsURL string
+	)
+
+	hostname, _ := os.Hostname()
+
+	flag.StringVar(&lbID, "lb-id", hostname, "Load balancer ID (defaults to hostname)")
+	flag.StringVar(&natsURL, "nats", "nats://127.0.0.1:4222", "NATS server URL")
+	flag.Parse()
+
+	if lbID == "" {
+		slog.Error("--lb-id is required (or set hostname to the LB ID)")
+		os.Exit(1)
+	}
+
+	slog.Info("Starting ALB agent", "lbId", lbID, "nats", natsURL)
+
+	agent, err := albagent.New(lbID, natsURL)
+	if err != nil {
+		slog.Error("Failed to create agent", "err", err)
+		os.Exit(1)
+	}
+
+	if err := agent.Start(); err != nil {
+		slog.Error("Failed to start agent", "err", err)
+		os.Exit(1)
+	}
+
+	// Wait for shutdown signal
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+	sig := <-sigCh
+
+	slog.Info("Received signal, shutting down", "signal", sig)
+	agent.Stop()
+}

cmd/spinifex/cmd/admin.go

@@ -1,23 +1,25 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package cmd
 
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
@@ -26,10 +28,12 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -156,6 +160,22 @@ var accountListCmd = &cobra.Command{
 	Run:   runAccountList,
 }
 
+var certCmd = &cobra.Command{
+	Use:   "cert",
+	Short: "Manage TLS certificates",
+	Long:  `Manage TLS certificates for the Spinifex platform.`,
+}
+
+var certRenewCmd = &cobra.Command{
+	Use:   "renew",
+	Short: "Regenerate the server certificate with current IPs",
+	Long: `Regenerate the server certificate signed by the existing CA.
+The new certificate will include all current network interface IPs
+and the machine hostname in its SANs. Use this after adding a new
+network interface or changing IP addresses.`,
+	Run: runCertRenew,
+}
+
 /*
 CLI ideas
 
@@ -193,6 +213,11 @@ func init() {
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
+
+	adminCmd.AddCommand(certCmd)
+	certCmd.AddCommand(certRenewCmd)
+	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
+	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
 	accountCreateCmd.MarkFlagRequired("name")
 
@@ -213,8 +238,19 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
+	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
 
+	// External networking flags
+	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
+	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
+	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
+	adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)")
+	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
+	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
+	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
+	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")
+
 	// Flags for admin join
 	adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node")
 	adminJoinCmd.Flags().String("az", "ap-southeast-2a", "Availability zone for this node")
@@ -226,6 +262,7 @@ func init() {
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
+	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 
@@ -508,6 +545,12 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 
 // TODO: Move all logic to a module, use minimal application logic in viper commands
 func runAdminInit(cmd *cobra.Command, args []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
+		fmt.Fprintln(os.Stderr, "   Run with sudo, or manually: sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates")
+		fmt.Fprintln(os.Stderr)
+	}
+
 	force, _ := cmd.Flags().GetBool("force")
 	configDir, _ := cmd.Flags().GetString("config-dir")
 	spxRoot, _ := cmd.Flags().GetString("spinifex-dir")
@@ -528,6 +571,157 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")
 
+	// External networking flags
+	externalMode, _ := cmd.Flags().GetString("external-mode")
+	externalIface, _ := cmd.Flags().GetString("external-iface")
+	externalSource, _ := cmd.Flags().GetString("external-source")
+	externalPool, _ := cmd.Flags().GetString("external-pool")
+	externalGateway, _ := cmd.Flags().GetString("external-gateway")
+	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
+	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
+	noExternal, _ := cmd.Flags().GetBool("no-external")
+
+	// Fire telemetry in background (completes during init work, waited at end)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID:    admin.ReadMachineID(),
+				Event:        "init",
+				Region:       region,
+				AZ:           az,
+				Node:         node,
+				Nodes:        nodes,
+				BindIP:       bindIP,
+				Version:      Version,
+				ExternalMode: externalMode,
+			})
+		})
+	}
+
+	// Auto-detect network topology
+	var poolStart, poolEnd string
+	var detectedNet *admin.DetectedNetwork
+	if !noExternal {
+		detected, err := admin.DetectNetwork()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
+			fmt.Fprintf(os.Stderr, "   Use --no-external to skip, or specify flags manually.\n")
+		} else {
+			detectedNet = detected
+
+			// Print detected topology
+			fmt.Println("\n🔍 Detected network topology:")
+			fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", "Interface", "IP", "Subnet", "Gateway", "Role")
+			for _, iface := range detected.Interfaces {
+				gw := "—"
+				if iface.Gateway != "" {
+					gw = iface.Gateway
+				}
+				fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role))
+			}
+			if detected.LANCount == 0 {
+				fmt.Println("\n  Mode: single-NIC (macvlan for external bridge)")
+			} else {
+				fmt.Printf("\n  Mode: %d LAN + 1 WAN (macvlan for external bridge)\n", detected.LANCount)
+			}
+
+			// Apply auto-detected values when flags not explicitly set
+			if detected.WAN != nil {
+				if externalIface == "" {
+					externalIface = detected.WAN.Name
+				}
+				if externalGateway == "" {
+					externalGateway = detected.WAN.Gateway
+				}
+				if !cmd.Flags().Changed("external-prefix-len") {
+					externalPrefixLen = detected.WAN.PrefixLen
+				}
+
+				// Default mode: always "pool" — with static range if --external-pool
+				// is given, otherwise with source=dhcp (gateway IP from router DHCP)
+				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+					externalMode = "pool"
+					if externalPool == "" && externalSource == "" {
+						externalSource = "dhcp"
+					}
+				}
+			}
+		}
+	}
+
+	// Validate external networking flags
+	if externalMode != "" && externalMode != "pool" {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool' or empty, got: %s\n", externalMode)
+		os.Exit(1)
+	}
+	if externalMode == "pool" {
+		// Resolve source: if not explicitly set, infer from whether a pool range was given
+		if externalSource == "" {
+			if externalPool != "" {
+				externalSource = "static"
+			} else {
+				externalSource = "dhcp"
+			}
+		}
+		if externalSource == "dhcp" {
+			// DHCP source: no static range needed, just gateway
+			if externalGateway == "" {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n")
+				os.Exit(1)
+			}
+		} else {
+			// Static source: need pool range
+			if externalPool == "" {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n")
+				fmt.Fprintf(os.Stderr, "   Or use --external-source=dhcp to get IPs from router DHCP\n")
+				if detectedNet != nil && detectedNet.WAN != nil {
+					sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
+					fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
+				}
+				os.Exit(1)
+			}
+			if externalGateway == "" {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n")
+				os.Exit(1)
+			}
+			parts := strings.SplitN(externalPool, "-", 2)
+			if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool)
+				os.Exit(1)
+			}
+			poolStart, poolEnd = parts[0], parts[1]
+		}
+	}
+	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
+		os.Exit(1)
+	}
+	if gatewayIP != "" && net.ParseIP(gatewayIP) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --gateway-ip is not a valid IP: %s\n", gatewayIP)
+		os.Exit(1)
+	}
+
+	// Detect DNS servers from the host for VM DHCP
+	var dnsServers []string
+	if externalMode != "" {
+		dnsServers = detectDNSServers(externalIface)
+		if len(dnsServers) > 0 {
+			fmt.Printf("  DNS servers: %s\n", strings.Join(dnsServers, ", "))
+		}
+	}
+
+	// For pool/dhcp source, the gateway IP is obtained via DHCP on the
+	// macvlan/bridge interface (no static pool range or explicit gateway-ip).
+	useExternalDHCP := externalMode == "pool" && externalSource == "dhcp" && gatewayIP == ""
+
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -609,6 +803,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
 	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
 
+	// Install CA certificate into system trust store
+	installCACertificate(filepath.Join(configDir, "ca.pem"))
+
 	// Generate NATS token
 	natsToken, err := admin.GenerateNATSToken()
 	if err != nil {
@@ -626,9 +823,35 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	isMultiNode := nodes >= 2 && bindIP != "0.0.0.0"
 
 	if isMultiNode {
+		// Build cluster-wide network config for propagation to joining nodes
+		var networkConfig *formation.NetworkConfig
+		if externalMode != "" {
+			bootstrapVpcId := utils.GenerateResourceID("vpc")
+			bootstrapSubnetId := utils.GenerateResourceID("subnet")
+			bootstrapIgwId := utils.GenerateResourceID("igw")
+			networkConfig = &formation.NetworkConfig{
+				ExternalMode:        externalMode,
+				ExternalDHCP:        useExternalDHCP,
+				PoolName:            "wan",
+				PoolSource:          externalSource,
+				PoolStart:           poolStart,
+				PoolEnd:             poolEnd,
+				PoolGateway:         externalGateway,
+				PoolGatewayIP:       gatewayIP,
+				PoolPrefixLen:       externalPrefixLen,
+				PoolDNSServers:      dnsServers,
+				BootstrapAccountId:  admin.DefaultAccountID(),
+				BootstrapVpcId:      bootstrapVpcId,
+				BootstrapSubnetId:   bootstrapSubnetId,
+				BootstrapIgwId:      bootstrapIgwId,
+				BootstrapCidr:       handlers_ec2_vpc.DefaultVPCCidr,
+				BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
+			}
+		}
+
 		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind,
-			port, nodes, formationTimeoutStr, services)
+			port, nodes, formationTimeoutStr, services, networkConfig)
 		return
 	}
 
@@ -695,6 +918,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
 	}
 
+	// Pre-generate default VPC/subnet/IGW IDs for bootstrap config.
+	// These are written to [bootstrap] in spinifex.toml so vpcd can
+	// create OVN topology on first boot. The daemon uses the same IDs
+	// when it creates the records in NATS KV via EnsureDefaultVPC.
+	bootstrapVpcId := utils.GenerateResourceID("vpc")
+	bootstrapSubnetId := utils.GenerateResourceID("subnet")
+	bootstrapIgwId := utils.GenerateResourceID("igw")
+
 	configSettings := admin.ConfigSettings{
 		AccessKey: accessKey,
 		SecretKey: secretKey,
@@ -717,6 +948,42 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode:   externalMode,
+		ExternalIface:  externalIface,
+		WanBridge:      detectedWanBridge(detectedNet),
+		ExternalDHCP:   useExternalDHCP,
+		PoolName:       "wan",
+		PoolSource:     externalSource,
+		PoolStart:      poolStart,
+		PoolEnd:        poolEnd,
+		PoolGateway:    externalGateway,
+		PoolGatewayIP:  gatewayIP,
+		PoolPrefixLen:  externalPrefixLen,
+		PoolDNSServers: dnsServers,
+
+		BootstrapAccountId:  admin.DefaultAccountID(),
+		BootstrapVpcId:      bootstrapVpcId,
+		BootstrapSubnetId:   bootstrapSubnetId,
+		BootstrapIgwId:      bootstrapIgwId,
+		BootstrapCidr:       handlers_ec2_vpc.DefaultVPCCidr,
+		BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr,
+	}
+
+	// Print external networking summary
+	if externalMode != "" {
+		fmt.Printf("\n📡 External networking: %s\n", externalMode)
+		fmt.Printf("  WAN interface: %s\n", externalIface)
+		if poolStart != "" {
+			fmt.Printf("  Source:        static (IP range)\n")
+			fmt.Printf("  IP pool:       %s - %s\n", poolStart, poolEnd)
+			fmt.Printf("  ⚠️  Ensure %s-%s is excluded from your router's DHCP range.\n", poolStart, poolEnd)
+		} else if useExternalDHCP {
+			fmt.Printf("  Source:        dhcp (from router DHCP)\n")
+			fmt.Println("  Gateway IP:    DHCP (obtained during OVN setup)")
+		} else if gatewayIP != "" {
+			fmt.Printf("  Gateway IP:    %s (static)\n", gatewayIP)
+		}
 	}
 
 	// Generate config files
@@ -759,13 +1026,6 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 	// Print success message
 	fmt.Println("\n🎉 Spinifex initialization complete!")
-	fmt.Println("\n📋 Next steps:")
-	fmt.Println("   1. Start services:")
-	fmt.Println("      ./scripts/start-dev.sh")
-	fmt.Println()
-	fmt.Println("   2. Test with AWS CLI:")
-	fmt.Println("      export AWS_PROFILE=spinifex")
-	fmt.Println("      aws ec2 describe-instances")
 	fmt.Println()
 	fmt.Println("🔗 Configuration:")
 	fmt.Printf("   Config file: %s\n", spinifexTomlPath)
@@ -778,7 +1038,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 // then generates configs with complete cluster topology.
 func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, clusterBind string,
-	port, expectedNodes int, formationTimeoutStr string, services []string) {
+	port, expectedNodes int, formationTimeoutStr string, services []string, networkConfig *formation.NetworkConfig) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid --formation-timeout: %v\n", err)
@@ -822,7 +1082,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		AdminSecretKey: bootstrapResult.AdminSecretKey,
 	}
 
-	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData))
+	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig)
 
 	// Include master key in formation server for distribution to joining nodes
 	fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey))
@@ -930,6 +1190,34 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		OVNSBAddr: "tcp:127.0.0.1:6642",
 	}
 
+	// Apply cluster-wide network config to init node's config
+	if networkConfig != nil {
+		configSettings.ExternalMode = networkConfig.ExternalMode
+		configSettings.ExternalDHCP = networkConfig.ExternalDHCP
+		configSettings.PoolName = networkConfig.PoolName
+		configSettings.PoolSource = networkConfig.PoolSource
+		configSettings.PoolStart = networkConfig.PoolStart
+		configSettings.PoolEnd = networkConfig.PoolEnd
+		configSettings.PoolGateway = networkConfig.PoolGateway
+		configSettings.PoolGatewayIP = networkConfig.PoolGatewayIP
+		configSettings.PoolPrefixLen = networkConfig.PoolPrefixLen
+		configSettings.PoolDNSServers = networkConfig.PoolDNSServers
+
+		configSettings.BootstrapAccountId = networkConfig.BootstrapAccountId
+		configSettings.BootstrapVpcId = networkConfig.BootstrapVpcId
+		configSettings.BootstrapSubnetId = networkConfig.BootstrapSubnetId
+		configSettings.BootstrapIgwId = networkConfig.BootstrapIgwId
+		configSettings.BootstrapCidr = networkConfig.BootstrapCidr
+		configSettings.BootstrapSubnetCidr = networkConfig.BootstrapSubnetCidr
+
+		// Auto-detect local network topology for per-node config
+		detected, err := admin.DetectNetwork()
+		if err == nil && detected.WAN != nil {
+			configSettings.ExternalIface = detected.WAN.Name
+			configSettings.WanBridge = detectedWanBridge(detected)
+		}
+	}
+
 	// Generate config files
 	configs := []admin.ConfigFile{
 		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
@@ -958,6 +1246,16 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	admin.CreateServiceDirectories(spxRoot)
 
+	// In production layout (running as root), fix ownership so the service user
+	// can read config and write data. SUDO_USER identifies the operator account.
+	if os.Getuid() == 0 {
+		sudoUser := os.Getenv("SUDO_USER")
+		if sudoUser != "" {
+			admin.ChownRecursive(configDir, sudoUser)
+			admin.ChownRecursive(spxRoot, sudoUser)
+		}
+	}
+
 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
 	time.Sleep(15 * time.Second)
@@ -980,6 +1278,11 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 }
 
 func runAdminJoin(cmd *cobra.Command, args []string) {
+	if os.Getuid() != 0 {
+		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
+		fmt.Fprintln(os.Stderr)
+	}
+
 	node, _ := cmd.Flags().GetString("node")
 	leaderHost, _ := cmd.Flags().GetString("host")
 	region, _ := cmd.Flags().GetString("region")
@@ -1056,9 +1359,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // formation server uses ephemeral self-signed cert
+		},
+	}
 
-	joinURL := fmt.Sprintf("http://%s/formation/join", leaderHost)
+	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
 	resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
@@ -1087,7 +1395,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Registered with formation server (%d/%d nodes joined)\n", joinResp.Joined, joinResp.Expected)
 
 	// Poll status until formation is complete
-	statusURL := fmt.Sprintf("http://%s/formation/status", leaderHost)
+	statusURL := fmt.Sprintf("https://%s/formation/status", leaderHost)
 	var statusResp formation.StatusResponse
 
 	for {
@@ -1119,6 +1427,30 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	fmt.Printf("✅ Cluster formation complete! (%d nodes)\n\n", statusResp.Expected)
 
+	// Fire telemetry after formation (now we know the cluster topology)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID: admin.ReadMachineID(),
+				Event:     "join",
+				Region:    region,
+				AZ:        az,
+				Node:      node,
+				Nodes:     statusResp.Expected,
+				BindIP:    bindIP,
+				Version:   Version,
+			})
+		})
+	}
+
 	// Extract credentials and CA from formation status
 	creds := statusResp.Credentials
 	if creds == nil {
@@ -1155,6 +1487,9 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ CA certificate received from leader: %s\n", caCertPath)
 
+	// Install CA certificate into system trust store
+	installCACertificate(caCertPath)
+
 	// Extract and write master key from formation server
 	if statusResp.MasterKey == "" {
 		fmt.Fprintf(os.Stderr, "❌ Error: formation server did not return master key\n")
@@ -1253,6 +1588,37 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
 	}
 
+	// Apply cluster-wide network config from leader
+	if statusResp.NetworkConfig != nil {
+		nc := statusResp.NetworkConfig
+		configSettings.ExternalMode = nc.ExternalMode
+		configSettings.ExternalDHCP = nc.ExternalDHCP
+		configSettings.PoolName = nc.PoolName
+		configSettings.PoolSource = nc.PoolSource
+		configSettings.PoolStart = nc.PoolStart
+		configSettings.PoolEnd = nc.PoolEnd
+		configSettings.PoolGateway = nc.PoolGateway
+		configSettings.PoolGatewayIP = nc.PoolGatewayIP
+		configSettings.PoolPrefixLen = nc.PoolPrefixLen
+		configSettings.PoolDNSServers = nc.PoolDNSServers
+
+		configSettings.BootstrapAccountId = nc.BootstrapAccountId
+		configSettings.BootstrapVpcId = nc.BootstrapVpcId
+		configSettings.BootstrapSubnetId = nc.BootstrapSubnetId
+		configSettings.BootstrapIgwId = nc.BootstrapIgwId
+		configSettings.BootstrapCidr = nc.BootstrapCidr
+		configSettings.BootstrapSubnetCidr = nc.BootstrapSubnetCidr
+
+		// Auto-detect local network topology for per-node config
+		if nc.ExternalMode != "" {
+			detected, err := admin.DetectNetwork()
+			if err == nil && detected.WAN != nil {
+				configSettings.ExternalIface = detected.WAN.Name
+				configSettings.WanBridge = detectedWanBridge(detected)
+			}
+		}
+	}
+
 	// Generate config files
 	configs := []admin.ConfigFile{
 		{Name: "spinifex.toml", Path: spinifexTomlPath, Template: spinifexTomlTemplate},
@@ -1282,6 +1648,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(dataDir)
 
+	// In production layout (running as root), fix ownership so the service user
+	// can read config and write data. SUDO_USER identifies the operator account.
+	if os.Getuid() == 0 {
+		sudoUser := os.Getenv("SUDO_USER")
+		if sudoUser != "" {
+			admin.ChownRecursive(configDir, sudoUser)
+			admin.ChownRecursive(dataDir, sudoUser)
+		}
+	}
+
 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
 	fmt.Printf("   Cluster: %s (%d nodes)\n", creds.ClusterName, len(statusResp.Nodes))
@@ -1370,7 +1746,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(&nodeConfig, nc)
 	if vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
-	} else if vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
+	} else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
 	}
 
@@ -1491,6 +1867,33 @@ func runAccountList(cmd *cobra.Command, args []string) {
 	}
 }
 
+func runCertRenew(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+
+	// Verify CA files exist.
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	if !admin.FileExists(caCertPath) || !admin.FileExists(caKeyPath) {
+		fmt.Fprintf(os.Stderr, "Error: CA files not found in %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	extraIPs, _ := cmd.Flags().GetStringSlice("extra-ip")
+	extraDNS, _ := cmd.Flags().GetStringSlice("extra-dns")
+
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("✅ Server certificate regenerated with current IPs and hostname")
+	fmt.Printf("   Certificate: %s\n", serverCertPath)
+	fmt.Println("\n⚠️  Restart awsgw and daemon services to pick up the new certificate.")
+}
+
 // writeBootstrapResult holds the admin credentials so callers can
 // write them to ~/.aws/credentials instead of the system credentials.
 type writeBootstrapResult struct {
@@ -1550,3 +1953,134 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey,
 
 	return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd)
 }
+
+// detectedWanBridge returns the OVS bridge name that OVN bridge-mappings should
+// use for WAN traffic. If the WAN interface is already an OVS bridge, returns
+// its name directly. If it's a Linux bridge, returns "br-ext" — setup-ovn.sh
+// will create this OVS bridge and link it to the Linux bridge via a veth pair.
+// If it's a physical NIC, returns "br-wan" as the default.
+func detectedWanBridge(detected *admin.DetectedNetwork) string {
+	if detected == nil || detected.WAN == nil {
+		return ""
+	}
+	name := detected.WAN.Name
+	if strings.HasPrefix(name, "br-") {
+		// Check if it's an OVS bridge (use directly) or Linux bridge (need br-ext)
+		if out, err := exec.Command("sudo", "ovs-vsctl", "br-exists", name).CombinedOutput(); err == nil && len(out) == 0 {
+			return name // OVS bridge — use directly
+		}
+		// Linux bridge — setup-ovn.sh creates br-ext + veth pair
+		return "br-ext"
+	}
+	return "br-wan"
+}
+
+// detectDNSServers auto-detects DNS servers from the host for the specified
+// interface. Uses resolvectl (systemd-resolved) first, then falls back to
+// /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.
+func detectDNSServers(iface string) []string {
+	// Try resolvectl for the specific link first (most reliable on modern systems)
+	if iface != "" {
+		out, err := exec.Command("resolvectl", "dns", iface).CombinedOutput()
+		if err == nil {
+			servers := parseDNSFromResolvectl(string(out))
+			if len(servers) > 0 {
+				return servers
+			}
+		}
+	}
+
+	// Try resolvectl global
+	out, err := exec.Command("resolvectl", "dns").CombinedOutput()
+	if err == nil {
+		servers := parseDNSFromResolvectl(string(out))
+		if len(servers) > 0 {
+			return servers
+		}
+	}
+
+	// Fall back to /etc/resolv.conf
+	data, err := os.ReadFile("/etc/resolv.conf")
+	if err == nil {
+		var servers []string
+		for line := range strings.SplitSeq(string(data), "\n") {
+			line = strings.TrimSpace(line)
+			if strings.HasPrefix(line, "nameserver ") {
+				ip := strings.TrimSpace(strings.TrimPrefix(line, "nameserver"))
+				// Skip localhost (systemd-resolved stub)
+				if ip != "127.0.0.53" && ip != "127.0.0.1" && net.ParseIP(ip) != nil {
+					servers = append(servers, ip)
+				}
+			}
+		}
+		if len(servers) > 0 {
+			if len(servers) > 3 {
+				servers = servers[:3]
+			}
+			return servers
+		}
+	}
+
+	// Fallback to well-known public DNS
+	return []string{"8.8.8.8", "1.1.1.1"}
+}
+
+// parseDNSFromResolvectl extracts IP addresses from resolvectl dns output.
+// Format: "Link 2 (enp0s3): 192.168.1.1 8.8.8.8 1.1.1.1"
+func parseDNSFromResolvectl(output string) []string {
+	var servers []string
+	for line := range strings.SplitSeq(output, "\n") {
+		// Find the colon separator, IPs come after it
+		_, after, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		fields := strings.FieldsSeq(after)
+		for f := range fields {
+			if net.ParseIP(f) != nil && f != "127.0.0.53" && f != "127.0.0.1" {
+				servers = append(servers, f)
+			}
+		}
+	}
+	if len(servers) > 3 {
+		servers = servers[:3]
+	}
+	return servers
+}
+
+// installCACertificate copies the Spinifex CA certificate into the system
+// trust store and runs update-ca-certificates so TLS clients (AWS CLI, etc.)
+// trust the self-signed gateway certificate without extra configuration.
+func installCACertificate(caPemPath string) {
+	if os.Getuid() != 0 {
+		return
+	}
+
+	const systemCertPath = "/usr/local/share/ca-certificates/spinifex-ca.crt"
+
+	data, err := os.ReadFile(caPemPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not read CA certificate %s: %v\n", caPemPath, err)
+		return
+	}
+
+	if err := os.MkdirAll(filepath.Dir(systemCertPath), 0755); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not create certificate directory: %v\n", err)
+		return
+	}
+
+	if err := os.WriteFile(systemCertPath, data, 0644); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not install CA certificate: %v\n", err)
+		return
+	}
+
+	cmd := exec.Command("update-ca-certificates")
+	cmd.Stdout = io.Discard
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: update-ca-certificates failed: %v\n", err)
+		return
+	}
+
+	fmt.Printf("✅ CA certificate installed to system trust store\n")
+}

cmd/spinifex/cmd/get.go

@@ -1,17 +1,18 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package cmd
 
@@ -138,6 +139,20 @@ func formatUptime(seconds int64) string {
 	return fmt.Sprintf("%dm", mins)
 }
 
+func formatRoles(resp types.NodeStatusResponse) string {
+	var roles []string
+	if resp.NATSRole != "" {
+		roles = append(roles, "nats:"+resp.NATSRole)
+	}
+	if resp.PredastoreRole != "" {
+		roles = append(roles, "predastore:"+resp.PredastoreRole)
+	}
+	if len(roles) == 0 {
+		return "-"
+	}
+	return strings.Join(roles, ",")
+}
+
 func formatMemGB(gb float64) string {
 	if gb >= 1.0 {
 		return fmt.Sprintf("%.1fGi", gb)
@@ -172,7 +187,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 
 	// Build table: union of config-known nodes + NATS responders
 	tableData := pterm.TableData{
-		{"NAME", "STATUS", "IP", "REGION", "AZ", "UPTIME", "VMs", "SERVICES"},
+		{"NAME", "STATUS", "ROLES", "IP", "REGION", "AZ", "UPTIME", "VMs", "SERVICES"},
 	}
 
 	// Collect all node names: config + responded (union)
@@ -195,6 +210,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 			tableData = append(tableData, []string{
 				resp.Node,
 				resp.Status,
+				formatRoles(resp),
 				resp.Host,
 				resp.Region,
 				resp.AZ,
@@ -206,6 +222,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 			tableData = append(tableData, []string{
 				name,
 				"NotReady",
+				"-",
 				nodeCfg.Host,
 				nodeCfg.Region,
 				nodeCfg.AZ,

cmd/spinifex/cmd/get_test.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatRoles(t *testing.T) {
+	tests := []struct {
+		name string
+		resp types.NodeStatusResponse
+		want string
+	}{
+		{
+			name: "no roles",
+			resp: types.NodeStatusResponse{},
+			want: "-",
+		},
+		{
+			name: "nats leader only",
+			resp: types.NodeStatusResponse{NATSRole: "leader"},
+			want: "nats:leader",
+		},
+		{
+			name: "predastore follower only",
+			resp: types.NodeStatusResponse{PredastoreRole: "follower"},
+			want: "predastore:follower",
+		},
+		{
+			name: "both roles",
+			resp: types.NodeStatusResponse{NATSRole: "leader", PredastoreRole: "follower"},
+			want: "nats:leader,predastore:follower",
+		},
+		{
+			name: "both leaders",
+			resp: types.NodeStatusResponse{NATSRole: "leader", PredastoreRole: "leader"},
+			want: "nats:leader,predastore:leader",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := formatRoles(tt.resp)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

cmd/spinifex/cmd/paths.go

@@ -2,18 +2,18 @@ package cmd
 
 import (
 	"os"
+	"os/user"
 	"path/filepath"
 )
 
 // DefaultConfigDir returns the default configuration directory.
-// Production: /etc/spinifex (when running as root or /etc/spinifex exists)
+// Production: /etc/spinifex (when running as root AND /etc/spinifex exists)
 // Development: ~/spinifex/config
 func DefaultConfigDir() string {
 	if isProductionLayout() {
 		return "/etc/spinifex"
 	}
-	homeDir, _ := os.UserHomeDir()
-	return filepath.Join(homeDir, "spinifex", "config")
+	return filepath.Join(realUserHomeDir(), "spinifex", "config")
 }
 
 // DefaultDataDir returns the default data directory.
@@ -23,8 +23,20 @@ func DefaultDataDir() string {
 	if isProductionLayout() {
 		return "/var/lib/spinifex"
 	}
+	return filepath.Join(realUserHomeDir(), "spinifex")
+}
+
+// realUserHomeDir returns the home directory of the real (non-sudo) user.
+// When running under sudo, SUDO_USER is set to the invoking user — resolve
+// their home directory so config/data land in the right place.
+func realUserHomeDir() string {
+	if sudoUser := os.Getenv("SUDO_USER"); sudoUser != "" {
+		if u, err := user.Lookup(sudoUser); err == nil {
+			return u.HomeDir
+		}
+	}
 	homeDir, _ := os.UserHomeDir()
-	return filepath.Join(homeDir, "spinifex")
+	return homeDir
 }
 
 // DefaultConfigFile returns the default path to spinifex.toml.
@@ -33,7 +45,8 @@ func DefaultConfigFile() string {
 }
 
 // isProductionLayout returns true when running in a production install.
-// Detected by: running as root, or /etc/spinifex directory exists.
+// Detected by: running as root AND /etc/spinifex directory already exists
+// (created by setup.sh during binary install).
 func isProductionLayout() bool {
 	if os.Getuid() == 0 {
 		if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {

cmd/spinifex/cmd/root.go

@@ -1,17 +1,18 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package cmd
 

cmd/spinifex/cmd/service.go

@@ -1,17 +1,18 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package cmd
 
@@ -24,6 +25,7 @@ import (
 	"github.com/mulgadc/predastore/s3"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/service"
+	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
@@ -243,13 +245,13 @@ var viperblockStartCmd = &cobra.Command{
 
 		accessKey := viper.GetString("access-key")
 		if accessKey != "" {
-			fmt.Println("Overwriting access-key:", accessKey)
+			fmt.Println("Overwriting access-key: ****")
 			nodeConfig.Predastore.AccessKey = accessKey
 		}
 
 		secretKey := viper.GetString("secret-key")
 		if secretKey != "" {
-			fmt.Println("Overwriting secret-key:", secretKey)
+			fmt.Println("Overwriting secret-key: ****")
 			nodeConfig.Predastore.SecretKey = secretKey
 		}
 
@@ -556,6 +558,7 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
 
+		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)
 
 		if err != nil {
@@ -694,13 +697,56 @@ var vpcdStartCmd = &cobra.Command{
 
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
+		// Map cluster-wide external pools to vpcd config
+		var extPools []vpcd.ExternalPoolConfig
+		for _, p := range clusterConfig.Network.ExternalPools {
+			extPools = append(extPools, vpcd.ExternalPoolConfig{
+				Name:       p.Name,
+				RangeStart: p.RangeStart,
+				RangeEnd:   p.RangeEnd,
+				Gateway:    p.Gateway,
+				GatewayIP:  p.GatewayIP,
+				PrefixLen:  p.PrefixLen,
+				DNSServers: p.DNSServers,
+				Region:     p.Region,
+				AZ:         p.AZ,
+			})
+		}
+
+		// Fallback chassis names derived from config node names. vpcd prefers
+		// discovering chassis from the OVN Southbound DB at startup; these are
+		// only used if that discovery fails.
+		var chassisNames []string
+		for nodeName := range clusterConfig.Nodes {
+			chassisNames = append(chassisNames, "chassis-"+nodeName)
+		}
+
+		var bootstrap *vpcd.BootstrapVPC
+		if clusterConfig.Bootstrap.VpcId != "" {
+			bootstrap = &vpcd.BootstrapVPC{
+				AccountID:  clusterConfig.Bootstrap.AccountID,
+				VpcId:      clusterConfig.Bootstrap.VpcId,
+				SubnetId:   clusterConfig.Bootstrap.SubnetId,
+				IgwId:      clusterConfig.Bootstrap.IgwId,
+				Cidr:       clusterConfig.Bootstrap.Cidr,
+				SubnetCidr: clusterConfig.Bootstrap.SubnetCidr,
+			}
+		}
+
 		svc, err := service.New("vpcd", &vpcd.Config{
-			NatsHost:  nodeConfig.NATS.Host,
-			NatsToken: nodeConfig.NATS.ACL.Token,
-			OVNNBAddr: nodeConfig.VPCD.OVNNBAddr,
-			OVNSBAddr: nodeConfig.VPCD.OVNSBAddr,
-			BaseDir:   nodeConfig.BaseDir,
-			Debug:     false,
+			NatsHost:          nodeConfig.NATS.Host,
+			NatsToken:         nodeConfig.NATS.ACL.Token,
+			OVNNBAddr:         nodeConfig.VPCD.OVNNBAddr,
+			OVNSBAddr:         nodeConfig.VPCD.OVNSBAddr,
+			BaseDir:           nodeConfig.BaseDir,
+			Debug:             false,
+			ExternalMode:      clusterConfig.Network.ExternalMode,
+			ExternalPools:     extPools,
+			ChassisNames:      chassisNames,
+			Bootstrap:         bootstrap,
+			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
+			WanBridge:         nodeConfig.VPCD.WanBridge,
+			BridgeMode:        nodeConfig.VPCD.BridgeMode,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)

cmd/spinifex/cmd/templates/spinifex.toml

@@ -2,6 +2,42 @@
 version = "1.0"
 epoch = 1
 node = "{{.Node}}"
+{{- if .ExternalMode}}
+
+[network]
+external_mode = "{{.ExternalMode}}"
+{{- if .ExternalDHCP}}
+external_dhcp = true
+{{- end}}
+
+[[network.external_pools]]
+name        = "{{.PoolName}}"
+{{- if .PoolSource}}
+source      = "{{.PoolSource}}"
+{{- end}}
+{{- if .PoolStart}}
+range_start = "{{.PoolStart}}"
+range_end   = "{{.PoolEnd}}"
+{{- end}}
+gateway     = "{{.PoolGateway}}"
+{{- if .PoolGatewayIP}}
+gateway_ip  = "{{.PoolGatewayIP}}"
+{{- end}}
+prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
+{{- if .PoolDNSServers}}
+dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
+{{- end}}
+{{- end}}
+{{- if .BootstrapVpcId}}
+
+[bootstrap]
+account_id  = "{{.BootstrapAccountId}}"
+vpc_id      = "{{.BootstrapVpcId}}"
+subnet_id   = "{{.BootstrapSubnetId}}"
+igw_id      = "{{.BootstrapIgwId}}"
+cidr        = "{{.BootstrapCidr}}"
+subnet_cidr = "{{.BootstrapSubnetCidr}}"
+{{- end}}
 
 [nodes.{{.Node}}]
 node = "{{.Node}}"
@@ -18,7 +54,11 @@ account_id = "{{.AccountID}}"
 host = "{{.BindIP}}:{{.Port}}"
 tlskey = "config/server.key"
 tlscert = "config/server.pem"
+{{- if .ExternalMode}}
+dev_networking = false
+{{- else}}
 dev_networking = true
+{{- end}}
 
 [nodes.{{.Node}}.awsgw]
 host = "{{.BindIP}}:9999"
@@ -41,6 +81,12 @@ shardwal = false
 [nodes.{{.Node}}.vpcd]
 ovn_nb_addr = "{{.OVNNBAddr}}"
 ovn_sb_addr = "{{.OVNSBAddr}}"
+{{- if .ExternalIface}}
+external_interface = "{{.ExternalIface}}"
+{{- end}}
+{{- if .WanBridge}}
+wan_bridge = "{{.WanBridge}}"
+{{- end}}
 
 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"
@@ -61,4 +107,4 @@ az = "{{.AZ}}"
 {{- if .Services}}
 services = [{{range $i, $s := .Services}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
-{{- end}}
+{{- end}}

cmd/spinifex/cmd/top.go

@@ -1,17 +1,18 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package cmd
 

cmd/spinifex/main.go

@@ -1,17 +1,18 @@
 /*
-Copyright © 2025 Mulga Defense Corporation
+Copyright © 2026 Mulga Defense Corporation
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-	http://www.apache.org/licenses/LICENSE-2.0
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package main
 

compute/geniso.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-# Linux
-#genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data
-
-# MacOS
-mkisofs -output seed.iso -volid cidata -joliet -rock meta-data user-data

compute/nbdkit.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-# Example to launch nbdkit
-
-./server/nbdkit -p 10810 --pidfile /tmp/vb-vol-1-efi.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=67108864 volume=vol-50c968e8dbcc74bd2-efi bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb/" host="https://127.0.0.1:8443" cache_size=1

compute/vm-nbd-daemon.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-qemu-system-aarch64 \
-   -nographic \
-   -M virt,highmem=off \
-   -accel hvf \
-   -cpu host \
-   -smp 4 \
-   -m 3000 \
-   -drive file=/Users/benduncan/qemu/QEMU_EFI.img,if=pflash,format=raw \
-   -device virtio-blk-pci,drive=debian,bootindex=1 \
-   -drive if=none,media=disk,id=debian,format=raw,file=nbd://192.168.64.5:10809/default \
-   -drive format=raw,file=nbd://192.168.64.5:10810/default,if=pflash \
-   -drive file=nbd://192.168.64.5:10811/default,format=raw,if=virtio \
-   -device qemu-xhci \
-   -device usb-kbd \
-   -device usb-tablet \
-   -device intel-hda \
-   -device hda-duplex \
-   -netdev user,id=net0,hostfwd=tcp::2222-:22 -device virtio-net-device,netdev=net0 \
-   -device virtio-rng-pci
-
-
-# ./server/nbdkit -p 10809 --pidfile /tmp/vb-vol-1.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=4294967296 volume=vol-e660ff0065ad21f74 bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb/" host="https://127.0.0.1:8443" cache_size=20
-
-# ./server/nbdkit -p 10810 --pidfile /tmp/vb-vol-1-efi.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=67108864 volume=vol-e660ff0065ad21f74-efi bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb/" host="https://127.0.0.1:8443" cache_size=1
-
-# ./server/nbdkit -p 10811 --pidfile /tmp/vb-vol-1-cloudinit.pid ./plugins/golang/examples/ramdisk/nbdkit-goramdisk-plugin.so -v -f size=1048576 volume=vol-e660ff0065ad21f74-cloudinit bucket=predastore region=ap-southeast-2 access_key="AKIAIOSFODNN7EXAMPLE" secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" base_dir="/tmp/vb/" host="https://127.0.0.1:8443" cache_size=1

docs/COMMANDS.md

@@ -54,11 +54,23 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured | **DONE** |
 | `spx admin join` | `--host` (required), `--node` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error) | **DONE** |
 
+### Version
+
+| Command | Flags | Prerequisites | Basic Logic | Status |
+|---------|-------|---------------|-------------|--------|
+| `spx version` | — | None | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) | **DONE** |
+
 ### Cluster Operations
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------|---------------|-------------|------------|--------|
-| `spx admin cluster shutdown` | — | Cluster must be running | Gracefully shuts down the entire cluster | 1. Shutdown running cluster<br>2. All nodes stop cleanly | **DONE** |
+| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Cluster must be running | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. | 1. Shutdown running cluster<br>2. All nodes stop cleanly<br>3. Force shutdown with unresponsive nodes<br>4. Dry-run prints plan | **DONE** |
+
+### Certificate Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
 
 ### Account Management
 
@@ -80,15 +92,17 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--subnet-id` (auto-creates ENI, assigns private IP), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination) | `--security-group-ids`, `--placement`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → if SubnetId provided, auto-creates ENI with private IP → cloud-init injects user-data/keys → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. Launch with subnet-id (auto-creates ENI)<br>6. Invalid instance type returns error | **DONE** |
-| `describe-instances` | `--instance-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes | **DONE** |
+| `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--subnet-id` (auto-creates ENI, assigns private IP), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName only — routes via spread or cluster strategy) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → if Placement.GroupName set, looks up strategy: spread → `distributeInstancesSpread()` (1 instance per node, atomic CAS reservation), cluster → `distributeInstancesCluster()` (pin all to single node); otherwise NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → if SubnetId provided, auto-creates ENI with private IP → cloud-init injects user-data/keys → on termination, removes instance from placement group → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. Launch with subnet-id (auto-creates ENI)<br>6. Invalid instance type returns error<br>7. Launch with spread placement group (1 per node)<br>8. Launch with cluster placement group (all on one node)<br>9. Insufficient capacity for placement group (error) | **DONE** |
+| `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:\*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list. Filters applied per-node before aggregation (reduces payload). Also applies to stopped/terminated instances via `describeInstancesFromKV()`. | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes<br>6. Filter by tag<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
 | `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
-| `reboot-instances` | — | `--instance-ids`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_reset` → instance reboots without stopping | 1. Reboot running instance<br>2. Reboot stopped instance (error)<br>3. Verify instance stays in running state after reboot | **NOT STARTED** (parser test exists) |
+| `reboot-instances` | `--instance-ids` | `--dry-run` | `run-instances` (instance must be running) | Gateway validates instance IDs → sends EC2InstanceCommand with `RebootInstance=true` via NATS `ec2.cmd.{instanceId}` → daemon validates instance is in StateRunning (returns IncorrectInstanceState if stopped) → issues QMP `system_reset` → instance reboots without stopping, stays in running state | 1. Reboot running instance<br>2. Reboot multiple instances<br>3. Reboot stopped instance (error: IncorrectInstanceState)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Verify instance stays in running state after reboot | **DONE** |
 | `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
 | `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data` | `--disable-api-termination`, `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must be stopped (in NATS KV) | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon loads stopped instance from JetStream KV → applies attribute change → writes back to KV → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Modify running instance (error: NotFound — running instances not in KV)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Instance not stopped (error: IncorrectInstanceState)<br>6. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>7. StateReason cleared on type change (recovery from capacity-unavailable)<br>8. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>9. No attribute set (error: InvalidParameterValue)<br>10. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest` (always returns latest), `--dry-run` | Instance must be running on a node | Gateway sends NATS `ec2.{instanceId}.GetConsoleOutput` (per-instance topic, routed to owning node) → daemon reads console log file from disk → returns last 64KB base64-encoded with timestamp. Always available regardless of serial console access setting (matches AWS behavior). | 1. Get output from running instance<br>2. Empty log file returns empty output<br>3. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`) return real values; unstored attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, `disableApiTermination`=false, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get default disableApiTermination<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | None | Gateway-only stub — returns `CpuCredits: "standard"` for each requested instance ID. No daemon round-trip. T-series credit mode is not persisted. | 1. Get credit spec for T-series instance<br>2. Multiple instance IDs | **DONE** |
 | `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |
 
@@ -97,7 +111,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format` (pem/ppk), `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreateKeyPair` → daemon generates SSH keypair → stores public key in Predastore S3 (`/bucket/ec2/{name}.pub`) → returns private key material (one-time) | 1. Create RSA key pair<br>2. Create ED25519 key pair<br>3. Duplicate key name (error: InvalidKeyPair.Duplicate)<br>4. Verify key material returned only on creation | **DONE** |
-| `describe-key-pairs` | `--key-names`, `--key-pair-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty | **DONE** |
+| `describe-key-pairs` | `--key-names`, `--key-pair-ids`, `--filters` (key-pair-id, key-name, fingerprint, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → applies filters → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 | `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | Key must exist | NATS `ec2.DeleteKeyPair` → daemon deletes public key from Predastore S3 → returns success | 1. Delete existing key pair<br>2. Delete non-existent key (idempotent, no error)<br>3. Verify key no longer in describe-key-pairs | **DONE** |
 | `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | None | NATS `ec2.ImportKeyPair` → daemon stores provided public key in Predastore S3 → returns key name and fingerprint | 1. Import valid SSH public key<br>2. Import invalid key material (error)<br>3. Import duplicate name (error)<br>4. Verify imported key usable with run-instances | **DONE** |
 
@@ -105,7 +119,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias) | `--executable-users`, `--filters`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds and Owners → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName) | **DONE** |
+| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias), `--filters` (name, state, architecture, image-id, is-public, owner-id, description, image-type, tag:\*) | `--executable-users`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds, Owners, and Filters → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName)<br>6. Filter by name wildcard<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `create-image` | `--instance-id`, `--name`, `--description`, `--tag-specifications` | `--no-reboot`, `--block-device-mappings`, `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.{instanceId}.CreateImage` (per-instance topic) → daemon extracts root volume → snapshots via `ebs.snapshot` NATS (running) or offline S3 copy (stopped) → creates AMI metadata in S3 (`{amiId}/config.json`) → stores tags → returns ami-ID. Duplicate AMI name validation enforced. | 1. Create image from running instance<br>2. Create image from stopped instance<br>3. Invalid instance ID (error)<br>4. Duplicate AMI name (error)<br>5. Verify new AMI appears in describe-images<br>6. Launch new instance from created AMI | **DONE** |
 | `register-image` | — | `--name`, `--description`, `--architecture`, `--root-device-name`, `--virtualization-type`, `--block-device-mappings`, `--boot-mode`, `--ena-support`, `--sriov-net-support`, `--tpm-support`, `--imds-support` | Image data must exist in S3 | Create AMI metadata entry in Predastore → assign ami-ID → set state to available → return ami-ID | 1. Register with valid metadata<br>2. Missing required name (error)<br>3. Verify registered image in describe-images | **NOT STARTED** |
 | `deregister-image` | — | `--image-id`, `--dry-run` | AMI must exist, no instances using it | Mark AMI metadata as deregistered in S3 → AMI no longer available for new launches → existing instances unaffected | 1. Deregister existing AMI<br>2. Deregister non-existent AMI (error)<br>3. Verify deregistered AMI not in describe-images<br>4. Existing instances from AMI still run | **NOT STARTED** |
@@ -119,13 +133,13 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata) | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value | **DONE** |
+| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata), `--filters` (volume-id, status, size, volume-type, attachment.instance-id, attachment.status, attachment.device, availability-zone, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → applies filters → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value<br>6. Filter by status, size, volume-type<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | Volume must exist | NATS `ec2.ModifyVolume` → daemon sends resize request to viperblock → NBD does not support live resize, instance must be stopped → returns modification state | 1. Increase volume size<br>2. Modify volume type<br>3. Decrease size (error - not supported)<br>4. Modify attached volume (requires stop/start) | **DONE** |
 | `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3 only), `--snapshot-id` (creates volume from snapshot) | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput`, `--tag-specifications` | Valid AZ configured via `spinifex init` | Gateway validates input → NATS `ec2.CreateVolume` → daemon generates vol-ID via viperblock → creates volume (empty or from snapshot) of specified size → persists config.json to Predastore S3 → returns vol-ID with state=available | 1. Create 80GB gp3 volume<br>2. Boundary sizes (1 GiB min, 16384 GiB max)<br>3. Invalid AZ (error)<br>4. Verify volume in describe-volumes<br>5. Unsupported volume type (error - only gp3)<br>6. Size out of range (error)<br>7. Create from snapshot | **DONE** |
 | `delete-volume` | `--volume-id` | `--dry-run` | Volume must exist and be detached (state=available) | Gateway validates vol- prefix → NATS `ec2.DeleteVolume` → daemon confirms state=available and no AttachedInstance → NATS `ebs.delete` to viperblockd (stops nbdkit/WAL) → deletes S3 objects under vol-id/, vol-id-efi/, vol-id-cloudinit/ → returns success | 1. Delete detached volume<br>2. Delete attached volume (error: VolumeInUse)<br>3. Delete non-existent volume (error: InvalidVolume.NotFound)<br>4. Verify volume gone from describe-volumes<br>5. Malformed volume ID (error: InvalidVolumeID.Malformed)<br>6. Double delete (idempotent NotFound) | **DONE** |
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (optional, auto-assigns `/dev/sd[f-p]`) | `--dry-run` | Volume must exist (available), instance must exist (running) | Gateway sends to `ec2.cmd.{instanceId}` → daemon validates volume (Predastore) → `ebs.mount` via NATS (viperblock starts NBD server) → QMP `blockdev-add` (nbd-{volId}) → QMP `device_add` (virtio-blk-pci, vdisk-{volId}) → three-phase rollback on failure → update EBSRequests + BlockDeviceMappings → persist to JetStream + Predastore → respond with VolumeAttachment | 1. Attach volume to running instance<br>2. Auto-assign device name<br>3. Attach already-attached volume (VolumeInUse)<br>4. Attach to non-existent instance (InvalidInstanceID.NotFound)<br>5. Attach to stopped instance (IncorrectInstanceState)<br>6. Volume not found (InvalidVolume.NotFound)<br>7. All device slots full (AttachmentLimitExceeded)<br>8. Volume persists across stop/start | **DONE** |
 | `detach-volume` | `--volume-id`, `--instance-id` (optional, resolved via DescribeVolumes), `--device` (optional cross-check), `--force` | `--dry-run` | Volume must be attached, instance must be running | Gateway resolves InstanceId if omitted (via DescribeVolumes) → sends to `ec2.cmd.{instanceId}` → daemon validates (running, attached, not boot/EFI/CloudInit, device match) → three-phase hot-unplug: QMP `device_del` (force continues on failure) → QMP `blockdev-del` (abort if fails, preserves state to prevent double-attach) → `ebs.unmount` via NATS (best-effort) → remove from EBSRequests + BlockDeviceMappings → update volume metadata to available → persist state → respond with VolumeAttachment (state=detaching) | 1. Detach with explicit InstanceId<br>2. Detach without InstanceId (gateway resolution)<br>3. Detach with correct --device cross-check<br>4. Missing VolumeId (InvalidParameterValue)<br>5. Volume not attached (IncorrectState)<br>6. Nonexistent volume (InvalidVolume.NotFound)<br>7. Nonexistent instance (InvalidInstanceID.NotFound)<br>8. Instance not running (IncorrectInstanceState)<br>9. Device mismatch (InvalidParameterValue)<br>10. Boot volume protection (OperationNotPermitted)<br>11. Force flag (continues past device_del failure)<br>12. Volume reusability (re-attach after detach) | **DONE** |
-| `describe-volume-status` | `--volume-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes | **DONE** |
+| `describe-volume-status` | `--volume-ids`, `--filters` (volume-id, volume-status.status, availability-zone) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → applies filters → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | None | Query pending/completed volume modifications → return modification state, progress, original/target size | 1. Check in-progress modification<br>2. Check completed modification<br>3. No modifications returns empty | **NOT STARTED** |
 
 ### EC2 - Snapshot Management
@@ -135,7 +149,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `create-snapshot` | `--volume-id`, `--description`, `--tag-specifications` | `--dry-run` | Volume must exist in Predastore | Gateway validates vol- prefix → NATS `ec2.CreateSnapshot` → daemon reads VolumeConfig from Predastore → generates snap-ID via viperblock → stores SnapshotConfig (metadata-only, points to source volume) as completed → returns Snapshot with state=completed, progress=100% | 1. Snapshot from valid volume<br>2. Missing volume ID (InvalidParameterValue)<br>3. Volume not found (InvalidVolume.NotFound)<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Snapshot with description<br>6. Snapshot with tag specifications | **DONE** |
 | `create-snapshots` | — | `--instance-specification`, `--description`, `--tag-specifications` | Instance must exist, instance-volume attachment tracking | Create snapshots of all volumes attached to instance → return list of snapshot IDs. Blocked: requires instance-volume attachment tracking to discover which volumes to snapshot. | 1. Snapshot all volumes on instance<br>2. Instance with no volumes | **NOT STARTED** |
 | `delete-snapshot` | `--snapshot-id` | `--dry-run` | Snapshot must exist | Gateway validates snap- prefix → NATS `ec2.DeleteSnapshot` → daemon verifies snapshot exists in Predastore → lists and deletes all objects under snapshot prefix → returns success | 1. Delete existing snapshot<br>2. Delete non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing snapshot ID (InvalidParameterValue)<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
-| `describe-snapshots` | `--snapshot-ids` | `--owner-ids`, `--filters`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → filters by snapshot ID if specified → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
+| `describe-snapshots` | `--snapshot-ids`, `--filters` (snapshot-id, status, volume-id, volume-size, owner-id, tag:\*) | `--owner-ids`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → applies filters → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed)<br>5. Filter by status, volume-id, volume-size<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
 | `copy-snapshot` | `--source-snapshot-id`, `--source-region`, `--description` | `--encrypted`, `--dry-run` | Source snapshot must exist | Gateway validates snap- prefix + source region → NATS `ec2.CopySnapshot` → daemon reads source SnapshotConfig → generates new snap-ID → copies metadata (preserves tags, description override) → stores as completed → returns new snapshot ID | 1. Copy within same region<br>2. Copy non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing source ID (InvalidParameterValue)<br>4. Missing source region (MissingParameter)<br>5. Copy preserves tags<br>6. Copy with description override | **DONE** |
 
 ### EC2 - Tags
@@ -188,15 +202,17 @@ Gateway routing and KV CRUD are wired up but no OVN/OVS integration exists. EIGW
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-egress-only-internet-gateway` | `--vpc-id`, `--tag-specifications` | `--client-token`, `--dry-run` | VPC must exist | Gateway validates VpcId → NATS `ec2.CreateEgressOnlyInternetGateway` → daemon generates eigw-ID → creates EgressOnlyIGWRecord in `spinifex-eigw` KV bucket with state=attached (always attached at creation, unlike IGW) → returns EgressOnlyInternetGateway with attachment | 1. Create with valid VpcId<br>2. Missing VpcId (error: MissingParameter)<br>3. Create with inline tags<br>4. Verify ID format eigw-{16hex} | **STARTED** (KV only, no OVN) |
 | `delete-egress-only-internet-gateway` | `--egress-only-internet-gateway-id` | `--dry-run` | EIGW must exist | NATS `ec2.DeleteEgressOnlyInternetGateway` → daemon verifies EIGW exists in KV → deletes from `spinifex-eigw` KV → returns ReturnCode=true. No detach required (unlike IGW). | 1. Delete existing EIGW<br>2. Delete non-existent EIGW (error)<br>3. Missing ID (error: MissingParameter) | **STARTED** (KV only, no OVN) |
-| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list | **STARTED** (KV only, no OVN) |
+| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids`, `--filters` (egress-only-internet-gateway-id, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → applies filters → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list<br>4. Unknown filter returns InvalidParameterValue | **STARTED** (KV only, no OVN) |
 
 ### EC2 - Placement Groups
 
+Placement groups are stored in `spinifex-placement-groups` JetStream KV bucket, keyed by `{accountID}.{groupName}`. All mutations use CAS (compare-and-swap) with 5-retry limit for optimistic concurrency. Supported strategies: **spread** (strict 1 instance per node) and **cluster** (all instances pinned to single node). Partition strategy is rejected. Internal NATS operations (ReserveSpreadNodes, FinalizeSpreadInstances, ReleaseSpreadNodes, RemoveInstanceFromPlacementGroup, ReserveClusterNode, FinalizeClusterInstances) support atomic multi-node coordination for RunInstances and TerminateInstances.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-placement-group` | — | `--group-name`, `--strategy` (cluster/spread/partition), `--partition-count`, `--spread-level`, `--tag-specifications` | None | NATS `ec2.CreatePlacementGroup` → daemon stores placement group in JetStream KV with strategy, state=available → return group metadata | 1. Create with cluster strategy<br>2. Create with spread strategy<br>3. Create with partition strategy + partition count<br>4. Duplicate name (error)<br>5. Invalid strategy (error) | **NOT STARTED** |
-| `delete-placement-group` | — | `--group-name`, `--dry-run` | Group must exist, no instances in group | NATS `ec2.DeletePlacementGroup` → daemon verifies group exists and is empty → delete from KV → return success | 1. Delete empty group<br>2. Delete group with instances (error)<br>3. Delete non-existent group (error) | **NOT STARTED** |
-| `describe-placement-groups` | — | `--group-names`, `--group-ids`, `--filters` | None | NATS `ec2.DescribePlacementGroups` → daemon lists groups from KV → return list with strategy and state | 1. List all groups<br>2. Filter by name<br>3. Filter by strategy | **NOT STARTED** |
+| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster) | `--partition-count`, `--spread-level` (hardcoded to "host"), `--tag-specifications`, `--dry-run` | None | Gateway validates GroupName + Strategy → NATS `ec2.CreatePlacementGroup` with `spinifex-workers` queue group → daemon generates pg-ID (pg-{16hex}) → stores PlacementGroupRecord in `spinifex-placement-groups` KV with state=available, SpreadLevel="host" for spread → rejects partition strategy (InvalidParameterValue) → rejects duplicate names (InvalidPlacementGroup.Duplicate) → returns PlacementGroup | 1. Create with spread strategy<br>2. Create with cluster strategy<br>3. Create with partition strategy (error: InvalidParameterValue)<br>4. Duplicate name (error: InvalidPlacementGroup.Duplicate)<br>5. Missing GroupName (error)<br>6. Missing Strategy (error) | **DONE** |
+| `delete-placement-group` | `--group-name` | `--dry-run` | Group must exist, no instances in group | NATS `ec2.DeletePlacementGroup` → daemon verifies group exists (InvalidPlacementGroup.Unknown) → counts instances in NodeInstances map → rejects if non-empty (InvalidPlacementGroup.InUse) → deletes from KV → returns success | 1. Delete empty group<br>2. Delete group with instances (error: InvalidPlacementGroup.InUse)<br>3. Delete non-existent group (error: InvalidPlacementGroup.Unknown) | **DONE** |
+| `describe-placement-groups` | `--group-names`, `--group-ids`, `--filters` (strategy, state, spread-level, group-name) | `--dry-run` | None | NATS `ec2.DescribePlacementGroups` → daemon lists all keys from `spinifex-placement-groups` KV → filters by GroupNames, GroupIds, or Filters → returns list with Strategy, State, SpreadLevel, GroupId | 1. List all groups<br>2. Filter by name<br>3. Filter by group ID<br>4. Filter by strategy<br>5. Filter by state<br>6. Empty result when no groups exist | **DONE** |
 
 ### EC2 - Dedicated Hosts
 
@@ -234,67 +250,76 @@ Gateway routing and KV CRUD are wired up but no OVN/OVS integration exists. EIGW
 
 ### EC2 - Elastic IP
 
+EIP operations are conditionally available — only registered when external IPAM is configured (pool mode). EIPs are stored in `spinifex-eip-allocations` KV bucket. Association publishes `vpc.add-nat`/`vpc.delete-nat` events to vpcd for OVN NAT configuration.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `allocate-address` | — | `--domain` (vpc), `--tag-specifications`, `--dry-run` | None | NATS `ec2.AllocateAddress` → daemon allocates IP from pool → stores in JetStream KV with eipalloc-ID → return AllocationId, PublicIp, Domain | 1. Allocate EIP<br>2. Verify in describe-addresses<br>3. Allocate without --domain uses default | **NOT STARTED** |
-| `release-address` | — | `--allocation-id`, `--dry-run` | EIP must exist and be disassociated | NATS `ec2.ReleaseAddress` → daemon verifies not associated → release IP back to pool → delete from KV → return success | 1. Release disassociated EIP<br>2. Release associated EIP (error)<br>3. Release non-existent (error) | **NOT STARTED** |
-| `associate-address` | — | `--allocation-id`, `--instance-id` or `--network-interface-id`, `--dry-run` | EIP and target must exist | NATS `ec2.AssociateAddress` → daemon links EIP to instance/ENI → configure OVS DNAT/SNAT rules → return eipassoc-ID | 1. Associate with instance<br>2. Re-associate (moves EIP)<br>3. Associate non-existent EIP (error) | **NOT STARTED** |
-| `disassociate-address` | — | `--association-id`, `--dry-run` | Association must exist | NATS `ec2.DisassociateAddress` → daemon unlinks EIP from instance/ENI → remove OVS DNAT/SNAT rules → return success | 1. Disassociate existing<br>2. Disassociate non-existent (error) | **NOT STARTED** |
-| `describe-addresses` | — | `--allocation-ids`, `--filters`, `--public-ips` | None | NATS `ec2.DescribeAddresses` → daemon lists EIPs from KV → return list with association info | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP | **NOT STARTED** |
+| `allocate-address` | `--public-ipv4-pool` (optional, specific IPAM pool), `--tag-specifications` | `--domain` (hardcoded vpc), `--dry-run` | EIP service must be initialized (external IPAM configured) | Gateway validates input → NATS `ec2.AllocateAddress` with `spinifex-workers` queue group → daemon allocates IP from external IPAM pool (named or best-matching) → creates EIPRecord with state=allocated in `spinifex-eip-allocations` KV → returns AllocationId, PublicIp, Domain=vpc, PublicIpv4Pool | 1. Allocate EIP from default pool<br>2. Allocate from specific pool<br>3. Pool exhausted (InsufficientAddressCapacity)<br>4. With tag specifications<br>5. Verify in describe-addresses | **DONE** |
+| `release-address` | `--allocation-id` | `--dry-run` | EIP must exist and be disassociated | Gateway validates AllocationId non-empty → NATS `ec2.ReleaseAddress` → daemon retrieves EIP record from KV → checks state != associated (InvalidAddressLocked if still associated) → releases IP back to IPAM pool → deletes KV record → returns success | 1. Release disassociated EIP<br>2. Release associated EIP (error: InvalidAddressLocked)<br>3. Release non-existent (error: InvalidAllocationID.NotFound)<br>4. Missing AllocationId (error: MissingParameter) | **DONE** |
+| `associate-address` | `--allocation-id`, `--network-interface-id` OR `--instance-id`, `--private-ip-address` (optional override) | `--dry-run`, `--allow-reassociation` | EIP and target ENI/instance must exist | Gateway validates AllocationId + one of NetworkInterfaceId/InstanceId required → NATS `ec2.AssociateAddress` → daemon retrieves EIP → resolves target ENI (by ID or instance lookup) → generates AssociationId → updates record state=associated with ENI/instance/VPC/private IP → publishes `vpc.add-nat` event to vpcd (OVN NAT configuration) → returns AssociationId | 1. Associate with instance<br>2. Associate with specific ENI<br>3. Associate with private IP override<br>4. Missing AllocationId (MissingParameter)<br>5. Missing target (MissingParameter)<br>6. Non-existent EIP (InvalidAllocationID.NotFound)<br>7. Non-existent ENI (InvalidNetworkInterfaceID.NotFound) | **DONE** |
+| `disassociate-address` | `--association-id` | `--dry-run` | Association must exist | Gateway validates AssociationId non-empty → NATS `ec2.DisassociateAddress` → daemon scans KV to find EIP by AssociationId → publishes `vpc.delete-nat` event to vpcd → clears association fields → reverts state to allocated → updates KV → returns success | 1. Disassociate existing<br>2. Non-existent association (InvalidAssociationID.NotFound)<br>3. Missing AssociationId (MissingParameter) | **DONE** |
+| `describe-addresses` | `--allocation-ids`, `--public-ips`, `--filters` (allocation-id, public-ip, instance-id, association-id, domain, tag:\*) | `--dry-run` | None | NATS `ec2.DescribeAddresses` → daemon scans all EIP records in account → filters by AllocationIds and/or PublicIps if specified → applies filters → returns Address list with allocation, association, PublicIp, PrivateIpAddress, NetworkInterfaceId, InstanceId, Domain | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP<br>4. Non-existent AllocationId returns error<br>5. Empty result when no EIPs<br>6. Filter by instance-id, domain<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
+| `describe-addresses-attribute` | `--allocation-ids` (optional filter) | `--attribute`, `--dry-run`, `--max-results`, `--next-token` | None | Gateway validates input → NATS `ec2.DescribeAddressesAttribute` with `spinifex-workers` queue group → daemon looks up EIP records by AllocationIds (or scans all account EIPs if none specified) → returns AddressAttribute list with AllocationId and PublicIp per entry. Unlike DescribeAddresses, returns empty list (not error) for non-existent AllocationIds. PtrRecord always nil (reverse-DNS not supported). | 1. List all EIP attributes<br>2. Filter by allocation IDs<br>3. Non-existent AllocationId returns empty (not error)<br>4. Nil input returns error | **DONE** |
 
 ### EC2 - NAT Gateway
 
+NAT gateways are stored in `spinifex-vpc-nat-gateways` KV bucket. Deleted gateways are moved to `spinifex-vpc-deleted-nat-gateways` with a 1-hour TTL for Terraform polling. SNAT rule coordination uses `vpc.add-nat-gateway` and `vpc.delete-nat-gateway` NATS events (published via route table service on CreateRoute and during DeleteNatGateway respectively).
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-nat-gateway` | — | `--subnet-id`, `--allocation-id`, `--connectivity-type`, `--tag-specifications` | Subnet must exist, EIP must be allocated (for public type) | NATS `ec2.CreateNatGateway` → daemon creates NAT gateway in JetStream KV with nat-ID → configure OVS SNAT rules → state=pending→available → return NatGateway | 1. Create public NAT gateway<br>2. Create private NAT gateway (no allocation-id)<br>3. Invalid subnet (error) | **NOT STARTED** |
-| `delete-nat-gateway` | — | `--nat-gateway-id`, `--dry-run` | NAT gateway must exist | NATS `ec2.DeleteNatGateway` → daemon marks as deleting → remove OVS SNAT rules → delete from KV → return NatGatewayId | 1. Delete existing NAT gateway<br>2. Delete non-existent (error) | **NOT STARTED** |
-| `describe-nat-gateways` | — | `--nat-gateway-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNatGateways` → daemon lists NAT gateways from KV → return list with state, subnet, addresses | 1. List all NAT gateways<br>2. Filter by ID<br>3. Filter by VPC ID | **NOT STARTED** |
+| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type` (public only), `--tag-specifications`, `--dry-run` | Subnet must exist, EIP must be allocated and unassociated | Gateway validates SubnetId + AllocationId (required) → NATS `ec2.CreateNatGateway` with `spinifex-workers` queue group → daemon verifies subnet exists in `spinifex-vpc-subnets` KV → verifies EIP exists in `spinifex-vpc-eips` KV and has no AssociationId (Resource.AlreadyAssociated if in use) → generates nat-ID → stores NatGatewayRecord in `spinifex-vpc-nat-gateways` KV with state=available → returns NatGateway with addresses | 1. Create NAT gateway (happy path)<br>2. Subnet not found (InvalidSubnetID.NotFound)<br>3. EIP not found (InvalidAllocationID.NotFound)<br>4. EIP already associated (Resource.AlreadyAssociated)<br>5. Missing SubnetId or AllocationId (MissingParameter) | **DONE** |
+| `delete-nat-gateway` | `--nat-gateway-id` | `--dry-run` | NAT gateway must exist | Gateway validates NatGatewayId (required) → NATS `ec2.DeleteNatGateway` → daemon retrieves record from KV → scans all route tables in VPC for routes targeting this NAT gateway → publishes `vpc.delete-nat-gateway` event per affected subnet (SNAT rule cleanup) → sets state=deleted → stores in `spinifex-vpc-deleted-nat-gateways` with 1-hour TTL → deletes from active KV bucket | 1. Delete existing NAT gateway<br>2. Delete non-existent (InvalidNatGatewayID.NotFound)<br>3. Deleted gateway visible when queried by ID (state=deleted)<br>4. Deleted gateway excluded from general list queries (TTL auto-expire) | **DONE** |
+| `describe-nat-gateways` | `--nat-gateway-ids`, `--filters` (vpc-id, state) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeNatGateways` → daemon scans `spinifex-vpc-nat-gateways` KV → applies NatGatewayIds filter and/or vpc-id/state filters → for specific IDs not found in active bucket, checks deleted bucket → returns InvalidNatGatewayID.NotFound if ID never existed → returns list with state, addresses, subnet, VPC | 1. List all NAT gateways<br>2. Filter by specific IDs<br>3. Filter by state<br>4. Filter by vpc-id<br>5. Non-existent ID (InvalidNatGatewayID.NotFound)<br>6. Deleted gateway visible by ID query | **DONE** |
 | `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | NAT gateway must exist (private type) | NATS `ec2.AssignPrivateNatGatewayAddress` → daemon assigns private IPs to NAT gateway → return assigned addresses | 1. Assign address<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 | `associate-nat-gateway-address` | — | `--nat-gateway-id`, `--allocation-ids` | NAT gateway and EIPs must exist | NATS `ec2.AssociateNatGatewayAddress` → daemon associates EIPs with NAT gateway → return association info | 1. Associate EIP<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 
 ### EC2 - VPC Networking (Core)
 
-Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, and routers. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket).
+Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI/SG CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, routers, and ACLs. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket). Security groups stored in `spinifex-vpc-security-groups` KV bucket with optimistic locking on rule updates.
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-vpc` | `--cidr-block`, `--tag-specifications` | `--instance-tenancy`, `--dry-run` | None | Gateway validates CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateVpc` → daemon allocates unique VNI from `spinifex-vpc-vni-counter` KV → generates vpc-ID → stores VPCRecord in `spinifex-vpc-vpcs` KV → publishes `vpc.create` event to vpcd (creates OVN logical switch) → returns VPC with state=available | 1. Create VPC with /16 CIDR<br>2. Invalid CIDR format (error: InvalidVpcRange)<br>3. CIDR prefix outside /16–/28 (error)<br>4. Verify in describe-vpcs<br>5. Tags applied at creation | **DONE** |
 | `delete-vpc` | `--vpc-id` | `--dry-run` | VPC must be empty (no subnets) | Gateway validates VpcId → NATS `ec2.DeleteVpc` → daemon checks for dependent subnets in `spinifex-vpc-subnets` KV → returns DependencyViolation if subnets exist → deletes from `spinifex-vpc-vpcs` KV → publishes `vpc.delete` event to vpcd (removes OVN resources) | 1. Delete empty VPC<br>2. Delete VPC with subnets (DependencyViolation)<br>3. Delete non-existent VPC (error) | **DONE** |
-| `describe-vpcs` | `--vpc-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error | **DONE** |
-| `modify-vpc-attribute` | — | `--vpc-id`, `--enable-dns-support`, `--enable-dns-hostnames` | VPC must exist | NATS `ec2.ModifyVpcAttribute` → daemon updates VPC attribute in KV → return success | 1. Enable DNS support<br>2. Enable DNS hostnames<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `describe-vpcs` | `--vpc-ids`, `--filters` (vpc-id, state, cidr-block, is-default, owner-id, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → applies filters → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error<br>4. Filter by state, cidr-block, is-default<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-vpc-attribute` | `--vpc-id`, `--enable-dns-hostnames`, `--enable-dns-support`, `--enable-network-address-usage-metrics` | `--dry-run` | VPC must exist | Gateway validates VpcId non-empty → NATS `ec2.ModifyVpcAttribute` → daemon retrieves VPC record from `spinifex-vpc-vpcs` KV → updates boolean flags if provided → updates KV with optimistic locking → returns empty output | 1. Enable DNS hostnames<br>2. Enable DNS support<br>3. Enable network address usage metrics<br>4. Missing VPC ID (MissingParameter)<br>5. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
+| `describe-vpc-attribute` | `--vpc-id`, `--attribute` (enableDnsHostnames, enableDnsSupport, enableNetworkAddressUsageMetrics) | `--dry-run` | VPC must exist | Gateway validates VpcId + Attribute non-empty → NATS `ec2.DescribeVpcAttribute` → daemon retrieves VPC record → returns single attribute as AttributeBooleanValue (one attribute per call, matches AWS behavior) → validates attribute name | 1. Get enableDnsHostnames<br>2. Get enableDnsSupport<br>3. Get enableNetworkAddressUsageMetrics<br>4. Invalid attribute name (InvalidParameterValue)<br>5. Missing VPC ID (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
 | `associate-vpc-cidr-block` | — | `--vpc-id`, `--cidr-block` | VPC must exist | NATS `ec2.AssociateVpcCidrBlock` → daemon adds secondary CIDR to VPC metadata → return association | 1. Add secondary CIDR<br>2. Overlapping CIDR (error)<br>3. Max CIDR blocks exceeded (error) | **NOT STARTED** |
 | `disassociate-vpc-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateVpcCidrBlock` → daemon removes secondary CIDR from VPC → return success | 1. Remove secondary CIDR<br>2. Remove primary CIDR (error) | **NOT STARTED** |
 | `create-subnet` | `--vpc-id`, `--cidr-block`, `--availability-zone`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates VpcId + CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateSubnet` → daemon verifies parent VPC exists → validates subnet CIDR is within VPC CIDR range → checks for overlapping subnets in same VPC → calculates available IPs (total hosts - 5 AWS-reserved: .0/.1/.2/.3/.255) → generates subnet-ID → stores SubnetRecord in `spinifex-vpc-subnets` KV → publishes `vpc.create-subnet` event to vpcd → defaults AZ from config if omitted | 1. Create subnet within VPC CIDR<br>2. Subnet CIDR outside VPC range (error: InvalidSubnetRange)<br>3. Overlapping subnet CIDRs (error: InvalidSubnetConflict)<br>4. CIDR prefix outside /16–/28 (error)<br>5. Missing VPC or CIDR (error: MissingParameter) | **DONE** |
 | `delete-subnet` | `--subnet-id` | `--dry-run` | Subnet must exist | NATS `ec2.DeleteSubnet` → daemon verifies subnet exists → deletes from `spinifex-vpc-subnets` KV → publishes `vpc.delete-subnet` event to vpcd | 1. Delete existing subnet<br>2. Delete non-existent subnet (error) | **DONE** |
-| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies vpc-id filter if present → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error | **DONE** |
-| `modify-subnet-attribute` | — | `--subnet-id`, `--map-public-ip-on-launch`, `--assign-ipv6-address-on-creation` | Subnet must exist | NATS `ec2.ModifySubnetAttribute` → daemon updates subnet attribute in KV → return success | 1. Enable auto-assign public IP<br>2. Missing subnet ID (error) | **NOT STARTED** |
+| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id, subnet-id, availability-zone, cidr-block, state, default-for-az, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies filters → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error<br>5. Filter by availability-zone, cidr-block, state<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-subnet-attribute` | `--subnet-id`, `--map-public-ip-on-launch` | `--assign-ipv6-address-on-creation`, `--dry-run` | Subnet must exist | Gateway validates SubnetId non-empty → NATS `ec2.ModifySubnetAttribute` → daemon retrieves Subnet record from `spinifex-vpc-subnets` KV → updates MapPublicIpOnLaunch flag if provided → updates KV with optimistic locking → returns empty output | 1. Enable auto-assign public IP<br>2. Disable auto-assign public IP<br>3. Missing SubnetId (MissingParameter)<br>4. Non-existent subnet (InvalidSubnetID.NotFound) | **DONE** |
 | `associate-subnet-cidr-block` | — | `--subnet-id`, `--ipv6-cidr-block` | Subnet must exist | NATS `ec2.AssociateSubnetCidrBlock` → daemon adds IPv6 CIDR to subnet → return association | 1. Add IPv6 CIDR<br>2. Missing subnet ID (error) | **NOT STARTED** |
 | `disassociate-subnet-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateSubnetCidrBlock` → daemon removes IPv6 CIDR from subnet → return success | 1. Remove IPv6 CIDR<br>2. Invalid association (error) | **NOT STARTED** |
-| `create-security-group` | — | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | VPC must exist | Create security group metadata → create default OVS flow rules (deny all inbound, allow all outbound) → assign sg-ID | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error)<br>3. Verify default rules | **NOT STARTED** |
-| `delete-security-group` | — | `--group-id` | SG must not be in use by instances | Verify no instances reference SG → remove OVS flow rules → delete metadata | 1. Delete unused SG<br>2. Delete SG in use (error) | **NOT STARTED** |
-| `describe-security-groups` | — | `--group-ids`, `--group-names`, `--filters`, `--max-results` | None | Query NATS KV for SG metadata → return SG list with rules | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name | **NOT STARTED** |
-| `authorize-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr`, `--source-group` | SG must exist | Add inbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Duplicate rule (idempotent) | **NOT STARTED** |
-| `authorize-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist | Add outbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR | **NOT STARTED** |
-| `revoke-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove inbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
-| `revoke-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove outbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
+| `create-security-group` | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates GroupName + VpcId required → NATS `ec2.CreateSecurityGroup` → daemon verifies VPC exists → checks duplicate name in same VPC → generates sg-ID → creates SecurityGroupRecord in `spinifex-vpc-security-groups` KV with default egress rule (allow all outbound 0.0.0.0/0), no ingress rules → publishes `vpc.create-sg` event to vpcd (OVN ACL configuration) → returns GroupId | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error: InvalidGroup.Duplicate)<br>3. Verify default egress rule (allow all)<br>4. Verify no default ingress rules<br>5. Missing GroupName (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound)<br>7. With tag specifications | **DONE** |
+| `delete-security-group` | `--group-id` | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.DeleteSecurityGroup` → daemon retrieves SG from KV → deletes from `spinifex-vpc-security-groups` KV → publishes `vpc.delete-sg` event to vpcd → returns success | 1. Delete existing SG<br>2. Delete non-existent SG (error: InvalidGroup.NotFound)<br>3. Missing GroupId (MissingParameter) | **DONE** |
+| `describe-security-groups` | `--group-ids`, `--filters` (vpc-id, group-name, group-id, description, ip-permission.cidr, tag:\*) | `--group-names`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSecurityGroups` → daemon scans all SG records in account → applies filters (AND logic across filter names, OR across values) → returns SecurityGroup list with ingress/egress rules → returns error for non-existent requested GroupIds | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name<br>4. Filter by GroupId<br>5. Non-existent GroupId (InvalidGroup.NotFound)<br>6. Empty result when no SGs<br>7. Filter by description, ip-permission.cidr<br>8. Unknown filter returns InvalidParameterValue | **DONE** |
+| `authorize-security-group-ingress` | `--group-id`, `--ip-permissions` (IpProtocol, FromPort, ToPort, IpRanges/CidrIp, UserIdGroupPairs/GroupId) | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.AuthorizeSecurityGroupIngress` → daemon retrieves SG → converts IpPermissions to SGRule objects → appends to IngressRules → updates KV with optimistic locking (revision check) → publishes `vpc.update-sg` event to vpcd → returns true | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Allow from source security group<br>4. Missing GroupId (MissingParameter)<br>5. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `authorize-security-group-egress` | `--group-id`, `--ip-permissions` | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.AuthorizeSecurityGroupEgress` → daemon retrieves SG → appends rules to EgressRules → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR<br>3. Missing GroupId (MissingParameter)<br>4. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-ingress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupIngress` → daemon retrieves SG → removes matching rules from IngressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-egress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupEgress` → daemon retrieves SG → removes matching rules from EgressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `create-internet-gateway` | `--tag-specifications` | `--dry-run` | None | NATS `ec2.CreateInternetGateway` → daemon generates igw-ID → creates IGWRecord in `spinifex-igw` KV bucket with state=available (not attached) → returns InternetGateway with empty Attachments | 1. Create IGW<br>2. Verify in describe-internet-gateways<br>3. Tags applied at creation | **DONE** |
 | `attach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW and VPC must exist | NATS `ec2.AttachInternetGateway` → daemon validates IGW exists and is not already attached (ResourceAlreadyAssociated) → sets VpcId on record → state=attached → publishes `vpc.igw-attach` event to vpcd (creates OVN external switch, gateway port, SNAT rules) | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (ResourceAlreadyAssociated)<br>3. Non-existent IGW (error)<br>4. Missing params (MissingParameter) | **DONE** |
 | `detach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW must be attached to specified VPC | NATS `ec2.DetachInternetGateway` → daemon validates IGW exists and is attached to specified VPC (GatewayNotAttached if wrong VPC) → clears VpcId → state=available → publishes `vpc.igw-detach` event to vpcd (removes OVN resources) | 1. Detach attached IGW<br>2. Detach from wrong VPC (GatewayNotAttached)<br>3. Detach unattached IGW (error) | **DONE** |
 | `delete-internet-gateway` | `--internet-gateway-id` | `--dry-run` | IGW must be detached | NATS `ec2.DeleteInternetGateway` → daemon validates IGW exists and VpcId is empty (DependencyViolation if attached) → deletes from `spinifex-igw` KV | 1. Delete detached IGW<br>2. Delete attached IGW (DependencyViolation)<br>3. Delete non-existent (error) | **DONE** |
-| `describe-internet-gateways` | `--internet-gateway-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info | **DONE** |
+| `describe-internet-gateways` | `--internet-gateway-ids`, `--filters` (internet-gateway-id, attachment.vpc-id, attachment.state, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → applies filters → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info<br>4. Filter by attachment.vpc-id, attachment.state<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 
 ### EC2 - Route Tables
 
+Route tables are stored in `spinifex-vpc-route-tables` JetStream KV bucket, keyed by `{accountID}.{routeTableId}`. Each route table has a Routes array (with local route pre-populated) and an Associations array. All operations validate VPC consistency. When a route targeting a NAT gateway is created, `vpc.add-nat-gateway` events are published per associated subnet for vpcd SNAT rule creation.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-route-table` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateRouteTable` → daemon creates route table in KV with rtb-ID → add default local route for VPC CIDR → return route table | 1. Create route table<br>2. Verify default local route<br>3. Missing VPC ID (error) | **NOT STARTED** |
-| `delete-route-table` | — | `--route-table-id`, `--dry-run` | Route table must exist and not be main route table | NATS `ec2.DeleteRouteTable` → daemon verifies not main table → delete from KV → return success | 1. Delete non-main route table<br>2. Delete main route table (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
-| `describe-route-tables` | — | `--route-table-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeRouteTables` → daemon lists route tables from KV → return tables with routes | 1. List all route tables<br>2. Filter by VPC | **NOT STARTED** |
-| `create-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` or `--egress-only-internet-gateway-id` | Route table and target must exist | NATS `ec2.CreateRoute` → daemon adds route entry → configure OVS flow for destination CIDR → update metadata | 1. Add internet route via IGW<br>2. Add route to NAT gateway<br>3. Add route to EIGW<br>4. Conflicting route (error) | **NOT STARTED** |
-| `delete-route` | — | `--route-table-id`, `--destination-cidr-block` | Route must exist | NATS `ec2.DeleteRoute` → daemon removes route entry → remove OVS flow → update metadata | 1. Delete existing route<br>2. Delete non-existent route (error)<br>3. Delete local route (error) | **NOT STARTED** |
-| `replace-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` | Route must exist | NATS `ec2.ReplaceRoute` → daemon replaces route target → update OVS flow → update metadata | 1. Replace IGW route with NAT<br>2. Replace non-existent route (error) | **NOT STARTED** |
-| `associate-route-table` | — | `--route-table-id`, `--subnet-id` or `--gateway-id` | Route table and target must exist | NATS `ec2.AssociateRouteTable` → daemon links route table to subnet/gateway → return rtbassoc-ID | 1. Associate with subnet<br>2. Missing route table ID (error) | **NOT STARTED** |
-| `disassociate-route-table` | — | `--association-id` | Association must exist, not main table association | NATS `ec2.DisassociateRouteTable` → daemon unlinks route table → return success | 1. Disassociate existing<br>2. Disassociate main table (error) | **NOT STARTED** |
+| `create-route-table` | `--vpc-id` | `--tag-specifications`, `--dry-run` | VPC must exist | Gateway validates VpcId (required) → NATS `ec2.CreateRouteTable` with `spinifex-workers` queue group → daemon verifies VPC exists in `spinifex-vpc-vpcs` KV → retrieves VPC CIDR → generates rtb-ID → creates RouteTableRecord with IsMain=false, local route pre-populated (destination=VPC CIDR, gateway="local"), empty associations → stores in `spinifex-vpc-route-tables` KV → returns RouteTable | 1. Create route table with local route (happy path)<br>2. VPC not found (InvalidVpcID.NotFound)<br>3. Missing VpcId (MissingParameter) | **DONE** |
+| `delete-route-table` | `--route-table-id` | `--dry-run` | Route table must exist, not main, no subnet associations | Gateway validates RouteTableId (required) → NATS `ec2.DeleteRouteTable` → daemon retrieves record → rejects if IsMain=true (DependencyViolation) → rejects if has non-main subnet associations (DependencyViolation) → deletes from KV → returns success | 1. Delete custom route table<br>2. Delete main route table (DependencyViolation)<br>3. Delete table with subnet associations (DependencyViolation)<br>4. Delete non-existent (InvalidRouteTableID.NotFound) | **DONE** |
+| `describe-route-tables` | `--route-table-ids`, `--filters` (vpc-id, route-table-id, association.main, association.route-table-association-id, association.subnet-id, route.destination-cidr-block, route.gateway-id) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeRouteTables` → daemon scans all keys in `spinifex-vpc-route-tables` KV → filters by RouteTableIds if specified → applies all provided filters (AND logic) → returns error for non-existent requested IDs → converts records to EC2 RouteTable objects | 1. List all route tables<br>2. Filter by vpc-id<br>3. Filter by association.main (true/false)<br>4. Non-existent ID (InvalidRouteTableID.NotFound)<br>5. Empty result when no route tables | **DONE** |
+| `create-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id`, `--nat-gateway-id` | `--egress-only-internet-gateway-id`, `--vpc-peering-connection-id`, `--dry-run` | Route table and target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.CreateRoute` → daemon retrieves route table → checks for duplicate destination (RouteAlreadyExists) → validates exactly one of GatewayId/NatGatewayId provided → if GatewayId: verifies IGW exists in `spinifex-igw` KV and is attached to same VPC → if NatGatewayId: verifies NAT GW exists in `spinifex-vpc-nat-gateways` KV and is in same VPC → for NAT GW routes, publishes `vpc.add-nat-gateway` event per associated subnet → appends RouteRecord (state=active, origin=CreateRoute) → updates KV → returns Return=true | 1. Add IGW route (happy path)<br>2. Duplicate destination (RouteAlreadyExists)<br>3. IGW not found (InvalidInternetGatewayID.NotFound)<br>4. NAT GW not found (InvalidNatGatewayID.NotFound)<br>5. Gateway not in same VPC (InvalidParameterValue)<br>6. Missing RouteTableId or DestinationCidrBlock (MissingParameter) | **DONE** |
+| `delete-route` | `--route-table-id`, `--destination-cidr-block` | `--dry-run` | Route must exist | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.DeleteRoute` → daemon retrieves route table → finds matching route → rejects if gateway is "local" (InvalidParameterValue — local routes cannot be deleted) → removes route from Routes array → updates KV → returns success | 1. Delete IGW route (happy path, local route remains)<br>2. Delete local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. Missing parameters (MissingParameter) | **DONE** |
+| `replace-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` | `--nat-gateway-id`, `--dry-run` | Route must exist, new target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock + GatewayId (required) → NATS `ec2.ReplaceRoute` → daemon retrieves route table → finds matching route → rejects if local route → verifies IGW exists and is attached to same VPC → updates route GatewayId, clears NatGatewayId, sets state=active → updates KV → returns success | 1. Replace route target (IGW swap)<br>2. Replace local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. IGW not found (InvalidInternetGatewayID.NotFound) | **DONE** |
+| `associate-route-table` | `--route-table-id`, `--subnet-id` | `--gateway-id`, `--dry-run` | Route table and subnet must exist in same VPC | Gateway validates RouteTableId + SubnetId (required) → NATS `ec2.AssociateRouteTable` → daemon retrieves route table → verifies subnet exists and is in same VPC → scans ALL route tables in VPC to check subnet isn't already explicitly associated (Resource.AlreadyAssociated) → generates rtbassoc-ID → appends AssociationRecord (Main=false) → updates KV → returns AssociationId with state=associated | 1. Associate subnet (happy path)<br>2. Duplicate subnet association (Resource.AlreadyAssociated)<br>3. Subnet not found (InvalidSubnetID.NotFound)<br>4. Subnet in different VPC (InvalidParameterValue)<br>5. Missing parameters (MissingParameter) | **DONE** |
+| `disassociate-route-table` | `--association-id` | `--dry-run` | Association must exist, not main table association | Gateway validates AssociationId (required) → NATS `ec2.DisassociateRouteTable` → daemon scans ALL route tables in account to find matching AssociationId → rejects if Main=true (InvalidParameterValue — main associations are immutable) → removes association from Associations array → updates KV → returns success | 1. Disassociate subnet (happy path)<br>2. Disassociate main table (InvalidParameterValue)<br>3. Association not found (InvalidAssociationID.NotFound) | **DONE** |
+| `replace-route-table-association` | `--association-id`, `--route-table-id` | `--dry-run` | Association and new route table must exist | Gateway validates AssociationId + RouteTableId (required) → NATS `ec2.ReplaceRouteTableAssociation` → daemon verifies new route table exists → scans all route tables to find matching AssociationId → if Main=true: atomically swaps IsMain flag between old and new table → two-phase commit: removes association from old table, creates new AssociationId on new table → compensating write on failure (logs CRITICAL if compensation fails) → returns new AssociationId with state=associated | 1. Move subnet between tables (happy path)<br>2. Replace main table association (swaps IsMain flags)<br>3. Association not found (InvalidAssociationID.NotFound)<br>4. Route table not found (InvalidRouteTableID.NotFound) | **DONE** |
 
 ### EC2 - VPC Peering
 
@@ -374,7 +399,7 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `describe-network-interfaces` | `--network-interface-ids`, `--filters` (subnet-id, vpc-id, attachment.instance-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeNetworkInterfaces` → daemon lists all keys from `spinifex-vpc-enis` KV → applies filters (subnet-id, vpc-id, attachment.instance-id) → filters by ENI IDs if specified → returns error for non-existent requested IDs | 1. List all ENIs<br>2. Filter by subnet-id<br>3. Filter by vpc-id<br>4. Filter by attachment.instance-id<br>5. Non-existent ENI returns error | **DONE** |
 | `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | ENI and instance must exist | Internal operation used by `run-instances --subnet-id`. Marks ENI as "in-use", sets instance attachment, generates attachment ID. No standalone API endpoint yet. | 1. Attach ENI to instance<br>2. Already attached (error)<br>3. Missing device index (error) | **NOT STARTED** (internal only) |
 | `detach-network-interface` | — | `--attachment-id`, `--force` | ENI must be attached | Internal operation used by terminate-instances. Marks ENI as "available", clears attachment info. No standalone API endpoint yet. | 1. Detach ENI<br>2. Force detach<br>3. Not attached (error) | **NOT STARTED** (internal only) |
-| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--source-dest-check`, `--groups` | ENI must exist | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon updates ENI attributes in KV → return success | 1. Modify description<br>2. Toggle source/dest check<br>3. Update security groups | **NOT STARTED** |
+| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--groups` | ENI must exist; at least one attribute required | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon validates ENI exists → updates description and/or security groups in `spinifex-vpc-enis` KV (optimistic locking) → return success | 1. Modify description<br>2. Update security groups<br>3. No attributes (InvalidParameterValue)<br>4. Non-existent ENI (error) | **DONE** |
 | `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` or `--secondary-private-ip-address-count` | ENI must exist | NATS `ec2.AssignPrivateIpAddresses` → daemon allocates IPs from subnet pool → assign to ENI → return assigned IPs | 1. Assign specific IPs<br>2. Assign by count<br>3. Cannot specify both (error) | **NOT STARTED** |
 | `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | IPs must be assigned to ENI | NATS `ec2.UnassignPrivateIpAddresses` → daemon releases IPs → return success | 1. Unassign secondary IP<br>2. Unassign primary IP (error) | **NOT STARTED** |
 
@@ -407,19 +432,6 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `list-snapshot-blocks` | — | `--snapshot-id`, `--max-results`, `--next-token` | Snapshot must exist | List all non-empty blocks in snapshot with tokens and sizes | 1. List blocks of populated snapshot<br>2. List blocks of empty snapshot | **NOT STARTED** |
 | `list-changed-blocks` | — | `--second-snapshot-id`, `--first-snapshot-id`, `--max-results` | Snapshots must exist | Compare two snapshots → return list of differing block indexes | 1. Compare parent and child snapshots<br>2. Compare unrelated snapshots | **NOT STARTED** |
 
-### S3 (via Predastore)
-
-Currently S3 operations are handled directly by Predastore. Consider moving control/data plane to Spinifex format for AWS gateway integration.
-
-| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
-|---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `s3 mb` (CreateBucket) | — | `s3://bucket-name` | None | NATS `s3.createbucket` → Predastore creates bucket → return success | 1. Create valid bucket<br>2. Duplicate bucket name (error)<br>3. Invalid bucket name (error) | **NOT STARTED** (Predastore handles directly) |
-| `s3 rb` (DeleteBucket) | — | `s3://bucket-name`, `--force` | Bucket must exist | NATS `s3.deletebucket` → verify bucket empty (or force) → Predastore deletes bucket | 1. Delete empty bucket<br>2. Delete non-empty bucket (error)<br>3. Force delete non-empty bucket | **NOT STARTED** |
-| `s3 ls` (ListBuckets/Objects) | — | `s3://bucket/prefix`, `--recursive` | None | NATS `s3.listobjects` → Predastore lists objects with prefix → return list | 1. List all buckets<br>2. List objects in bucket<br>3. List with prefix filter | **NOT STARTED** |
-| `s3 cp` (PutObject/GetObject) | — | `source dest`, `--recursive`, `--acl` | Bucket must exist | NATS `s3.putobject`/`s3.getobject` → Predastore stores/retrieves with Reed-Solomon encoding | 1. Upload single file<br>2. Download single file<br>3. Recursive upload/download<br>4. Large file (multipart) | **NOT STARTED** |
-| `s3 rm` (DeleteObject) | — | `s3://bucket/key`, `--recursive` | Object must exist | NATS `s3.deleteobject` → Predastore deletes object shards | 1. Delete single object<br>2. Recursive delete<br>3. Delete non-existent (idempotent) | **NOT STARTED** |
-| `s3 sync` | — | `source dest`, `--delete`, `--exclude`, `--include` | Bucket must exist | Compare source and dest → upload changed/new files → optionally delete removed files | 1. Sync local dir to S3<br>2. Sync with --delete<br>3. Sync with exclude filter | **NOT STARTED** |
-
 ### IAM - User Management
 
 All IAM operations are account-scoped. Users, policies, and access keys are namespaced per account. The gateway extracts the account ID from the authenticated access key via `c.Locals("sigv4.accountId")` and passes it to all IAM service methods. Root user (account `000000000000`) bypasses policy evaluation entirely.
@@ -475,5 +487,140 @@ The gateway evaluates IAM policies on every request (except root user bypass). P
 | Explicit deny | Deny statements override Allow (deny wins) | **DONE** |
 | Wildcard matching | `*` matches all, `ec2:*` matches all EC2, `ec2:Describe*` matches prefix | **DONE** |
 | Account-scoped evaluation | Policies resolved from caller's account context | **DONE** |
-| EC2 action mapping | 54 EC2 actions mapped to IAM policy strings | **DONE** |
+| EC2 action mapping | All registered EC2 actions mapped to IAM policy strings | **DONE** |
 | IAM action mapping | 16 IAM actions mapped to IAM policy strings | **DONE** |
+
+### IAM - Roles
+
+IAM roles are required for instance profiles, cross-account access, and service-linked permissions. Without roles, workloads must use static access keys. Roles should be stored in a `spinifex-iam-roles` KV bucket keyed by `{accountID}.{roleName}`. Trust policies (AssumeRolePolicyDocument) define who can assume the role.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-role` | — | `--role-name`, `--assume-role-policy-document`, `--path`, `--description`, `--max-session-duration`, `--permissions-boundary`, `--tags` | None | IAMService.CreateRole(accountID, input) → validate RoleName + AssumeRolePolicyDocument (required) → validate trust policy JSON → generate AROA-prefixed RoleId → store RoleRecord in `spinifex-iam-roles` KV at `{accountID}.{roleName}` → ARN: `arn:aws:iam::{accountID}:role/{path}{roleName}` → return Role | 1. Create with valid trust policy<br>2. Malformed trust policy (MalformedPolicyDocument)<br>3. Duplicate name (EntityAlreadyExists)<br>4. Missing RoleName (InvalidInput)<br>5. With path and description | **NOT STARTED** |
+| `get-role` | — | `--role-name` | Role must exist | IAMService.GetRole(accountID, input) → reads from KV at `{accountID}.{roleName}` → returns Role with ARN, Path, AssumeRolePolicyDocument, CreateDate | 1. Get existing role<br>2. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-roles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListRoles(accountID, input) → prefix-scans KV for `{accountID}.*` → filters by path prefix → returns role list scoped to caller's account | 1. List all roles<br>2. List with path prefix<br>3. Empty list<br>4. Account isolation | **NOT STARTED** |
+| `delete-role` | — | `--role-name` | Role must exist, no attached policies, no instance profiles | IAMService.DeleteRole(accountID, input) → verify no attached policies (DeleteConflict) → verify not in any instance profile (DeleteConflict) → delete from KV | 1. Delete role with no attachments<br>2. Delete role with policies (DeleteConflict)<br>3. Delete role in instance profile (DeleteConflict)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-role` | — | `--role-name`, `--description`, `--max-session-duration` | Role must exist | IAMService.UpdateRole(accountID, input) → update description and/or session duration in KV → return success | 1. Update description<br>2. Update max session duration<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-assume-role-policy` | — | `--role-name`, `--policy-document` | Role must exist | IAMService.UpdateAssumeRolePolicy(accountID, input) → validate trust policy JSON → update AssumeRolePolicyDocument in KV → return success | 1. Update trust policy<br>2. Malformed policy (MalformedPolicyDocument)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `attach-role-policy` | — | `--role-name`, `--policy-arn` | Role and policy must exist | IAMService.AttachRolePolicy(accountID, input) → verify policy exists → add ARN to role's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent role (NoSuchEntity)<br>4. Non-existent policy (NoSuchEntity) | **NOT STARTED** |
+| `detach-role-policy` | — | `--role-name`, `--policy-arn` | Policy must be attached | IAMService.DetachRolePolicy(accountID, input) → remove ARN from role's AttachedPolicies | 1. Detach attached policy<br>2. Not attached (NoSuchEntity)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-role-policies` | — | `--role-name`, `--path-prefix`, `--max-items`, `--marker` | Role must exist | IAMService.ListAttachedRolePolicies(accountID, input) → reads role's AttachedPolicies list → returns ARNs and names | 1. List attached policies<br>2. No policies (empty list)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Instance Profiles
+
+Instance profiles are containers for IAM roles that allow EC2 instances to assume a role and obtain temporary credentials. Required for `run-instances --iam-instance-profile`. Stored in `spinifex-iam-instance-profiles` KV bucket keyed by `{accountID}.{profileName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-instance-profile` | — | `--instance-profile-name`, `--path`, `--tags` | None | IAMService.CreateInstanceProfile(accountID, input) → validate name (required) → generate AIPA-prefixed InstanceProfileId → store in `spinifex-iam-instance-profiles` KV with empty Roles array → ARN: `arn:aws:iam::{accountID}:instance-profile/{path}{name}` → return InstanceProfile | 1. Create profile<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-instance-profile` | — | `--instance-profile-name` | Profile must exist | IAMService.GetInstanceProfile(accountID, input) → reads from KV → returns InstanceProfile with Roles array | 1. Get existing profile<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-instance-profiles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListInstanceProfiles(accountID, input) → prefix-scans KV → returns list | 1. List all profiles<br>2. Empty list | **NOT STARTED** |
+| `list-instance-profiles-for-role` | — | `--role-name`, `--max-items`, `--marker` | Role must exist | IAMService.ListInstanceProfilesForRole(accountID, input) → scan all profiles → filter where Roles contains role name → return list | 1. List profiles for role<br>2. Role not in any profile (empty) | **NOT STARTED** |
+| `delete-instance-profile` | — | `--instance-profile-name` | Profile must exist, Roles must be empty | IAMService.DeleteInstanceProfile(accountID, input) → verify Roles empty (DeleteConflict if not) → delete from KV | 1. Delete empty profile<br>2. Delete profile with role (DeleteConflict)<br>3. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-role-to-instance-profile` | — | `--instance-profile-name`, `--role-name` | Both must exist, max 1 role per profile | IAMService.AddRoleToInstanceProfile(accountID, input) → verify profile exists → verify role exists → check Roles array is empty (LimitExceeded if role already present) → append role → update KV | 1. Add role to empty profile<br>2. Profile already has role (LimitExceeded)<br>3. Non-existent profile (NoSuchEntity)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `remove-role-from-instance-profile` | — | `--instance-profile-name`, `--role-name` | Role must be in profile | IAMService.RemoveRoleFromInstanceProfile(accountID, input) → remove role from Roles array → update KV | 1. Remove role<br>2. Role not in profile (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Groups
+
+Groups provide a way to attach policies to multiple users. Stored in `spinifex-iam-groups` KV bucket keyed by `{accountID}.{groupName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-group` | — | `--group-name`, `--path` | None | IAMService.CreateGroup(accountID, input) → validate GroupName (required) → generate AGPA-prefixed GroupId → store in KV → return Group | 1. Create group<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-group` | — | `--group-name` | Group must exist | IAMService.GetGroup(accountID, input) → reads group from KV → returns Group with member Users list | 1. Get existing group<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-groups` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListGroups(accountID, input) → prefix-scans KV → returns list | 1. List all groups<br>2. Empty list | **NOT STARTED** |
+| `delete-group` | — | `--group-name` | Group must exist, no members, no attached policies | IAMService.DeleteGroup(accountID, input) → verify no members → verify no attached policies → delete from KV | 1. Delete empty group<br>2. Delete group with members (DeleteConflict)<br>3. Delete group with policies (DeleteConflict)<br>4. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-user-to-group` | — | `--group-name`, `--user-name` | Both must exist | IAMService.AddUserToGroup(accountID, input) → verify group + user exist → add user to group's Members list → idempotent | 1. Add user<br>2. Already a member (idempotent)<br>3. Non-existent group (NoSuchEntity)<br>4. Non-existent user (NoSuchEntity) | **NOT STARTED** |
+| `remove-user-from-group` | — | `--group-name`, `--user-name` | User must be in group | IAMService.RemoveUserFromGroup(accountID, input) → remove user from Members list → update KV | 1. Remove member<br>2. Not a member (NoSuchEntity) | **NOT STARTED** |
+| `list-groups-for-user` | — | `--user-name`, `--max-items`, `--marker` | User must exist | IAMService.ListGroupsForUser(accountID, input) → scan all groups → filter where Members contains user → return list | 1. List groups for user<br>2. User in no groups (empty) | **NOT STARTED** |
+| `attach-group-policy` | — | `--group-name`, `--policy-arn` | Group and policy must exist | IAMService.AttachGroupPolicy(accountID, input) → verify policy exists → add ARN to group's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent group (NoSuchEntity) | **NOT STARTED** |
+| `detach-group-policy` | — | `--group-name`, `--policy-arn` | Policy must be attached | IAMService.DetachGroupPolicy(accountID, input) → remove ARN from AttachedPolicies | 1. Detach policy<br>2. Not attached (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-group-policies` | — | `--group-name`, `--path-prefix`, `--max-items`, `--marker` | Group must exist | IAMService.ListAttachedGroupPolicies(accountID, input) → reads group's AttachedPolicies → returns ARNs and names | 1. List attached policies<br>2. No policies (empty) | **NOT STARTED** |
+
+### STS (Security Token Service)
+
+STS provides temporary credentials and identity verification. `GetCallerIdentity` is called by every AWS SDK/CLI invocation. `AssumeRole` is required once IAM roles are implemented. STS needs its own gateway service endpoint (`sts` service in the `supportedServices` map) with a dedicated action map.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `get-caller-identity` | — | *(no flags)* | Valid SigV4 credentials | Gateway extracts accountID and username from authenticated request → returns Account, Arn (`arn:aws:iam::{accountID}:user/{userName}` or `arn:aws:sts::{accountID}:assumed-role/{roleName}/{sessionName}`), UserId. Note: a Spinifex-custom version exists at `spinifex.GetCallerIdentity` but needs a proper STS endpoint. | 1. Get identity for IAM user<br>2. Get identity for assumed role<br>3. Get identity for root user<br>4. Invalid credentials (AuthFailure) | **NOT STARTED** |
+| `assume-role` | — | `--role-arn`, `--role-session-name`, `--duration-seconds`, `--policy`, `--external-id`, `--serial-number`, `--token-code`, `--tags`, `--transitive-tag-keys` | Role must exist, caller must be allowed by trust policy | Gateway validates RoleArn + RoleSessionName (required) → looks up role → evaluates trust policy (AssumeRolePolicyDocument) against caller → generates temporary credentials (AccessKeyId with ASIA prefix, SecretAccessKey, SessionToken) → credentials expire after DurationSeconds (default 3600, max 43200) → return Credentials + AssumedRoleUser (ARN, AssumedRoleId) | 1. Assume role with valid trust policy<br>2. Trust policy denies caller (AccessDenied)<br>3. Non-existent role (NoSuchEntity)<br>4. Expired session token rejected at SigV4<br>5. Custom duration<br>6. Session tags | **NOT STARTED** |
+| `get-session-token` | — | `--duration-seconds`, `--serial-number`, `--token-code` | Valid IAM user credentials (not role) | Gateway validates caller is IAM user (not assumed role) → generates temporary credentials (ASIA prefix) → session inherits user's permissions → return Credentials | 1. Get token for IAM user<br>2. Duration within range<br>3. Reject if caller is assumed role | **NOT STARTED** |
+
+### ELBv2 (Elastic Load Balancing v2 — Application Load Balancer)
+
+All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key format: `lb.{lbId}`, `tg.{tgId}`, `listener.{listenerId}`. The data plane uses a system-managed ALB VM running HAProxy, launched automatically during CreateLoadBalancer. HAProxy configuration is pushed via NATS (`elbv2.alb.{lbId}.config` topic) whenever listeners or targets change. Agent readiness is polled via `elbv2.alb.{lbId}.ping`.
+
+#### ELBv2 - Load Balancers
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-load-balancer` | `--name`, `--subnets`, `--security-groups`, `--scheme` (internet-facing/internal), `--tags`, `--ip-address-type` (ipv4 only) | `--type` (hardcoded application), `--customer-owned-ipv4-pool`, `--dry-run` | None (subnets optional) | Gateway validates Name (required) → NATS `elbv2.CreateLoadBalancer` with `spinifex-workers` queue group → daemon validates scheme (internet-facing or internal, InvalidScheme if other) → checks duplicate name in account (DuplicateLoadBalancerName) → generates LB ID → builds DNS name `{name}-{lbId}.{region}.elb.spinifex.local` → if subnets provided, creates one ENI per subnet with `spinifex:managed-by=elbv2` tag (rolls back on failure) → if system AMI available, launches t3.nano ALB VM with cloud-init running `/usr/local/bin/alb-agent` → state=provisioning → background goroutine polls agent ping for ~5min → on agent ready: state=active, on timeout: state=failed → if no VM launcher: state=active immediately → persists to KV → returns LoadBalancer with ARN, DNSName, State | 1. Create with name (happy path)<br>2. Internal scheme<br>3. Invalid scheme (InvalidScheme)<br>4. Duplicate name (DuplicateLoadBalancerName)<br>5. Missing name (MissingParameter)<br>6. With tags<br>7. With subnets (ENI creation) | **DONE** |
+| `delete-load-balancer` | `--load-balancer-arn` | `--dry-run` | LB must exist | Gateway validates LoadBalancerArn (required) → NATS `elbv2.DeleteLoadBalancer` → daemon verifies LB exists and belongs to account → terminates ALB VM (async background) → deletes all listeners for this LB → deletes system-managed ENIs → deletes LB record from KV → returns success | 1. Delete existing LB<br>2. Not found (LoadBalancerNotFound)<br>3. Cascades listener cleanup<br>4. ENI cleanup | **DONE** |
+| `describe-load-balancers` | `--load-balancer-arns`, `--names` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeLoadBalancers` → daemon lists all LB records from KV → filters by account ID → applies ARN filter and/or Name filter → returns LoadBalancer list with State, DNSName, Scheme, VpcId, AvailabilityZones | 1. List all LBs<br>2. Filter by name<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Target Groups
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-target-group` | `--name`, `--protocol` (HTTP), `--port` (default 80), `--vpc-id`, `--target-type` (instance only), `--health-check-protocol`, `--health-check-port`, `--health-check-path` (default /), `--health-check-interval-seconds` (default 30), `--health-check-timeout-seconds` (default 5), `--healthy-threshold-count` (default 5), `--unhealthy-threshold-count` (default 2), `--matcher` (HttpCode, default 200), `--tags` | `--health-check-enabled`, `--protocol-version`, `--ip-address-type`, `--dry-run` | None | Gateway validates Name (required) → NATS `elbv2.CreateTargetGroup` → daemon checks duplicate name in same VPC (DuplicateTargetGroupName) → generates TG ID → creates HealthCheckConfig with provided values or defaults → stores record with empty Targets array → returns TargetGroup with ARN | 1. Create with defaults<br>2. Custom health check<br>3. Duplicate name (DuplicateTargetGroupName)<br>4. Missing name (MissingParameter) | **DONE** |
+| `delete-target-group` | `--target-group-arn` | `--dry-run` | TG must exist, not referenced by listener | Gateway validates TargetGroupArn (required) → NATS `elbv2.DeleteTargetGroup` → daemon verifies TG exists → scans all listeners to check if any DefaultActions reference this TG (TargetGroupInUse if referenced) → deletes from KV | 1. Delete TG<br>2. TG in use by listener (TargetGroupInUse)<br>3. Not found (TargetGroupNotFound) | **DONE** |
+| `describe-target-groups` | `--target-group-arns`, `--names`, `--load-balancer-arn` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeTargetGroups` → daemon lists all TG records → filters by account → applies ARN/Name filters → if LoadBalancerArn specified, lists listeners for that LB and includes only TGs referenced by those listeners → returns TargetGroup list | 1. List all TGs<br>2. Filter by LB ARN (via listener lookup)<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Targets
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `register-targets` | `--target-group-arn`, `--targets` (Id=instance-id, Port=optional override) | `--dry-run` | TG must exist, instances should exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.RegisterTargets` → daemon fetches TG → deduplicates against existing targets (key: `{id}:{port}`, idempotent) → resolves instance ID → private IP via VPC ENI lookup → adds target with state=initial → persists to KV → calls `pushConfigForTargetGroup()` to regenerate HAProxy config on all LBs using this TG → returns success | 1. Register targets<br>2. Idempotent re-registration<br>3. TG not found (TargetGroupNotFound)<br>4. Missing ARN or targets (MissingParameter) | **DONE** |
+| `deregister-targets` | `--target-group-arn`, `--targets` (Id, Port) | `--dry-run` | TG must exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.DeregisterTargets` → daemon fetches TG → builds removal set (key: `{id}:{port}`) → filters Targets array to keep only non-matching → persists to KV → calls `pushConfigForTargetGroup()` to update HAProxy config → returns success | 1. Deregister targets<br>2. TG not found (TargetGroupNotFound)<br>3. Missing parameters (MissingParameter) | **DONE** |
+| `describe-target-health` | `--target-group-arn`, `--targets` (optional filter to specific targets) | `--include` | TG must exist | Gateway validates TargetGroupArn (required) → NATS `elbv2.DescribeTargetHealth` → daemon fetches TG → builds filter set from Targets param (if specified) → for each target, returns TargetHealthDescription with Target (Id, Port) and TargetHealth (State, Description). Note: health state is stored in records, not actively polled — HAProxy runs its own independent health checks. | 1. Describe all targets<br>2. Filter to specific targets<br>3. TG not found (TargetGroupNotFound) | **DONE** |
+
+#### ELBv2 - Listeners
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-listener` | `--load-balancer-arn`, `--default-actions` (Type=forward, TargetGroupArn), `--protocol` (HTTP, default), `--port` (default 80) | `--ssl-policy`, `--certificates`, `--alpn-policy`, `--mutual-authentication`, `--dry-run` | LB must exist | Gateway validates LoadBalancerArn + DefaultActions (required) → NATS `elbv2.CreateListener` → daemon verifies LB exists → checks for duplicate port on same LB (DuplicateListener) → generates listener ID → builds ARN → stores ListenerRecord in KV → calls `pushConfig(lb)` to regenerate HAProxy config (creates frontend for listener port, backend for target group with health checks) → returns Listener with ARN, Protocol, Port, DefaultActions | 1. Create listener<br>2. Duplicate port (DuplicateListener)<br>3. LB not found (LoadBalancerNotFound)<br>4. Graceful handling when NATS unavailable | **DONE** |
+| `delete-listener` | `--listener-arn` | `--dry-run` | Listener must exist | Gateway validates ListenerArn (required) → NATS `elbv2.DeleteListener` → daemon verifies listener exists and belongs to account → deletes from KV → fetches parent LB → calls `pushConfig(lb)` to regenerate HAProxy config without this listener → returns success | 1. Delete listener<br>2. Not found (ListenerNotFound) | **DONE** |
+| `describe-listeners` | `--load-balancer-arn`, `--listener-arns` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeListeners` → daemon lists listeners → if LoadBalancerArn specified, filters to that LB → applies ARN filter → account isolation → returns Listener list | 1. Filter by LB ARN<br>2. Account isolation | **DONE** |
+
+#### ELBv2 - Not Implemented
+
+| Feature | Description | Priority | Status |
+|---------|-------------|----------|--------|
+| Listener rules | Path-based and hostname-based routing (CreateRule, DeleteRule, DescribeRules, ModifyRule) | High | **NOT STARTED** |
+| HTTPS/TLS termination | SSL certificates, SSL policies, ALPN | High | **NOT STARTED** |
+| Modify operations | ModifyLoadBalancerAttributes, ModifyTargetGroup, ModifyTargetGroupAttributes, ModifyListener | Medium | **NOT STARTED** |
+| Connection draining | Deregistration delay for graceful target removal | Medium | **NOT STARTED** |
+| Stickiness | Session affinity / sticky sessions on target groups | Medium | **NOT STARTED** |
+| Active health checking | API-driven health state updates (currently HAProxy-only) | Medium | **NOT STARTED** |
+| Target types | IP and Lambda target types (only instance supported) | Low | **NOT STARTED** |
+| Access logs | S3 access log delivery | Low | **NOT STARTED** |
+| WAF integration | AWS WAF association for web application firewall | Low | **NOT STARTED** |
+
+### CloudWatch (Basic Monitoring)
+
+Basic CloudWatch metrics support is required for `monitor-instances`/`unmonitor-instances` to be meaningful. Daemon nodes should publish CPU, disk, and network metrics for running instances. Metrics stored in a time-series-friendly structure in JetStream KV or dedicated NATS subjects.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `put-metric-data` | — | `--namespace`, `--metric-data` (MetricName, Value, Unit, Timestamp, Dimensions) | None | Gateway validates Namespace + MetricData (required) → NATS `cloudwatch.PutMetricData` → daemon stores metric data points in time-series KV (key: `{namespace}.{metricName}.{dimensionHash}`) → return success | 1. Put single metric<br>2. Put with dimensions (InstanceId)<br>3. Missing namespace (MissingParameter) | **NOT STARTED** |
+| `get-metric-statistics` | — | `--namespace`, `--metric-name`, `--start-time`, `--end-time`, `--period`, `--statistics` (Average, Sum, Min, Max, SampleCount), `--dimensions` | None | Gateway validates required fields → NATS `cloudwatch.GetMetricStatistics` → daemon queries time-series data → aggregates by period → returns Datapoints with requested statistics | 1. Get CPU utilization over 1 hour<br>2. Get with dimensions (InstanceId)<br>3. No data returns empty | **NOT STARTED** |
+| `list-metrics` | — | `--namespace`, `--metric-name`, `--dimensions`, `--recently-active` | None | NATS `cloudwatch.ListMetrics` → daemon scans metric namespaces → returns Metrics list with Namespace, MetricName, Dimensions | 1. List all metrics<br>2. Filter by namespace<br>3. Filter by metric name | **NOT STARTED** |
+| `describe-alarms` | — | `--alarm-names`, `--alarm-name-prefix`, `--state-value`, `--action-prefix` | None | NATS `cloudwatch.DescribeAlarms` → daemon lists alarms from KV → return MetricAlarms with State, Threshold, ComparisonOperator | 1. List all alarms<br>2. Filter by state (OK, ALARM, INSUFFICIENT_DATA) | **NOT STARTED** |
+| `put-metric-alarm` | — | `--alarm-name`, `--namespace`, `--metric-name`, `--statistic`, `--period`, `--evaluation-periods`, `--threshold`, `--comparison-operator`, `--alarm-actions`, `--dimensions` | None | NATS `cloudwatch.PutMetricAlarm` → daemon stores alarm config in KV → alarm evaluation runs on metric publish → state transitions trigger alarm actions | 1. Create alarm<br>2. Update existing alarm<br>3. Missing required fields | **NOT STARTED** |
+| `delete-alarms` | — | `--alarm-names` | Alarms must exist | NATS `cloudwatch.DeleteAlarms` → daemon deletes alarm records from KV → return success | 1. Delete existing alarm<br>2. Non-existent alarm (ResourceNotFound) | **NOT STARTED** |
+
+### Auto Scaling
+
+Auto Scaling manages instance fleets based on scaling policies. Depends on Launch Templates (EC2) and optionally ELBv2 target groups for health-based scaling. Stored in `spinifex-autoscaling` KV bucket.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-auto-scaling-group` | — | `--auto-scaling-group-name`, `--launch-template` (Id/Name, Version), `--min-size`, `--max-size`, `--desired-capacity`, `--vpc-zone-identifier` (subnet IDs), `--target-group-arns`, `--health-check-type` (EC2/ELB), `--health-check-grace-period`, `--tags` | Launch template must exist | NATS `autoscaling.CreateAutoScalingGroup` → daemon validates launch template → stores ASG record in KV → launches DesiredCapacity instances using template → registers with target groups if specified → starts health check loop → return success | 1. Create ASG with launch template<br>2. Scale to desired capacity<br>3. Missing launch template (ValidationError)<br>4. Min > Max (ValidationError) | **NOT STARTED** |
+| `update-auto-scaling-group` | — | `--auto-scaling-group-name`, `--min-size`, `--max-size`, `--desired-capacity`, `--launch-template`, `--health-check-type`, `--health-check-grace-period` | ASG must exist | NATS `autoscaling.UpdateAutoScalingGroup` → daemon updates ASG config in KV → if DesiredCapacity changed, scale up/down accordingly → return success | 1. Update desired capacity<br>2. Update launch template<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-auto-scaling-group` | — | `--auto-scaling-group-name`, `--force-delete` | ASG must exist | NATS `autoscaling.DeleteAutoScalingGroup` → if force: terminate all instances → deregister from target groups → delete from KV → return success. Without force: reject if instances > 0 | 1. Delete empty ASG<br>2. Force delete with instances<br>3. Delete with instances (ResourceInUse) | **NOT STARTED** |
+| `describe-auto-scaling-groups` | — | `--auto-scaling-group-names`, `--filters`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingGroups` → daemon lists ASG records → return list with Instances, DesiredCapacity, MinSize, MaxSize | 1. List all ASGs<br>2. Filter by name | **NOT STARTED** |
+| `set-desired-capacity` | — | `--auto-scaling-group-name`, `--desired-capacity`, `--honor-cooldown` | ASG must exist | NATS `autoscaling.SetDesiredCapacity` → daemon updates desired count → scale up (launch) or scale down (terminate oldest) → return success | 1. Scale up<br>2. Scale down<br>3. Exceeds max (ValidationError) | **NOT STARTED** |
+| `describe-auto-scaling-instances` | — | `--instance-ids`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingInstances` → daemon lists all ASG-managed instances → return list with ASG name, lifecycle state, health status | 1. List all ASG instances<br>2. Filter by instance ID | **NOT STARTED** |
+| `put-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name`, `--policy-type` (TargetTrackingScaling, StepScaling, SimpleScaling), `--target-tracking-configuration`, `--scaling-adjustment`, `--cooldown` | ASG must exist | NATS `autoscaling.PutScalingPolicy` → daemon stores policy in KV → links to CloudWatch alarms for metric-based triggers → return PolicyARN | 1. Create target tracking policy<br>2. Create step scaling policy<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name` | Policy must exist | NATS `autoscaling.DeleteScalingPolicy` → daemon deletes policy → remove CloudWatch alarm links → return success | 1. Delete existing policy<br>2. Non-existent policy | **NOT STARTED** |

docs/creating-accounts/README.md

@@ -32,19 +32,13 @@ Spinifex supports multi-tenant account isolation. Each account gets its own IAM
 ## Create Account
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
-## Verify
-
-```bash
-aws sts get-caller-identity
-```
-
 ## Troubleshooting
 
-## Credentials not working
+## Credentials Not Working
 
 Verify the AWS CLI configuration files exist and contain the correct profile:
 

docs/creating-an-environment/README.md

@@ -1,120 +0,0 @@
----
-title: "Creating an Environment"
-description: "End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances."
-category: "Environments"
-tags:
-  - environment
-  - vpc
-  - ec2
-  - ssh
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Debian Cloud Images"
-    url: "https://cloud.debian.org/images/cloud/bookworm/latest/"
----
-
-# Creating an Environment
-
-> End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-This guide walks through the complete workflow for creating a production-ready Spinifex environment.
-
-**Steps:**
-
-1. Create a user account
-2. Import or create SSH keys
-3. Create a VPC and subnet
-4. Select and import an AMI
-5. Launch EC2 instances
-6. Connect via SSH
-7. Monitor instance health
-
-## Instructions
-
-## 1. Import SSH Key
-
-```bash
-aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub
-```
-
-## 2. Create VPC
-
-```bash
-aws ec2 create-vpc --cidr-block 10.200.0.0/16
-export SPINIFEX_VPC="vpc-XXX"
-
-aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24
-export SPINIFEX_SUBNET="subnet-XXX"
-```
-
-## 3. Import AMI
-
-```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
-export SPINIFEX_AMI="ami-XXX"
-```
-
-## 4. Launch Instance
-
-```bash
-aws ec2 run-instances \
-  --image-id $SPINIFEX_AMI \
-  --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
-```
-
-## 5. Connect via SSH
-
-```bash
-ps auxw | grep $INSTANCE_ID
-# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
-
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```
-
-## Troubleshooting
-
-## Instance stuck in pending
-
-Check the Spinifex daemon and QEMU logs for boot errors:
-
-```bash
-ls ~/spinifex/logs/
-cat ~/spinifex/logs/spinifex.log
-```
-
-Verify the AMI exists and architecture matches your host:
-
-```bash
-aws ec2 describe-images --image-ids $SPINIFEX_AMI
-```
-
-## SSH connection refused
-
-cloud-init takes 30-60 seconds to configure networking and SSH after boot. Wait and retry.
-
-Verify the SSH port forwarding is active:
-
-```bash
-ps auxw | grep hostfwd
-```
-
-Ensure you're using the correct key and port:
-
-```bash
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```

docs/hybrid-sync/README.md

@@ -29,6 +29,14 @@ Spinifex's hybrid mode enables bidirectional data synchronization between local
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Push Local to Cloud
 
 ```bash
@@ -51,7 +59,7 @@ rsync -avz /data/ user@cloud-server:/backup/spinifex-data/
 
 ## Troubleshooting
 
-## Sync fails mid-transfer
+## Sync Fails Mid Transfer
 
 S3 sync is idempotent — re-run the same command to resume where it left off:
 
@@ -61,14 +69,3 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \
 ```
 
 Only changed or missing files will be transferred on subsequent runs.
-
-## Permissions errors during sync
-
-Verify your AWS credentials are configured for both the source and destination:
-
-```bash
-aws sts get-caller-identity --profile spinifex
-aws sts get-caller-identity --profile default
-```
-
-Ensure the IAM user has `s3:GetObject`, `s3:PutObject`, and `s3:ListBucket` permissions on both buckets.

docs/iam-users-and-policies/README.md

@@ -0,0 +1,538 @@
+---
+title: "IAM Users and Policies"
+description: "Create IAM users, manage access keys, and control permissions with policies."
+category: "Administration"
+tags:
+  - iam
+  - users
+  - policies
+  - access keys
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IAM CLI Reference"
+    url: "https://docs.aws.amazon.com/cli/latest/reference/iam/"
+  - title: "IAM Policy Reference"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html"
+---
+
+# IAM Users and Policies
+
+> Create IAM users, manage access keys, and control permissions with policies.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Users](#users)
+- [Access Keys](#access-keys)
+- [Policies](#policies)
+- [Attaching Policies](#attaching-policies)
+- [Putting It All Together](#putting-it-all-together)
+- [Command Reference](#command-reference)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex implements AWS-compatible IAM with 16 operations covering user management, access key lifecycle, policy CRUD, and policy attachment. All IAM resources are scoped to the account that creates them — users in one account cannot see or modify resources in another.
+
+When you create an account with `spx admin account create`, Spinifex bootstraps a root user with an `AdministratorAccess` policy and writes the credentials to `~/.aws/credentials`. From there, you use the standard AWS CLI to manage additional users and permissions.
+
+**How authentication works:** Every AWS CLI request is signed with SigV4 using an access key pair. The gateway verifies the signature, resolves the caller's account, and evaluates attached policies before routing the request. The root user (account `000000000000`) bypasses policy evaluation entirely.
+
+## Instructions
+
+## Prerequisites
+
+- A running Spinifex cluster with the AWS gateway started
+- An account created via `spx admin account create` (see [Creating Accounts](/docs/creating-accounts))
+- AWS CLI configured with the account's profile:
+
+```bash
+export AWS_PROFILE=spinifex-myteam
+```
+
+## Users
+
+### Create a User
+
+```bash
+aws iam create-user --user-name alice
+```
+
+The response includes the user's ARN, unique ID, and creation date:
+
+```json
+{
+  "User": {
+    "UserName": "alice",
+    "UserId": "AIDA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:user/alice",
+    "Path": "/",
+    "CreateDate": "2026-03-24T10:00:00Z"
+  }
+}
+```
+
+Use `--path` to organise users into hierarchical groups:
+
+```bash
+aws iam create-user --user-name bob --path /developers/
+```
+
+### Get a User
+
+```bash
+aws iam get-user --user-name alice
+```
+
+### List Users
+
+```bash
+aws iam list-users
+```
+
+Filter by path prefix:
+
+```bash
+aws iam list-users --path-prefix /developers/
+```
+
+### Delete a User
+
+Before deleting a user, remove all access keys and detach all policies:
+
+```bash
+# Remove access keys
+aws iam list-access-keys --user-name alice
+aws iam delete-access-key --user-name alice --access-key-id AKIA...
+
+# Detach policies
+aws iam list-attached-user-policies --user-name alice
+aws iam detach-user-policy --user-name alice \
+  --policy-arn arn:aws:iam::000000000001:policy/MyPolicy
+
+# Now delete
+aws iam delete-user --user-name alice
+```
+
+## Access Keys
+
+Access keys are how users authenticate with the AWS CLI. Each user can have up to **2 access keys** at a time, allowing key rotation without downtime.
+
+### Create an Access Key
+
+```bash
+aws iam create-access-key --user-name alice
+```
+
+```json
+{
+  "AccessKey": {
+    "UserName": "alice",
+    "AccessKeyId": "AKIA1A2B3C4D5E6F7890ABCD",
+    "Status": "Active",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "CreateDate": "2026-03-24T10:05:00Z"
+  }
+}
+```
+
+> **The secret access key is only shown once.** Save it immediately. If lost, delete the key and create a new one.
+
+Add the credentials to `~/.aws/credentials` under a new profile and set `AWS_PROFILE` to use them.
+
+### List Access Keys
+
+```bash
+aws iam list-access-keys --user-name alice
+```
+
+### Rotate Access Keys
+
+Create a second key, update your configuration, then delete the old one:
+
+```bash
+# 1. Create new key (while old key still works)
+aws iam create-access-key --user-name alice
+
+# 2. Update ~/.aws/credentials with the new key
+
+# 3. Verify the new key works
+AWS_PROFILE=spinifex-alice aws sts get-caller-identity
+
+# 4. Delete the old key
+aws iam delete-access-key --user-name alice --access-key-id AKIA_OLD_KEY_ID
+```
+
+### Deactivate an Access Key
+
+Temporarily disable a key without deleting it:
+
+```bash
+aws iam update-access-key --user-name alice \
+  --access-key-id AKIA1A2B3C4D5E6F7890ABCD \
+  --status Inactive
+```
+
+Reactivate it later:
+
+```bash
+aws iam update-access-key --user-name alice \
+  --access-key-id AKIA1A2B3C4D5E6F7890ABCD \
+  --status Active
+```
+
+## Policies
+
+Policies are JSON documents that define what actions a user can perform on which resources.
+
+### Policy Document Format
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:RunInstances", "ec2:DescribeInstances"],
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+Each statement has:
+
+- **Effect** — `Allow` or `Deny`
+- **Action** — Service actions (e.g. `ec2:RunInstances`, `s3:GetObject`). Supports wildcards: `ec2:*`, `s3:Get*`, or `*` for all actions.
+- **Resource** — Target resources. Use `*` for all resources.
+
+**Evaluation order:** An explicit `Deny` always wins. If no statement matches, access is denied by default.
+
+### Create a Policy
+
+Save the policy document to a file, then create the policy:
+
+```bash
+cat > /tmp/ec2-readonly.json << 'EOF'
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:DescribeInstances",
+                "ec2:DescribeImages",
+                "ec2:DescribeKeyPairs",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeVpcs"
+            ],
+            "Resource": "*"
+        }
+    ]
+}
+EOF
+
+aws iam create-policy \
+  --policy-name EC2ReadOnly \
+  --policy-document file:///tmp/ec2-readonly.json
+```
+
+```json
+{
+  "Policy": {
+    "PolicyName": "EC2ReadOnly",
+    "PolicyId": "ANPA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:policy/EC2ReadOnly",
+    "Path": "/",
+    "DefaultVersionId": "v1",
+    "CreateDate": "2026-03-24T10:10:00Z"
+  }
+}
+```
+
+### Common Policy Examples
+
+**Full administrator access:**
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "*",
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+**S3 read/write (for hybrid sync, backups):**
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:ListBucket",
+        "s3:DeleteObject"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+**EC2 operator (launch and manage instances, no VPC changes):**
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:RunInstances",
+        "ec2:StartInstances",
+        "ec2:StopInstances",
+        "ec2:TerminateInstances",
+        "ec2:DescribeInstances",
+        "ec2:DescribeImages"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+**Deny terminate (attach alongside a broader Allow to prevent accidental termination):**
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": "ec2:TerminateInstances",
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+### Get a Policy
+
+```bash
+aws iam get-policy \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+### Get the Policy Document
+
+Use `get-policy-version` with version `v1` to retrieve the actual JSON document:
+
+```bash
+aws iam get-policy-version \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly \
+  --version-id v1
+```
+
+### List Policies
+
+```bash
+aws iam list-policies
+```
+
+### Delete a Policy
+
+A policy must be detached from all users before it can be deleted:
+
+```bash
+aws iam delete-policy \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+## Attaching Policies
+
+Policies have no effect until attached to a user.
+
+### Attach a Policy to a User
+
+```bash
+aws iam attach-user-policy --user-name alice \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+### List a User's Policies
+
+```bash
+aws iam list-attached-user-policies --user-name alice
+```
+
+### Detach a Policy
+
+```bash
+aws iam detach-user-policy --user-name alice \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+## Putting It All Together
+
+A complete workflow for onboarding a developer with scoped EC2 and S3 access:
+
+```bash
+# 1. Create the user
+aws iam create-user --user-name dev-carol --path /developers/
+
+# 2. Create an access key
+aws iam create-access-key --user-name dev-carol
+# Save the AccessKeyId and SecretAccessKey from the output
+
+# 3. Create a scoped policy
+cat > /tmp/dev-policy.json << 'EOF'
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "EC2Access",
+            "Effect": "Allow",
+            "Action": ["ec2:*"],
+            "Resource": "*"
+        },
+        {
+            "Sid": "S3ReadOnly",
+            "Effect": "Allow",
+            "Action": ["s3:GetObject", "s3:ListBucket"],
+            "Resource": "*"
+        },
+        {
+            "Sid": "NoTerminate",
+            "Effect": "Deny",
+            "Action": "ec2:TerminateInstances",
+            "Resource": "*"
+        }
+    ]
+}
+EOF
+
+aws iam create-policy \
+  --policy-name DeveloperAccess \
+  --policy-document file:///tmp/dev-policy.json
+
+# 4. Attach the policy
+aws iam attach-user-policy --user-name dev-carol \
+  --policy-arn arn:aws:iam::000000000001:policy/DeveloperAccess
+
+# 5. Configure the developer's AWS CLI
+aws configure --profile spinifex-carol
+# Enter the access key and secret from step 2
+
+# 6. Verify
+AWS_PROFILE=spinifex-carol aws ec2 describe-instances
+AWS_PROFILE=spinifex-carol aws ec2 terminate-instances --instance-ids i-123
+# ^ This will be denied by the NoTerminate statement
+```
+
+## Command Reference
+
+| Command                       | Key Parameters                               | Notes                             |
+| ----------------------------- | -------------------------------------------- | --------------------------------- |
+| `create-user`                 | `--user-name`, `--path`                      | Path default: `/`                 |
+| `get-user`                    | `--user-name`                                |                                   |
+| `list-users`                  | `--path-prefix`                              |                                   |
+| `delete-user`                 | `--user-name`                                | Must have no keys or policies     |
+| `create-access-key`           | `--user-name`                                | Max 2 per user; secret shown once |
+| `list-access-keys`            | `--user-name`                                |                                   |
+| `delete-access-key`           | `--user-name`, `--access-key-id`             |                                   |
+| `update-access-key`           | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive`            |
+| `create-policy`               | `--policy-name`, `--policy-document`         | Max 6144 bytes                    |
+| `get-policy`                  | `--policy-arn`                               | Metadata only                     |
+| `get-policy-version`          | `--policy-arn`, `--version-id`               | Use `v1`; includes document       |
+| `list-policies`               |                                              |                                   |
+| `delete-policy`               | `--policy-arn`                               | Must not be attached              |
+| `attach-user-policy`          | `--user-name`, `--policy-arn`                | Idempotent                        |
+| `detach-user-policy`          | `--user-name`, `--policy-arn`                |                                   |
+| `list-attached-user-policies` | `--user-name`                                |                                   |
+
+## Troubleshooting
+
+### AccessDenied on IAM Commands
+
+The calling user must have IAM permissions. Attach a policy with `iam:*` actions, or use the root account profile:
+
+```bash
+export AWS_PROFILE=spinifex-myteam
+aws iam list-users
+```
+
+### InvalidClientTokenId
+
+The access key is either inactive or does not exist. Check the key status:
+
+```bash
+aws iam list-access-keys --user-name alice
+```
+
+If the key shows `Inactive`, reactivate it:
+
+```bash
+aws iam update-access-key --user-name alice \
+  --access-key-id AKIA... --status Active
+```
+
+If the key was deleted, create a new one.
+
+### DeleteConflict When Deleting a User
+
+The user still has access keys or attached policies. Remove them first:
+
+```bash
+# Check for access keys
+aws iam list-access-keys --user-name alice
+
+# Check for attached policies
+aws iam list-attached-user-policies --user-name alice
+```
+
+### DeleteConflict When Deleting a Policy
+
+The policy is still attached to one or more users. Detach it from all users before deleting:
+
+```bash
+# Find who has it attached, then detach
+aws iam detach-user-policy --user-name alice \
+  --policy-arn arn:aws:iam::000000000001:policy/MyPolicy
+
+aws iam delete-policy \
+  --policy-arn arn:aws:iam::000000000001:policy/MyPolicy
+```
+
+### LimitExceeded When Creating Access Keys
+
+Each user can have at most 2 access keys. Delete an existing key before creating a new one:
+
+```bash
+aws iam list-access-keys --user-name alice
+aws iam delete-access-key --user-name alice --access-key-id AKIA_OLD
+aws iam create-access-key --user-name alice
+```
+
+### MalformedPolicyDocument
+
+The policy JSON is invalid. Check that:
+
+- `Version` is exactly `"2012-10-17"`
+- At least one `Statement` exists
+- Each statement has `Effect` (`Allow`/`Deny`), `Action`, and `Resource`
+- The document is under 6144 bytes
+- The JSON is well-formed (no trailing commas, correct quoting)

docs/install-airgapped/README.md

@@ -1,12 +1,11 @@
 ---
-title: "Air-Gapped Installation"
+title: "Air-Gapped Install"
 description: "Deploy Spinifex in environments without internet connectivity. Covers offline package caching, USB deployment, and package verification."
 category: "Getting Started"
 tags:
   - install
   - air-gapped
   - offline
-badge: advanced
 resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
@@ -38,7 +37,7 @@ In air-gapped or disconnected environments, Spinifex can be deployed without int
 
 ## Instructions
 
-## Step 1. Prepare packages (on a connected machine)
+## Step 1. Prepare Packages (on a connected machine)
 
 Set up a local APT cache to collect all required packages:
 
@@ -56,7 +55,7 @@ sudo apt install --download-only nbdkit nbdkit-plugin-dev pkg-config \
   ovn-central ovn-host openvswitch-switch
 ```
 
-## Step 2. Create USB deployment media
+## Step 2. Create USB Deployment Media
 
 ```bash
 mkdir -p /media/spinifex-deploy/{apt-packages,go-cache,spinifex-source}
@@ -65,7 +64,7 @@ cp -r ~/go/pkg/mod/cache/ /media/spinifex-deploy/go-cache/
 cp -r ~/Development/mulga/spinifex /media/spinifex-deploy/spinifex-source/
 ```
 
-## Step 3. Verify package integrity
+## Step 3. Verify Package Integrity
 
 Sign and verify packages before transferring to the target:
 
@@ -75,7 +74,7 @@ gpg --verify spinifex-v1.0.tar.gz.sig spinifex-v1.0.tar.gz
 sha256sum -c /media/spinifex-deploy/checksums.sha256
 ```
 
-## Step 4. Install on the target server
+## Step 4. Install on the Target Server
 
 Mount the USB and install:
 
@@ -86,11 +85,11 @@ cp -r /mnt/usb/spinifex-source ~/Development/mulga/spinifex
 cd ~/Development/mulga/spinifex && make build
 ```
 
-Then follow the [Source Install](/docs/source-install) guide from Step 3 (Setup OVN) onwards.
+Then follow the [Source Install](/docs/install-source) guide from Step 3 (Setup OVN) onwards.
 
 ## Troubleshooting
 
-## Missing dependencies after dpkg install
+### Missing Dependencies After dpkg Install
 
 Some packages may have unresolved dependencies. Fix with:
 
@@ -101,7 +100,7 @@ sudo apt-get install -f
 
 The `-f` flag tells apt to fix broken dependencies using what's available locally.
 
-## GPG verification fails
+### GPG Verification Fails
 
 The signing key may not match or the download was corrupted. Re-import the key and verify:
 
@@ -116,7 +115,7 @@ If verification still fails, re-download the package on the connected machine an
 sha256sum spinifex-v1.0.tar.gz
 ```
 
-## Go module cache errors
+### Go Module Cache Errors
 
 The Go module cache may have been incompletely copied. On the connected machine, re-export the full cache:
 

docs/install-multi-node/README.md

@@ -0,0 +1,177 @@
+---
+title: "Multi-Node Install"
+description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
+category: "Getting Started"
+tags:
+  - install
+  - multi node
+  - cluster
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+---
+
+# Multi-Node Installation
+
+> Deploy Spinifex across multiple servers to create an availability zone.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A Spinifex cluster distributes services across multiple servers for high availability, data durability, and fault tolerance. Cluster formation is automatic — the init node waits for peers to join, then distributes credentials, CA certificates, and configuration.
+
+**Network requirements:**
+
+- Minimum 1 NIC per server (2 recommended for production)
+- UDP port 6081 open between hosts (Geneve tunnels)
+- TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)
+
+## Instructions
+
+## Step 1. Install Spinifex on Each Server
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash
+```
+
+## Step 2. Set Node IP Variables
+
+On **each server**, export the management IPs for all nodes:
+
+```bash
+export SPINIFEX_NODE1=192.168.1.10
+export SPINIFEX_NODE2=192.168.1.11
+export SPINIFEX_NODE3=192.168.1.12
+export AWS_REGION=us-east-1
+export AWS_AZ=us-east-1a
+```
+
+## Step 3. Setup OVN Networking
+
+Server 1 runs OVN central and must be set up first. If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it. Otherwise use `--wan-bridge=br-wan --wan-iface=eth1` (dedicated WAN NIC) or `--macvlan --wan-iface=eth0` (single-NIC).
+
+**Server 1:**
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh \
+  --management \
+  --encap-ip=$SPINIFEX_NODE1
+```
+
+**Server 2** (after server 1 is ready):
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh \
+  --ovn-remote=tcp:$SPINIFEX_NODE1:6642 \
+  --encap-ip=$SPINIFEX_NODE2
+```
+
+**Server 3** (after server 1 is ready):
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh \
+  --ovn-remote=tcp:$SPINIFEX_NODE1:6642 \
+  --encap-ip=$SPINIFEX_NODE3
+```
+
+Verify all 3 chassis registered:
+
+```bash
+sudo ovn-sbctl show
+```
+
+## Step 4. Form the Cluster
+
+Run init and join concurrently — init blocks until all nodes join.
+
+**Server 1 — Initialize:**
+
+```bash
+sudo spx admin init \
+  --node node1 --nodes 3 \
+  --bind $SPINIFEX_NODE1 --cluster-bind $SPINIFEX_NODE1 \
+  --port 4432 --region $AWS_REGION --az $AWS_AZ
+```
+
+Save the admin credentials — they will not be shown again.
+
+**Server 2 — Join** (while init is running):
+
+```bash
+sudo spx admin join \
+  --node node2 --bind $SPINIFEX_NODE2 --cluster-bind $SPINIFEX_NODE2 \
+  --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ
+```
+
+**Server 3 — Join** (while init is running):
+
+```bash
+sudo spx admin join \
+  --node node3 --bind $SPINIFEX_NODE3 --cluster-bind $SPINIFEX_NODE3 \
+  --host $SPINIFEX_NODE1:4432 --region $AWS_REGION --az $AWS_AZ
+```
+
+## Step 5. Start Services
+
+On **all servers**:
+
+```bash
+sudo systemctl start spinifex.target
+```
+
+## Step 6. Verify
+
+From any node:
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+If this returns a list of available instance types, your cluster is working.
+
+**Congratulations! Your Spinifex cluster is installed.**
+
+Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.
+
+## Troubleshooting
+
+### Nodes Not Joining
+
+The init command must still be running when join executes. If init exited, re-run with `--force`.
+
+```bash
+curl -s http://$SPINIFEX_NODE1:4432/health
+```
+
+### OVN Chassis Not Registering
+
+```bash
+sudo ovn-sbctl show
+sudo ss -tlnp | grep 6642
+```
+
+### CA Certificate Not Trusted
+
+```bash
+sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+```
+
+### Cross-Host VMs Cannot Communicate
+
+```bash
+sudo ovs-vsctl show | grep -i geneve
+sudo ss -ulnp | grep 6081
+```

docs/install-source/README.md

@@ -0,0 +1,124 @@
+---
+title: "Source Install"
+description: "Build Spinifex from source for development, custom builds, or contributing."
+category: "Getting Started"
+tags:
+  - install
+  - source
+  - development
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Go Downloads"
+    url: "https://go.dev/dl/"
+---
+
+# Source Installation
+
+> Build Spinifex from source for development, custom builds, or contributing.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/install) is recommended.
+
+**Requirements:**
+
+- Ubuntu 22.04+ / Debian 12+
+- Go 1.26.1+
+- GCC, make, pkg-config
+- QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
+
+## Instructions
+
+## Step 1. Install Dependencies
+
+```bash
+mkdir -p ~/Development/mulga && cd ~/Development/mulga
+git clone https://github.com/mulgadc/spinifex.git
+sudo make -C spinifex quickinstall
+export PATH=$PATH:/usr/local/go/bin
+```
+
+## Step 2. Clone and Build
+
+```bash
+cd spinifex
+./scripts/clone-deps.sh
+./scripts/dev-setup.sh
+```
+
+Confirm `./bin/spx` exists.
+
+## Step 3. Setup OVN
+
+If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:
+
+```bash
+./scripts/setup-ovn.sh --management
+```
+
+If your WAN is a physical NIC, choose one:
+
+```bash
+# Dedicated WAN NIC (not your SSH connection):
+./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
+
+# Single-NIC host (SSH-safe macvlan):
+./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
+```
+
+## Step 4. Initialize
+
+```bash
+sudo ./bin/spx admin init --node node1 --nodes 1
+```
+
+Save the admin credentials printed during init.
+
+## Step 5. Start Services
+
+```bash
+./scripts/start-dev.sh
+```
+
+## Step 6. Verify
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+If this returns a list of available instance types, your installation is working.
+
+**Congratulations! Spinifex is installed from source.**
+
+Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.
+
+## Troubleshooting
+
+### Go Not Found in PATH
+
+```bash
+export PATH=$PATH:/usr/local/go/bin
+```
+
+### ./bin/spx Missing After Build
+
+```bash
+./scripts/dev-setup.sh
+```
+
+### CA Certificate Not Trusted
+
+```bash
+sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+```

docs/install/README.md

@@ -0,0 +1,132 @@
+---
+title: "Single-Node Install"
+description: "Install Spinifex on a single server using the binary installer."
+category: "Getting Started"
+tags:
+  - install
+  - single node
+  - quickstart
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "Viperblock (EBS)"
+    url: "https://github.com/mulgadc/viperblock"
+---
+
+# Installing Spinifex
+
+> Install Spinifex on a single server using the binary installer.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Spinifex is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments.
+
+This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Install](/docs/install-multi-node). To build from source, see [Source Install](/docs/install-source).
+
+**Supported Operating Systems:**
+
+- Ubuntu 22.04 / 24.04 / 25.10
+- Debian 12 / 13
+
+**What Gets Installed:**
+
+- Spinifex daemon and CLI
+- QEMU/KVM for virtual machine management
+- OVN/Open vSwitch for VPC networking
+- Predastore (S3-compatible object storage)
+- Viperblock (EBS-compatible block storage)
+- AWS CLI v2
+
+## Instructions
+
+## Step 1. Install Spinifex
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash
+```
+
+The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
+
+## Step 2. Setup OVN Networking
+
+If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+```
+
+If your WAN is a physical NIC, choose one:
+
+```bash
+# Dedicated WAN NIC (not your SSH connection):
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
+```
+
+**Separating VPC tunnel traffic from WAN:** If you want Geneve tunnel traffic (inter-VM east-west for VPC traffic) to use a dedicated interface instead of the WAN IP, add `--encap-ip=<IP>` to specify the tunnel endpoint address:
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --encap-ip=10.0.0.1
+```
+
+This is recommended for production and required in multi-node deployments. See [Multi-Node Install](/docs/install-multi-node) for details.
+
+## Step 3. Initialize
+
+```bash
+sudo spx admin init --node node1 --nodes 1
+```
+
+This auto-detects your network topology, generates configuration and TLS certificates, installs the CA into the system trust store, and configures AWS CLI credentials. Save the admin credentials printed during init — they will not be shown again.
+
+## Step 4. Start Services
+
+```bash
+sudo systemctl start spinifex.target
+```
+
+## Step 5. Verify
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+If this returns a list of available instance types, your installation is working.
+
+**Congratulations! Spinifex is installed.**
+
+Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.
+
+## Troubleshooting
+
+### spx Command Not Found
+
+```bash
+export PATH=$PATH:/usr/local/bin
+```
+
+### CA Certificate Not Trusted
+
+`sudo spx admin init` installs the CA automatically. If you need to re-install it manually:
+
+```bash
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+```
+
+### OVN Services Not Starting
+
+```bash
+sudo systemctl is-active ovn-controller
+journalctl -u ovn-controller --no-pager -n 20
+```

docs/installing-spinifex-multi-node/README.md

@@ -1,288 +0,0 @@
----
-title: "Multi-Node Installation"
-description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
-category: "Getting Started"
-tags:
-  - install
-  - multi node
-  - cluster
-badge: availability-zone
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Predastore (S3)"
-    url: "https://github.com/mulgadc/predastore"
-  - title: "Viperblock (EBS)"
-    url: "https://github.com/mulgadc/viperblock"
----
-
-# Multi-Node Installation
-
-> Deploy Spinifex across multiple servers to create an availability zone.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-A Spinifex cluster operates as a fully distributed infrastructure region — similar to an AWS Region — where multiple nodes provide high availability, data durability, and fault tolerance.
-
-**Services distributed across nodes:**
-
-- **NATS** — Clustered message bus for request routing and JetStream replication
-- **Predastore** — Raft consensus with Reed-Solomon erasure coding (RS 2+1)
-- **Viperblock** — Block storage co-located with compute nodes
-- **Spinifex Daemon** — Independent AWS API request serving per node
-
-Cluster formation is automatic. When initializing with `--nodes 3`, the init node starts a formation server and waits for peers to join. Once all nodes register, each receives the full cluster topology — credentials, CA certificates, NATS routes, Predastore peer lists — no manual configuration synchronization required.
-
-**Network requirements:**
-
-- Minimum 1 NIC per server
-- 2 NICs recommended for production (management + overlay)
-- UDP port 6081 open between hosts (Geneve tunnels)
-
-## Instructions
-
-## Step 1. Install Spinifex on each server
-
-Run the binary installer on **every server** in the cluster:
-
-```bash
-curl https://install.mulgadc.com | bash
-```
-
-## Step 2. Set node IP variables
-
-On **each server**, export the IPs for all nodes (replace with your actual IPs):
-
-```bash
-export SPINIFEX_NODE1=192.168.1.10
-export SPINIFEX_NODE2=192.168.1.11
-export SPINIFEX_NODE3=192.168.1.12
-```
-
-To find your server's IP:
-
-```bash
-ip addr show | grep "inet "
-```
-
-Next, define the AWS region and AZ you are deploying locally.
-
-```bash
-export AWS_REGION=us-east-1
-export AWS_AZ=us-east-1a
-```
-
-## Step 3. Setup OVN networking
-
-OVN must be configured on every server before forming the cluster. Server 1 runs OVN central (NB and SB databases) and must be set up first.
-
-**Server 1 — OVN central + compute (run first):**
-
-```bash
-sudo /usr/local/share/spinifex/setup-ovn.sh --management --encap-ip=$SPINIFEX_NODE1
-```
-
-Verify OVN central is ready before proceeding:
-
-```bash
-sudo ovn-sbctl show
-```
-
-**Server 2 — Compute node (after server 1 is ready):**
-
-```bash
-sudo /usr/local/share/spinifex/setup-ovn.sh --ovn-remote=tcp:$SPINIFEX_NODE1:6642 --encap-ip=$SPINIFEX_NODE2
-```
-
-**Server 3 — Compute node (after server 1 is ready):**
-
-```bash
-sudo /usr/local/share/spinifex/setup-ovn.sh --ovn-remote=tcp:$SPINIFEX_NODE1:6642 --encap-ip=$SPINIFEX_NODE3
-```
-
-Verify all chassis have registered (from server 1):
-
-```bash
-sudo ovn-sbctl show
-```
-
-You should see 3 chassis entries, one per server, each with a Geneve encap IP.
-
-## Step 4. Form the cluster
-
-The formation process requires running init and join commands concurrently. The init node blocks until all expected nodes have joined.
-
-**Server 1 — Initialize the cluster:**
-
-```bash
-sudo spx admin init \
-  --node node1 \
-  --nodes 3 \
-  --bind $SPINIFEX_NODE1 \
-  --cluster-bind $SPINIFEX_NODE1 \
-  --port 4432 \
-  --region $AWS_REGION \
-  --az $AWS_AZ
-```
-
-**Server 2 — Join the cluster (while init is running):**
-
-```bash
-spx admin join \
-  --node node2 \
-  --bind $SPINIFEX_NODE2 \
-  --cluster-bind $SPINIFEX_NODE2 \
-  --host $SPINIFEX_NODE1:4432 \
-  --region $AWS_REGION \
-  --az $AWS_AZ
-```
-
-**Server 3 — Join the cluster (while init is running):**
-
-```bash
-spx admin join \
-  --node node3 \
-  --bind $SPINIFEX_NODE3 \
-  --cluster-bind $SPINIFEX_NODE3 \
-  --host $SPINIFEX_NODE1:4432 \
-  --region $AWS_REGION \
-  --az $AWS_AZ
-```
-
-All three processes will exit with a cluster summary once formation is complete.
-
-## Step 5. Start services and verify
-
-Start services on **all servers**:
-
-```bash
-sudo systemctl start spinifex.target
-```
-
-From any node, verify the cluster:
-
-```bash
-spx get nodes
-```
-
-```
-NAME    STATUS    IP              REGION      AZ           UPTIME   VMs
-node1   Ready     192.168.1.10    us-east-1   us-east-1a   2m       0
-node2   Ready     192.168.1.11    us-east-1   us-east-1a   2m       0
-node3   Ready     192.168.1.12    us-east-1   us-east-1a   2m       0
-```
-
-## Step 6. Trust the CA certificate
-
-On **each server**, trust the CA generated during init:
-
-```bash
-sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
-sudo update-ca-certificates
-export AWS_PROFILE=spinifex
-```
-
-## Step 7. Launch instances across the cluster
-
-Connect to any node's AWS Gateway — all nodes serve the same cluster state:
-
-```bash
-aws ec2 run-instances \
-  --image-id $SPINIFEX_AMI \
-  --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 3
-```
-
-Spinifex distributes instances across nodes based on available capacity. Check which node each instance landed on:
-
-```bash
-spx get vms
-```
-
-## Shutdown
-
-A graceful shutdown coordinates all nodes — drains VMs, flushes storage, and stops services:
-
-```bash
-spx admin cluster shutdown
-```
-
-## Troubleshooting
-
-## Nodes not joining the cluster
-
-The init command must still be running when you execute join on other servers. If init has already exited, re-run it.
-
-Check that all servers can reach the init node on the formation port:
-
-```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
-```
-
-## OVN chassis not registering
-
-Compute nodes need to reach OVN central on server 1. Verify from server 1:
-
-```bash
-sudo ovn-sbctl show
-```
-
-If chassis are missing, check firewall rules for port 6642:
-
-```bash
-sudo ss -tlnp | grep 6642
-```
-
-Re-run the OVN setup on the affected compute node.
-
-## Cross-host VMs cannot communicate
-
-This indicates a Geneve tunnel issue. Verify tunnels are established:
-
-```bash
-sudo ovs-vsctl show | grep -i geneve
-```
-
-Ensure UDP port 6081 is open between all hosts:
-
-```bash
-sudo ss -ulnp | grep 6081
-```
-
-If tunnels are missing, verify `--encap-ip` was set correctly during OVN setup.
-
-## Health check fails on a node
-
-Check if the daemon is running and listening on the correct IP:
-
-```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
-```
-
-Review the daemon logs for errors:
-
-```bash
-cat ~/spinifex/logs/spinifex.log
-```
-
-Verify `--bind` was set to the correct IP during cluster formation.
-
-## NATS cluster routing issues
-
-Check NATS logs for connection errors between nodes:
-
-```bash
-grep -i "route\|cluster" ~/spinifex/logs/nats.log
-```
-
-Ensure `--cluster-bind` IPs are reachable between all servers.

docs/installing-spinifex/README.md

@@ -1,200 +0,0 @@
----
-title: "Installing Spinifex"
-description: "Install Spinifex on a single server using the binary installer. Get a working region with EC2, VPC, EBS, and S3 in minutes."
-category: "Getting Started"
-tags:
-  - install
-  - single node
-  - quickstart
-badge: quickstart
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Predastore (S3)"
-    url: "https://github.com/mulgadc/predastore"
-  - title: "Viperblock (EBS)"
-    url: "https://github.com/mulgadc/viperblock"
----
-
-# Installing Spinifex
-
-> Install Spinifex on a single server using the binary installer.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-Spinifex is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments.
-
-This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Installation](/docs/installing-spinifex-multi-node). To build from source, see [Source Install](/docs/source-install).
-
-**Supported Operating Systems:**
-
-- Ubuntu 22.04 / 24.04 / 25.10
-- Debian 12.13
-
-**What gets installed:**
-
-- Spinifex daemon and CLI
-- QEMU/KVM for virtual machine management
-- OVN/Open vSwitch for VPC networking
-- Predastore (S3-compatible object storage)
-- Viperblock (EBS-compatible block storage)
-- AWS CLI v2
-
-## Instructions
-
-## Step 1. Install Spinifex
-
-```bash
-curl https://install.mulgadc.com | bash
-```
-
-The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
-
-## Step 2. Initialize a region
-
-Create your first region and availability zone, replace `ap-southeast-2` with your desired region:
-
-```bash
-spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
-```
-
-## Step 3. Trust the CA certificate
-
-Spinifex generates a local CA during initialization. Add it to your system trust store:
-
-```bash
-sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
-sudo update-ca-certificates
-```
-
-## Step 4. Start services
-
-```bash
-sudo systemctl start spinifex.target
-```
-
-Set the AWS profile to use the default admin account:
-
-```bash
-export AWS_PROFILE=spinifex
-```
-
-## Step 5. Import an AMI
-
-List available images and import one matching your architecture:
-
-```bash
-spx admin images list
-spx admin images import --name debian-12-arm64
-```
-
-## Step 6. Create a VPC and launch an instance
-
-```bash
-aws ec2 create-vpc --cidr-block 10.200.0.0/16
-export SPINIFEX_VPC="vpc-XXX"
-
-aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24
-export SPINIFEX_SUBNET="subnet-XXX"
-
-aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub
-
-aws ec2 run-instances \
-  --image-id $SPINIFEX_AMI \
-  --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
-```
-
-## Step 7. Connect via SSH
-
-Find the forwarded SSH port and connect:
-
-```bash
-ps auxw | grep hostfwd
-# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
-
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```
-
-## Troubleshooting
-
-## spx command not found
-
-The binary is not in your PATH. Run the installer again or add the install directory to your PATH:
-
-```bash
-export PATH=$PATH:~/spinifex/bin/
-```
-
-Add this to your `~/.bashrc` or `~/.zshrc` for persistence.
-
-## OVN services not starting
-
-Check if the OVN controller is active:
-
-```bash
-sudo systemctl is-active ovn-controller
-sudo ovn-sbctl show
-```
-
-If inactive, re-run the OVN setup that the installer performs. Check system logs for details:
-
-```bash
-journalctl -u ovn-controller --no-pager -n 20
-```
-
-## CA certificate not trusted
-
-AWS CLI will reject HTTPS connections if the Spinifex CA is not trusted. Re-add the certificate:
-
-```bash
-sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
-sudo update-ca-certificates
-```
-
-Verify it was added:
-
-```bash
-ls -la /usr/local/share/ca-certificates/spinifex-ca.crt
-```
-
-## Instance stuck in pending
-
-Check the Spinifex daemon logs for errors:
-
-```bash
-ls ~/spinifex/logs/
-cat ~/spinifex/logs/spinifex.log
-```
-
-Verify the AMI was imported successfully:
-
-```bash
-aws ec2 describe-images
-```
-
-## SSH connection refused
-
-cloud-init takes 30-60 seconds to configure the instance after boot. Wait and retry.
-
-Verify the SSH port forwarding is active:
-
-```bash
-ps auxw | grep hostfwd
-```
-
-If no `hostfwd` entry appears, the instance may not have started correctly. Check instance state:
-
-```bash
-aws ec2 describe-instances --instance-ids $INSTANCE_ID
-```

docs/launching-instances/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Launching Instances"
 description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex."
-category: "Environments"
+category: "Compute"
 tags:
   - ec2
   - instances
@@ -20,8 +20,16 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
-- [Instructions](#instructions)
+- [Launch](#launch)
+- [Manage](#manage)
+- [Reboot](#reboot)
+- [Modify Instance Attributes](#modify-instance-attributes)
+- [Console Output](#console-output)
+- [Instance Types](#instance-types)
+- [SSH (Development)](#ssh-development)
 - [Troubleshooting](#troubleshooting)
+  - [Instance Fails to Boot](#instance-fails-to-boot)
+  - [Cannot SSH Into Instance](#cannot-ssh-into-instance)
 
 ---
 
@@ -34,19 +42,29 @@ Spinifex provides EC2-compatible VM management built on QEMU/KVM. Instances supp
 - `run-instances` — Launch new VMs
 - `describe-instances` — Query state
 - `stop-instances` / `start-instances` — Lifecycle
+- `reboot-instances` — In-place restart (QMP reset)
 - `terminate-instances` — Permanent removal
+- `modify-instance-attribute` — Change type, user data, or source/dest check
+- `get-console-output` — Retrieve serial console log
+- `describe-instance-types` — List available instance types
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Launch
 
 ```bash
 aws ec2 run-instances \
   --image-id $SPINIFEX_AMI \
   --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
+  --key-name spinifex-key
 
 export INSTANCE_ID="i-XXX"
 ```
@@ -58,18 +76,75 @@ aws ec2 describe-instances --instance-ids $INSTANCE_ID
 aws ec2 stop-instances --instance-ids $INSTANCE_ID
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 reboot-instances --instance-ids $INSTANCE_ID
+```
+
+## Modify Instance Attributes
+
+Change instance type, user data, or source/dest check. Instance type and user data require the instance to be **stopped** first.
+
+### Change Instance Type
+
+```bash
+aws ec2 stop-instances --instance-ids $INSTANCE_ID
+
+aws ec2 modify-instance-attribute \
+  --instance-id $INSTANCE_ID \
+  --instance-type t3.medium
+
+aws ec2 start-instances --instance-ids $INSTANCE_ID
+```
+
+## Console Output
+
+Retrieve the serial console log for a running instance. Output is base64-encoded.
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID
+```
+
+Decode the output:
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID \
+  --query 'Output' --output text | base64 -d
+```
+
+## Instance Types
+
+List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.
+
+```bash
+aws ec2 describe-instance-types
+```
+
+Filter to a specific type:
+
+```bash
+aws ec2 describe-instance-types --instance-types t3.micro t3.small
+```
+
+Show capacity (how many of each type can still be launched):
+
+```bash
+aws ec2 describe-instance-types \
+  --filters Name=capacity,Values=true
 ```
 
 ## SSH (Development)
 
+In development mode, find the QEMU port forward and connect via localhost:
+
 ```bash
 ps auxw | grep $INSTANCE_ID
 ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
 ```
 
+For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
+
 ## Troubleshooting
 
-## Instance fails to boot
+### Instance Fails to Boot
 
 Check QEMU logs for the instance and verify the AMI architecture matches your host:
 
@@ -81,11 +156,11 @@ aws ec2 describe-images --image-ids $SPINIFEX_AMI
 If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), import the correct image:
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
+spx admin images list
+spx admin images import --name debian-12-x86_64
 ```
 
-## Cannot SSH into instance
+### Cannot SSH Into Instance
 
 cloud-init needs time to configure the instance after boot. Wait 30-60 seconds and retry.
 

docs/moving-aws-workload/README.md

@@ -6,7 +6,6 @@ tags:
   - migration
   - aws
   - terraform
-badge: new
 resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
@@ -38,10 +37,17 @@ Spinifex provides drop-in compatibility with AWS APIs, making it possible to mig
 
 ## Instructions
 
-## Configure AWS CLI
+## Prerequisites
+
+Ensure the AWS profile is set:
 
 ```bash
 export AWS_PROFILE=spinifex
+```
+
+## Configure AWS CLI
+
+```bash
 aws ec2 describe-instances
 aws s3 ls
 ```
@@ -73,7 +79,7 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ --source-region spinifex --reg
 
 ## Troubleshooting
 
-## Terraform provider errors
+## Terraform Provider Errors
 
 Ensure all three skip flags are set in your provider configuration:
 
@@ -85,7 +91,7 @@ skip_requesting_account_id  = true
 
 Without these, Terraform will try to validate credentials and metadata against real AWS endpoints.
 
-## S3 signature errors
+## S3 Signature Errors
 
 Spinifex uses AWS Signature V4. Ensure your AWS CLI is version 2.0 or higher:
 

docs/placement-groups/README.md

@@ -0,0 +1,121 @@
+---
+title: "Placement Groups"
+description: "Create and manage spread and cluster placement groups for hardware-level instance placement control."
+category: "Compute"
+tags:
+  - ec2
+  - placement
+  - spread
+  - cluster
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS Placement Groups"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html"
+---
+
+# Placement Groups
+
+> Create and manage spread and cluster placement groups for hardware-level instance placement control.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Create](#create)
+- [Launch Into a Placement Group](#launch-into-a-placement-group)
+- [Describe](#describe)
+- [Delete](#delete)
+- [Troubleshooting](#troubleshooting)
+  - [InsufficientInstanceCapacity Error](#insufficientinstancecapacity-error)
+  - [Cannot Delete Placement Group](#cannot-delete-placement-group)
+
+---
+
+## Overview
+
+Placement groups control how Spinifex distributes instances across physical hosts. Two strategies are supported:
+
+- **Spread** — One instance per physical host, maximising fault isolation
+- **Cluster** — All instances on the same host, minimising latency
+
+The `partition` strategy is not supported.
+
+**Supported operations:**
+
+- `create-placement-group` — Create a new group
+- `describe-placement-groups` — Query groups with optional filters
+- `delete-placement-group` — Remove an empty group
+
+## Instructions
+
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Create
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-spread-group \
+  --strategy spread
+```
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-cluster-group \
+  --strategy cluster
+```
+
+## Launch Into a Placement Group
+
+```bash
+aws ec2 run-instances \
+  --image-id $SPINIFEX_AMI \
+  --instance-type t3.small \
+  --key-name spinifex-key \
+  --placement GroupName=my-spread-group \
+  --count 3
+```
+
+## Describe
+
+```bash
+aws ec2 describe-placement-groups
+aws ec2 describe-placement-groups --group-names my-spread-group
+aws ec2 describe-placement-groups --group-ids pg-abc123
+aws ec2 describe-placement-groups \
+  --filters Name=strategy,Values=spread
+```
+
+## Delete
+
+The group must have no running instances before it can be deleted:
+
+```bash
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 delete-placement-group --group-name my-spread-group
+```
+
+## Troubleshooting
+
+### InsufficientInstanceCapacity Error
+
+Not enough distinct physical hosts for a spread launch. Reduce `--count` or terminate existing instances to free host slots:
+
+```bash
+spx admin nodes list
+```
+
+### Cannot Delete Placement Group
+
+The group still has running instances. Terminate them first:
+
+```bash
+aws ec2 describe-instances \
+  --filters Name=placement-group-name,Values=my-spread-group
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+```

docs/setting-up-your-cluster/README.md

@@ -0,0 +1,336 @@
+---
+title: "Setting Up Your Cluster"
+description: "Import an AMI, create a VPC with a public subnet, and launch your first EC2 instance on Spinifex."
+category: "Getting Started"
+tags:
+  - setup
+  - vpc
+  - ec2
+  - quickstart
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS EC2 CLI Reference"
+    url: "https://docs.aws.amazon.com/cli/latest/reference/ec2/"
+---
+
+# Setting Up Your Cluster
+
+> Import an AMI, create a VPC with a public subnet, and launch your first instance.
+
+## Table of Contents
+
+- [1. Import an AMI](#1-import-an-ami)
+- [2. Create an SSH Key](#2-create-an-ssh-key)
+- [3. Create a VPC and Public Subnet](#3-create-a-vpc-and-public-subnet)
+- [4. Launch an Instance](#4-launch-an-instance)
+- [5. Connect via SSH](#5-connect-via-ssh)
+- [6. Managing Instances](#6-managing-instances)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
+- [Binary Install (Single Node)](/docs/install)
+- [Multi-Node Install](/docs/install-multi-node)
+- [Source Install](/docs/install-source)
+
+## Instructions
+
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## 1. Import an AMI
+
+List available images and import one matching your architecture:
+
+```bash
+spx admin images list
+```
+
+```
+NAME                 | DISTRO | VERSION | ARCH   | BOOT
+debian-12-arm64      | debian | 12      | arm64  | bios
+debian-12-x86_64     | debian | 12      | x86_64 | bios
+ubuntu-24.04-arm64   | ubuntu | 24.04   | arm64  | bios
+ubuntu-24.04-x86_64  | ubuntu | 24.04   | x86_64 | bios
+```
+
+Import an image:
+
+```bash
+spx admin images import --name ubuntu-24.04-x86_64
+```
+
+Or import a local image file:
+
+```bash
+spx admin images import --file ~/images/ubuntu-24.04.img --distro ubuntu --version 24.04 --arch x86_64
+```
+
+Verify the import and note the AMI ID:
+
+```bash
+aws ec2 describe-images
+```
+
+## 2. Create an SSH Key
+
+### Option A: Import an existing key
+
+```bash
+aws ec2 import-key-pair \
+  --key-name "spinifex-key" \
+  --public-key-material fileb://~/.ssh/id_rsa.pub
+```
+
+### Option B: Create a new key pair
+
+```bash
+aws ec2 create-key-pair --key-name spinifex-key \
+  | jq -r '.KeyMaterial | rtrimstr("\n")' > ~/.ssh/spinifex-key
+
+chmod 600 ~/.ssh/spinifex-key
+ssh-keygen -y -f ~/.ssh/spinifex-key > ~/.ssh/spinifex-key.pub
+```
+
+Verify:
+
+```bash
+aws ec2 describe-key-pairs
+```
+
+## 3. Create a VPC and Public Subnet
+
+### Create a VPC
+
+```bash
+VPC_ID=$(aws ec2 create-vpc --cidr-block 10.200.0.0/16 \
+  --query 'Vpc.VpcId' --output text)
+
+echo "VPC: $VPC_ID"
+```
+
+### Create an Internet Gateway
+
+An Internet Gateway enables instances in public subnets to reach the internet and be reachable from the LAN/WAN.
+
+```bash
+IGW_ID=$(aws ec2 create-internet-gateway \
+  --query 'InternetGateway.InternetGatewayId' --output text)
+
+aws ec2 attach-internet-gateway \
+  --internet-gateway-id $IGW_ID \
+  --vpc-id $VPC_ID
+
+echo "IGW: $IGW_ID (attached to $VPC_ID)"
+```
+
+### Create a Public Subnet
+
+A public subnet auto-assigns a routable IP to each instance, making it directly reachable from your network.
+
+```bash
+SUBNET_ID=$(aws ec2 create-subnet \
+  --vpc-id $VPC_ID \
+  --cidr-block 10.200.1.0/24 \
+  --query 'Subnet.SubnetId' --output text)
+
+# Enable auto-assign public IP
+aws ec2 modify-subnet-attribute \
+  --subnet-id $SUBNET_ID \
+  --map-public-ip-on-launch
+
+echo "Subnet: $SUBNET_ID (public)"
+```
+
+### Verify
+
+```bash
+aws ec2 describe-vpcs --vpc-ids $VPC_ID
+aws ec2 describe-subnets --subnet-ids $SUBNET_ID
+```
+
+## 4. Launch an Instance
+
+Query available instance types for your hardware:
+
+```bash
+aws ec2 describe-instance-types
+```
+
+Launch an instance in the public subnet:
+
+```bash
+AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
+
+INSTANCE_ID=$(aws ec2 run-instances \
+  --image-id $AMI_ID \
+  --instance-type t3.small \
+  --key-name spinifex-key \
+  --subnet-id $SUBNET_ID \
+  --count 1 \
+  --query 'Instances[0].InstanceId' --output text)
+
+echo "Instance: $INSTANCE_ID"
+```
+
+Wait for the instance to reach `running` state:
+
+```bash
+aws ec2 describe-instances --instance-ids $INSTANCE_ID \
+  --query 'Reservations[0].Instances[0].[State.Name, PrivateIpAddress, PublicIpAddress]' \
+  --output text
+```
+
+Expected output:
+
+```
+running    10.200.1.4    192.168.1.155
+```
+
+The instance has both a private IP (VPC overlay) and a public IP (from your external pool, routable on your network).
+
+## 5. Connect via SSH
+
+SSH directly to the instance's public IP:
+
+```bash
+PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID \
+  --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
+
+ssh -i ~/.ssh/spinifex-key ec2-user@$PUBLIC_IP
+```
+
+> **Note:** cloud-init takes 30-60 seconds to configure the instance. If SSH is refused, wait and retry.
+
+Once connected, verify the instance has internet access:
+
+```bash
+curl -s http://ifconfig.me
+```
+
+This should return the instance's public IP or the gateway's SNAT address.
+
+## 6. Managing Instances
+
+### Stop
+
+```bash
+aws ec2 stop-instances --instance-ids $INSTANCE_ID
+```
+
+### Start
+
+```bash
+aws ec2 start-instances --instance-ids $INSTANCE_ID
+```
+
+### Terminate
+
+```bash
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+```
+
+### Console Output
+
+View the instance's serial console log (useful for debugging boot issues):
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID \
+  --query 'Output' --output text
+```
+
+### Multi-Node: Check Instance Placement
+
+On a multi-node cluster, instances are distributed across nodes:
+
+```bash
+spx get vms
+```
+
+## Additional Options
+
+### Private Subnets (No Public IP)
+
+Create a subnet without `--map-public-ip-on-launch`:
+
+```bash
+PRIVATE_SUBNET=$(aws ec2 create-subnet \
+  --vpc-id $VPC_ID \
+  --cidr-block 10.200.2.0/24 \
+  --query 'Subnet.SubnetId' --output text)
+```
+
+Instances in private subnets get a private IP only. They can still reach the internet via SNAT through the VPC router, but are not directly reachable from your network.
+
+### Multiple Accounts
+
+Create isolated accounts with their own resources:
+
+```bash
+spx admin account create --name myteam
+export AWS_PROFILE=spinifex-myteam
+```
+
+### Web UI
+
+The Spinifex web interface is available at `https://localhost:3000` when services are running.
+
+## Troubleshooting
+
+### Instance Stuck in Pending
+
+```bash
+cat ~/spinifex/logs/spinifex.log
+aws ec2 describe-images
+```
+
+### SSH Connection Refused
+
+cloud-init needs 30-60 seconds after boot. Check instance state:
+
+```bash
+aws ec2 describe-instances --instance-ids $INSTANCE_ID
+```
+
+### No Public IP Assigned
+
+Verify the subnet has `MapPublicIpOnLaunch` enabled:
+
+```bash
+aws ec2 describe-subnets --subnet-ids $SUBNET_ID
+```
+
+If `MapPublicIpOnLaunch` is false:
+
+```bash
+aws ec2 modify-subnet-attribute --subnet-id $SUBNET_ID --map-public-ip-on-launch
+```
+
+Also verify an Internet Gateway is attached to the VPC:
+
+```bash
+aws ec2 describe-internet-gateways
+```
+
+### Instance Has No Internet Access
+
+Check the VPC router's NAT rules (from the host):
+
+```bash
+sudo ovn-nbctl lr-nat-list $(sudo ovn-nbctl lr-list | awk '{print $2}' | head -1)
+```
+
+Verify the default route exists:
+
+```bash
+sudo ovn-nbctl lr-route-list $(sudo ovn-nbctl lr-list | awk '{print $2}' | head -1)
+```

docs/source-install/README.md

@@ -1,194 +0,0 @@
----
-title: "Source Install"
-description: "Build Spinifex from source for development, custom builds, or contributing. Includes dependency setup, compilation, and OVN configuration."
-category: "Getting Started"
-tags:
-  - install
-  - source
-  - development
-badge: source
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Predastore (S3)"
-    url: "https://github.com/mulgadc/predastore"
-  - title: "Viperblock (EBS)"
-    url: "https://github.com/mulgadc/viperblock"
-  - title: "Go Downloads"
-    url: "https://go.dev/dl/"
-  - title: "AWS CLI Install"
-    url: "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
----
-
-# Source Install
-
-> Build Spinifex from source for development, custom builds, or contributing.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-This guide covers building Spinifex from source code. Use this if you want to contribute to Spinifex, build custom versions, or inspect the codebase. For production deployments, the [binary installer](/docs/installing-spinifex) is recommended.
-
-**Supported Operating Systems:**
-- Ubuntu 22.04 / 24.04 / 25.10
-- Debian 12.13
-
-**Requirements:**
-- Go 1.26.1+
-- GCC, make, pkg-config
-- QEMU/KVM
-- OVN/Open vSwitch
-- AWS CLI v2
-
-## Instructions
-
-## Step 1. Install system dependencies
-
-### Quick Install
-
-Bootstrap all dependencies in one step:
-
-```bash
-sudo make -C spinifex quickinstall
-```
-
-Ensure Go is in your PATH:
-
-```bash
-export PATH=$PATH:/usr/local/go/bin/
-```
-
-### Manual Install
-
-Alternatively, install packages individually:
-
-```bash
-sudo add-apt-repository universe
-sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev make gcc unzip xz-utils file ovn-central ovn-host openvswitch-switch
-```
-
-Install Go 1.26.1+ from [https://go.dev/dl/](https://go.dev/dl/) and verify:
-
-```bash
-go version
-```
-
-Install AWS CLI v2:
-
-```bash
-curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-unzip awscliv2.zip
-sudo ./aws/install
-aws --version
-```
-
-## Step 2. Clone and build
-
-```bash
-mkdir -p ~/Development/mulga/
-cd ~/Development/mulga/
-git clone https://github.com/mulgadc/spinifex.git
-cd spinifex
-./scripts/clone-deps.sh    # Clone viperblock + predastore
-./scripts/dev-setup.sh     # Setup complete dev environment
-```
-
-Confirm `./bin/spx` exists and is executable.
-
-## Step 3. Setup OVN
-
-For a single-node dev environment:
-
-```bash
-./scripts/setup-ovn.sh --management
-```
-
-This creates the `br-int` integration bridge, starts `ovn-controller`, configures Geneve tunnel endpoints, and enables IP forwarding.
-
-## Step 4. Initialize
-
-Create a region and availability zone:
-
-```bash
-./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
-```
-
-## Step 5. Trust the CA certificate
-
-```bash
-sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
-sudo update-ca-certificates
-```
-
-## Step 6. Start services
-
-```bash
-./scripts/start-dev.sh
-export AWS_PROFILE=spinifex
-```
-
-## Troubleshooting
-
-## Go not found in PATH
-
-After installing Go, it may not be in your current shell session. Add it:
-
-```bash
-export PATH=$PATH:/usr/local/go/bin/
-```
-
-Add this to your `~/.bashrc` or `~/.zshrc` for persistence. Verify with:
-
-```bash
-go version
-```
-
-## ./bin/spx missing after build
-
-The build did not complete successfully. Re-run the setup:
-
-```bash
-./scripts/dev-setup.sh
-```
-
-Check the output for compilation errors. Common causes are missing Go version or missing system packages.
-
-## OVN services not starting
-
-Verify the setup script was run with `--management`:
-
-```bash
-./scripts/setup-ovn.sh --management
-```
-
-Check OVN controller status:
-
-```bash
-sudo systemctl is-active ovn-controller
-sudo ovn-sbctl show
-```
-
-## CA certificate not trusted
-
-AWS CLI will reject HTTPS connections to Spinifex services. Re-add the certificate:
-
-```bash
-sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
-sudo update-ca-certificates
-```
-
-## clone-deps.sh fails
-
-Ensure you have access to the GitHub repositories. If using SSH and it fails, try HTTPS:
-
-```bash
-git config --global url."https://github.com/".insteadOf "git@github.com:"
-./scripts/clone-deps.sh
-```

docs/spinifex-admin-cli/README.md

@@ -6,7 +6,6 @@ tags:
   - cli
   - admin
   - reference
-badge: cli
 resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
@@ -28,7 +27,7 @@ resources:
 
 The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
 
-**Binary location:** `~/Development/mulga/spinifex/bin/spx`
+**Binary location:** `/usr/local/bin/spx` (binary install) or `~/Development/mulga/spinifex/bin/spx` (source install)
 
 All services in the Spinifex platform are managed through this single binary.
 
@@ -37,14 +36,14 @@ All services in the Spinifex platform are managed through this single binary.
 ## Account Management
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
 ## Node Management
 
 ```bash
-./bin/spx get nodes
+spx get nodes
 ```
 
 ```
@@ -57,35 +56,35 @@ node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
 ## Monitor Resources
 
 ```bash
-./bin/spx top nodes
+spx top nodes
 ```
 
 ## Image Management
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-arm64
+spx admin images list
+spx admin images import --name debian-12-arm64
 ```
 
 ## Cluster Shutdown
 
 ```bash
-./bin/spx admin cluster shutdown
+spx admin cluster shutdown
 ```
 
 ## Troubleshooting
 
-## Permission denied running spinifex
+### Permission Denied Running Spinifex
 
 The binary may not be executable. Fix permissions:
 
 ```bash
-chmod +x ./bin/spx
+chmod +x /usr/local/bin/spx
 ```
 
 If you get permission errors during operations, ensure you're running with appropriate privileges. Some OVN and networking commands require `sudo`.
 
-## Services fail to start
+### Services Fail to Start
 
 Check the daemon logs for specific errors:
 

docs/terraform/bastion-private-subnet/README.md

@@ -0,0 +1,158 @@
+---
+title: "Bastion with Private Subnet"
+description: "Deploy a bastion host to access an isolated private compute instance with no internet connectivity."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - bastion
+  - vpc
+  - security
+  - private subnet
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Bastion with Private Subnet
+
+> Deploy a bastion host to access an isolated private compute instance with no internet connectivity.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a VPC with public and private subnets where the private subnet has no internet access. A bastion host in the public subnet is the only way to reach instances in the private subnet. This pattern is used for sensitive workloads that must remain isolated from the internet — the private instance cannot make or receive any connections outside the VPC.
+
+**Architecture:**
+
+```
+WAN ──SSH──▶ Bastion (public subnet)
+                 │
+                 ▼ SSH (private IP)
+              App Server (private subnet, no internet)
+```
+
+**What you'll learn:**
+
+- Creating public and private subnets in a VPC
+- Isolating compute instances from the internet using route tables and security groups
+- Using a bastion host as the sole access point to private resources
+- SSH hopping through a bastion to reach private instances
+
+**Use cases:**
+
+- Sensitive data processing that must not have internet egress
+- Internal services that should only be reachable within the VPC
+- Compliance workloads requiring network isolation
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/bastion-private-subnet
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Connect
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry.
+
+SSH into the bastion:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_public_ip>
+```
+
+From the bastion, SSH to the private instance. The key is pre-installed at `~/.ssh/bastion-demo.pem` via cloud-init:
+
+```bash
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+### Step 4. Verify Isolation
+
+From the private instance, confirm there is no internet connectivity:
+
+```bash
+# This should time out — the private instance has no route to the internet
+curl --connect-timeout 5 https://deb.debian.org || echo "No internet access (expected)"
+```
+
+The private instance can only communicate within the VPC (`10.20.0.0/16`). Its security group restricts egress to VPC-internal traffic only.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Cannot SSH to Private Instance
+
+The private instance has no public IP — it is only reachable from the bastion. SSH to the bastion first, then use the pre-installed key to hop:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+If the key is missing on the bastion, check that cloud-init completed successfully:
+
+```bash
+sudo journalctl -u cloud-init --no-pager
+ls -la ~/.ssh/bastion-demo.pem
+```
+
+### Private Instance Can Reach the Internet
+
+If the private instance unexpectedly has internet access, check that its route table has no default route:
+
+```bash
+aws ec2 describe-route-tables --profile spinifex
+```
+
+The private route table should have no `0.0.0.0/0` route. Also verify the private security group egress is restricted to the VPC CIDR (`10.20.0.0/16`).
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/bastion-private-subnet/main.tf

@@ -0,0 +1,346 @@
+# Example 2: Bastion Host with Private Subnet
+#
+# Deploys a VPC with both public and private subnets. A bastion host in the
+# public subnet provides SSH access to an isolated instance in the private
+# subnet. The private instance has no internet connectivity — ideal for
+# sensitive workloads that must remain air-gapped from the internet.
+#
+# Architecture:
+#
+#   WAN ──SSH──▶ Bastion (public subnet)
+#                    │
+#                    ▼ SSH (private IP)
+#                 App Server (private subnet, no internet)
+#
+# Usage:
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   # SSH to the bastion
+#   ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+#
+#   # From the bastion, SSH to the private instance
+#   # (the key is pre-installed at ~/.ssh/bastion-demo.pem via cloud-init)
+#   ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair (shared by bastion and private instances)
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "bastion" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "bastion" {
+  key_name   = "bastion-demo"
+  public_key = tls_private_key.bastion.public_key_openssh
+}
+
+resource "local_file" "bastion_pem" {
+  filename        = "${path.module}/bastion-demo.pem"
+  content         = tls_private_key.bastion.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "bastion-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — only the public subnet routes through this
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet — bastion host lives here
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "bastion-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "bastion-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Private Subnet — isolated instances live here (no public IPs, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "private" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.2.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = false
+
+  tags = {
+    Name = "bastion-demo-private"
+  }
+}
+
+# Private route table — no default route, no internet access
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private" {
+  subnet_id      = aws_subnet.private.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Groups
+# ---------------------------------------------------------------------------
+
+# Bastion: SSH from anywhere
+resource "aws_security_group" "bastion" {
+  name        = "bastion-demo-bastion-sg"
+  description = "Bastion: SSH from WAN"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "bastion-demo-bastion-sg"
+  }
+}
+
+# Private instances: SSH only from the bastion security group
+resource "aws_security_group" "private" {
+  name        = "bastion-demo-private-sg"
+  description = "Private: SSH from bastion only"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description     = "SSH from bastion"
+    from_port       = 22
+    to_port         = 22
+    protocol        = "tcp"
+    security_groups = [aws_security_group.bastion.id]
+  }
+
+  egress {
+    description = "VPC internal only"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["10.20.0.0/16"]
+  }
+
+  tags = {
+    Name = "bastion-demo-private-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Bastion Host (public subnet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "bastion" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.bastion.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  associate_public_ip_address = true
+
+  # Copy the SSH private key onto the bastion so you can hop to private instances
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+    mkdir -p /home/ec2-user/.ssh
+    cat > /home/ec2-user/.ssh/bastion-demo.pem <<'KEY'
+    ${tls_private_key.bastion.private_key_openssh}
+    KEY
+    chmod 600 /home/ec2-user/.ssh/bastion-demo.pem
+    chown -R ec2-user:ec2-user /home/ec2-user/.ssh
+  USERDATA
+  )
+
+  tags = {
+    Name = "bastion-demo-bastion"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Private Instance (private subnet — no public IP, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "private" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.private.id
+  vpc_security_group_ids = [aws_security_group.private.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  tags = {
+    Name = "bastion-demo-private-app"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry."
+}
+
+output "bastion_public_ip" {
+  value = aws_instance.bastion.public_ip
+}
+
+output "private_instance_ip" {
+  value = aws_instance.private.private_ip
+}
+
+output "ssh_to_bastion" {
+  description = "SSH to the bastion host"
+  value       = "ssh -i bastion-demo.pem ec2-user@${aws_instance.bastion.public_ip}"
+}
+
+output "ssh_to_private_from_bastion" {
+  description = "From the bastion, SSH to the private instance (key is pre-installed via cloud-init)"
+  value       = "ssh -i ~/.ssh/bastion-demo.pem ec2-user@${aws_instance.private.private_ip}"
+}

docs/terraform/nginx-webserver/README.md

@@ -0,0 +1,129 @@
+---
+title: "Nginx Web Server"
+description: "Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - nginx
+  - ec2
+  - vpc
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Nginx Web Server
+
+> Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC, public subnet, internet gateway, route table, security group, SSH key pair, and an EC2 instance with cloud-init user-data that installs and starts Nginx.
+
+**What you'll learn:**
+
+- Configuring the AWS Terraform provider to target Spinifex
+- Creating a VPC with public internet access
+- Provisioning an EC2 instance with cloud-init user-data
+- Generating SSH key pairs with the TLS provider
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/nginx-webserver
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Verify
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+After apply completes, use the outputs to test:
+
+```bash
+curl http://<public_ip>
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+```
+
+Open the `web_url` output in your browser to see the Nginx welcome page.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image. Check available AMIs:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```
+
+### Provider Connection Refused
+
+Verify Spinifex services are running:
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
+
+### SSH Connection Timeout
+
+Check that the security group allows inbound SSH (port 22) and that the instance has a public IP assigned. Verify the instance is running:
+
+```bash
+aws ec2 describe-instances --profile spinifex
+```
+
+### Nginx Not Responding
+
+SSH into the instance and check cloud-init logs:
+
+```bash
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+sudo journalctl -u cloud-init --no-pager
+sudo systemctl status nginx
+```

docs/terraform/nginx-webserver/main.tf

@@ -0,0 +1,286 @@
+# Example 1: Nginx Web Server on Spinifex
+#
+# Deploys a VPC with a public subnet and an EC2 instance running Nginx.
+# Demonstrates: VPC, subnet, internet gateway, route table, security group,
+# key pair, cloud-init user-data, and EC2 instance provisioning.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/01-nginx-webserver
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>        # Nginx welcome page
+#   ssh -i nginx-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — point the AWS provider at Spinifex
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"] # Spinifex system images
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "nginx" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "nginx" {
+  key_name   = "nginx-demo"
+  public_key = tls_private_key.nginx.public_key_openssh
+}
+
+resource "local_file" "nginx_pem" {
+  filename        = "${path.module}/nginx-demo.pem"
+  content         = tls_private_key.nginx.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.10.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "nginx-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — gives the public subnet a route to the internet
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "nginx-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.10.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "nginx-demo-public"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Route Table — send 0.0.0.0/0 through the internet gateway
+# ---------------------------------------------------------------------------
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "nginx-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "web" {
+  name        = "nginx-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nginx-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Debian 12 with Nginx installed via cloud-init
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install Nginx
+    apt-get update -y
+    apt-get install -y nginx
+
+    # Write a custom landing page
+    cat > /var/www/html/index.html <<'HTML'
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p>This Nginx server was deployed with Terraform on Spinifex infrastructure.</p>
+      <hr>
+      <p><small>Instance provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    # Ensure Nginx is running
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.nginx.id
+}
+
+output "public_ip" {
+  value = aws_instance.nginx.public_ip
+}
+
+output "ssh_command" {
+  value = "ssh -i nginx-demo.pem ec2-user@${aws_instance.nginx.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.nginx.public_ip}"
+}

docs/terraform/s3-webapp/README.md

@@ -0,0 +1,162 @@
+---
+title: "S3-Backed Web App"
+description: "Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - s3
+  - predastore
+  - flask
+  - webapp
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: S3-Backed Web App
+
+> Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy an EC2 instance running a Flask file-sharing web application backed by S3 (Predastore). Users can upload files through a web form and browse uploaded content — demonstrating Terraform managing both compute and object storage together.
+
+**Architecture:**
+
+```
+Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
+                      │
+                      ▼ S3 API (boto3)
+                  Predastore (port 8443)
+```
+
+**What you'll learn:**
+
+- Configuring the AWS provider with both Spinifex and Predastore endpoints
+- Creating S3 buckets on Predastore via Terraform
+- Deploying a Python webapp with cloud-init that talks to S3
+- Passing credentials and configuration to instances via user-data
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- Predastore running (S3 API on port 8443)
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+- The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/s3-webapp
+```
+
+Or create the files manually and paste the full configuration below.
+
+### Step 2. Create terraform.tfvars
+
+Before deploying, create a `terraform.tfvars` with your Predastore credentials. The `predastore_host` must be reachable from inside the VPC — use the host's br-wan or LAN IP, not localhost.
+
+<!-- INCLUDE: terraform.tfvars.example lang:hcl -->
+
+### Step 3. Create main.tf
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 4. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 5. Test the Application
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+Open the `web_url` output in your browser. You should see the file browser UI. Upload a file and verify it appears in the list.
+
+```bash
+# Verify via CLI
+curl http://<public_ip>
+
+# Check the S3 bucket directly
+aws s3 ls s3://webapp-uploads/ --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Predastore Connection Refused from Instance
+
+The EC2 instance cannot reach `localhost` on the host. Set `predastore_host` to the host's br-wan or LAN IP address:
+
+```hcl
+predastore_host = "192.168.1.10:8443"
+```
+
+### S3 Bucket Creation Fails
+
+Verify Predastore is running and accessible:
+
+```bash
+curl -k https://localhost:8443/
+aws s3 ls --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Flask App Not Starting
+
+SSH into the instance and check the service:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+sudo systemctl status s3-webapp
+sudo journalctl -u s3-webapp --no-pager -n 50
+```
+
+### Upload Fails or Files Not Appearing
+
+Check that the S3 credentials in `/opt/webapp/.env` are correct and that the bucket exists:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+cat /opt/webapp/.env
+/opt/webapp/venv/bin/python -c "import boto3; print('boto3 OK')"
+```
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/s3-webapp/main.tf

@@ -0,0 +1,445 @@
+# Example 3: S3-Backed Web Application
+#
+# Deploys an EC2 instance running a simple file-sharing webapp backed by S3
+# (Predastore). Users can upload files through a web form and browse uploaded
+# content — demonstrating Terraform managing both compute (Spinifex) and
+# object storage (Predastore) resources together.
+#
+# Architecture:
+#
+#   Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
+#                         │
+#                         ▼ S3 API (boto3)
+#                     Predastore (port 8443)
+#
+# Prerequisites:
+#   - Spinifex services running (gateway on port 9999)
+#   - Predastore running (S3 API on port 8443)
+#   - The EC2 instance must be able to reach the Predastore endpoint.
+#     Set `predastore_host` to the IP reachable from inside the VPC
+#     (e.g. the host's br-wan IP, NOT localhost).
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/03-s3-webapp
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>          # File browser UI
+#   ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint (EC2/IAM)"
+}
+
+variable "predastore_endpoint" {
+  type        = string
+  default     = "https://localhost:8443"
+  description = "Predastore S3 endpoint (for Terraform to create buckets)"
+}
+
+variable "predastore_host" {
+  type        = string
+  description = "Predastore host:port reachable from inside the VPC (e.g. 192.168.1.10:8443)"
+}
+
+variable "s3_access_key" {
+  type        = string
+  description = "S3 access key for Predastore"
+}
+
+variable "s3_secret_key" {
+  type        = string
+  sensitive   = true
+  description = "S3 secret key for Predastore"
+}
+
+variable "bucket_name" {
+  type    = string
+  default = "webapp-uploads"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — EC2 via Spinifex gateway, S3 via Predastore
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region     = var.region
+  access_key = var.s3_access_key
+  secret_key = var.s3_secret_key
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    s3  = var.predastore_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  s3_use_path_style = true
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# S3 Bucket (created on Predastore)
+# ---------------------------------------------------------------------------
+
+resource "aws_s3_bucket" "uploads" {
+  bucket = var.bucket_name
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "webapp" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "webapp" {
+  key_name   = "s3-webapp-demo"
+  public_key = tls_private_key.webapp.public_key_openssh
+}
+
+resource "local_file" "webapp_pem" {
+  filename        = "${path.module}/s3-webapp-demo.pem"
+  content         = tls_private_key.webapp.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC + Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.30.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "s3-webapp-demo-vpc"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "s3-webapp-demo-igw"
+  }
+}
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.30.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "s3-webapp-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "webapp" {
+  name        = "s3-webapp-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Flask webapp that talks to Predastore S3
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "webapp" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.webapp.id]
+  key_name               = aws_key_pair.webapp.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install dependencies
+    apt-get update -y
+    apt-get install -y python3-pip python3-venv
+
+    # Create app directory and virtualenv
+    mkdir -p /opt/webapp
+    python3 -m venv /opt/webapp/venv
+    /opt/webapp/venv/bin/pip install flask boto3
+
+    # Write S3 credentials config
+    cat > /opt/webapp/.env <<'ENVFILE'
+    S3_ENDPOINT=https://${var.predastore_host}
+    S3_BUCKET=${var.bucket_name}
+    S3_ACCESS_KEY=${var.s3_access_key}
+    S3_SECRET_KEY=${var.s3_secret_key}
+    S3_REGION=${var.region}
+    ENVFILE
+
+    # Write the Flask application
+    cat > /opt/webapp/app.py <<'PYEOF'
+    import os, io, urllib3
+    from flask import Flask, request, redirect, url_for, Response
+
+    # Suppress TLS warnings for self-signed certs
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    # Load env
+    env = {}
+    with open("/opt/webapp/.env") as f:
+        for line in f:
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, v = line.split("=", 1)
+                env[k] = v
+
+    import boto3
+    from botocore.config import Config
+
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=env["S3_ENDPOINT"],
+        aws_access_key_id=env["S3_ACCESS_KEY"],
+        aws_secret_access_key=env["S3_SECRET_KEY"],
+        region_name=env["S3_REGION"],
+        verify=False,
+        config=Config(s3={"addressing_style": "path"}),
+    )
+
+    BUCKET = env["S3_BUCKET"]
+    app = Flask(__name__)
+
+    @app.route("/")
+    def index():
+        # List objects in the bucket
+        try:
+            resp = s3.list_objects_v2(Bucket=BUCKET)
+            objects = resp.get("Contents", [])
+        except Exception as e:
+            objects = []
+
+        rows = ""
+        for obj in objects:
+            key = obj["Key"]
+            size = obj["Size"]
+            rows += f'<tr><td><a href="/files/{key}">{key}</a></td><td>{size} bytes</td></tr>\n'
+
+        return f"""<!DOCTYPE html>
+    <html>
+    <head><title>Spinifex S3 File Browser</title></head>
+    <body style="font-family: sans-serif; max-width: 700px; margin: 40px auto;">
+      <h1>Spinifex S3 File Browser</h1>
+      <p>Bucket: <code>{BUCKET}</code></p>
+
+      <h2>Upload a File</h2>
+      <form method="POST" action="/upload" enctype="multipart/form-data">
+        <input type="file" name="file" required>
+        <button type="submit">Upload</button>
+      </form>
+
+      <h2>Files</h2>
+      <table border="1" cellpadding="6" cellspacing="0" style="border-collapse: collapse;">
+        <tr><th>Key</th><th>Size</th></tr>
+        {rows if rows else '<tr><td colspan="2">No files yet</td></tr>'}
+      </table>
+
+      <hr>
+      <p><small>Powered by Spinifex + Predastore</small></p>
+    </body>
+    </html>"""
+
+    @app.route("/upload", methods=["POST"])
+    def upload():
+        f = request.files.get("file")
+        if not f or not f.filename:
+            return redirect("/")
+        s3.put_object(Bucket=BUCKET, Key=f.filename, Body=f.read())
+        return redirect("/")
+
+    @app.route("/files/<path:key>")
+    def download(key):
+        try:
+            obj = s3.get_object(Bucket=BUCKET, Key=key)
+            return Response(
+                obj["Body"].read(),
+                headers={"Content-Disposition": f'inline; filename="{key}"'},
+            )
+        except Exception:
+            return "Not found", 404
+
+    if __name__ == "__main__":
+        app.run(host="0.0.0.0", port=80)
+    PYEOF
+
+    # Create a systemd service so the webapp starts on boot
+    cat > /etc/systemd/system/s3-webapp.service <<'SVCEOF'
+    [Unit]
+    Description=S3 File Browser Webapp
+    After=network.target
+
+    [Service]
+    Type=simple
+    ExecStart=/opt/webapp/venv/bin/python /opt/webapp/app.py
+    WorkingDirectory=/opt/webapp
+    Restart=always
+    RestartSec=3
+
+    [Install]
+    WantedBy=multi-user.target
+    SVCEOF
+
+    systemctl daemon-reload
+    systemctl enable s3-webapp
+    systemctl start s3-webapp
+  USERDATA
+  )
+
+  tags = {
+    Name = "s3-webapp-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.webapp.id
+}
+
+output "public_ip" {
+  value = aws_instance.webapp.public_ip
+}
+
+output "bucket_name" {
+  value = aws_s3_bucket.uploads.id
+}
+
+output "ssh_command" {
+  value = "ssh -i s3-webapp-demo.pem ec2-user@${aws_instance.webapp.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.webapp.public_ip}"
+}

docs/terraform/s3-webapp/terraform.tfvars.example

@@ -0,0 +1,14 @@
+# Copy this to terraform.tfvars and fill in your values.
+#
+# The predastore_host must be reachable from INSIDE the VPC guest — not
+# localhost. Use the host's br-wan or LAN IP, e.g. "192.168.1.10:8443".
+
+predastore_host = "192.168.1.10:8443"
+s3_access_key   = "AKIAIOSFODNN7EXAMPLE"
+s3_secret_key   = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+
+# Optional overrides:
+# spinifex_endpoint    = "https://localhost:9999"
+# predastore_endpoint  = "https://localhost:8443"
+# bucket_name          = "webapp-uploads"
+# region               = "ap-southeast-2"

go.mod

@@ -8,12 +8,12 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.1.1
-	github.com/mulgadc/viperblock v1.1.1
-	github.com/nats-io/nats-server/v2 v2.12.4
-	github.com/nats-io/nats.go v1.49.0
+	github.com/mulgadc/predastore v1.1.4
+	github.com/mulgadc/viperblock v1.1.5
+	github.com/nats-io/nats-server/v2 v2.12.6
+	github.com/nats-io/nats.go v1.50.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.2.4
+	github.com/pelletier/go-toml/v2 v2.3.0
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
@@ -25,7 +25,7 @@ require (
 	atomicgo.dev/cursor v0.2.0 // indirect
 	atomicgo.dev/keyboard v0.2.9 // indirect
 	atomicgo.dev/schedule v0.1.0 // indirect
-	github.com/antithesishq/antithesis-sdk-go v0.6.0 // indirect
+	github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -40,7 +40,7 @@ require (
 	github.com/dgraph-io/badger/v4 v4.9.1 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fatih/color v1.18.0 // indirect
+	github.com/fatih/color v1.19.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
@@ -62,17 +62,17 @@ require (
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
-	github.com/klauspost/reedsolomon v1.13.2 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/klauspost/reedsolomon v1.13.3 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.21 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
-	github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect
+	github.com/minio/highwayhash v1.0.4 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nats-io/jwt/v2 v2.8.0 // indirect
+	github.com/nats-io/jwt/v2 v2.8.1 // indirect
 	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -89,21 +89,21 @@ require (
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel v1.41.0 // indirect
-	go.opentelemetry.io/otel/metric v1.41.0 // indirect
-	go.opentelemetry.io/otel/trace v1.41.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/mod v0.33.0 // indirect
-	golang.org/x/net v0.51.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260304144227-18da59047661 // indirect
-	golang.org/x/term v0.40.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect

go.sum

@@ -22,8 +22,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antithesishq/antithesis-sdk-go v0.6.0 h1:v/YViLhFYkZOEEof4AXjD5AgGnGM84YHF4RqEwp6I2g=
-github.com/antithesishq/antithesis-sdk-go v0.6.0/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
+github.com/antithesishq/antithesis-sdk-go v0.7.0 h1:uWDG8BqLD1lI2ps38WDz2vXflrTX2+vLX0SvZtztJtE=
+github.com/antithesishq/antithesis-sdk-go v0.7.0/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
@@ -104,8 +104,8 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUn
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
-github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
+github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -212,15 +212,15 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg=
 github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.2 h1:9qtQy2tKEVpVB8Pfq87ZljHZb60/LbeTQ1OxV8EGzdE=
-github.com/klauspost/reedsolomon v1.13.2/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
+github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -244,31 +244,31 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ=
-github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
+github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
 github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
-github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
-github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
+github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
+github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.1.1 h1:rqpvA/qmewzJBHT2r2N9gNuUaLUKJrONFAyVu2zsRbY=
-github.com/mulgadc/predastore v1.1.1/go.mod h1:WFls3cw+0g0wYBDyKNjEvTihJxuMjIWe2lVNWcYtY7A=
-github.com/mulgadc/viperblock v1.1.1 h1:Z+G6JtR03lhWUGp39iC6pMOjBzYQfLEjyk8c3jcqVCQ=
-github.com/mulgadc/viperblock v1.1.1/go.mod h1:wlLRcHc/28+deiWWXLw+mXEDcmTv9WK2V3SVKkpoyp0=
+github.com/mulgadc/predastore v1.1.4 h1:czeGQWWqmmZuGnUuvyrORavySN/SGPTwjUxBd/9swuU=
+github.com/mulgadc/predastore v1.1.4/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
+github.com/mulgadc/viperblock v1.1.5 h1:ojmoJQrY7VkDGzdQj07w2F7ICRJX9RlGAKyn87Wa55o=
+github.com/mulgadc/viperblock v1.1.5/go.mod h1:utoIQLwCmPujOBaT0mL5NhW3m2fwjQPQiQRJH6+Wtk8=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
-github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
-github.com/nats-io/nats-server/v2 v2.12.4 h1:ZnT10v2LU2Xcoiy8ek9X6Se4YG8EuMfIfvAEuFVx1Ts=
-github.com/nats-io/nats-server/v2 v2.12.4/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
-github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
-github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
+github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
+github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8=
+github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo=
+github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=
+github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -277,8 +277,8 @@ github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
-github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
+github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -369,26 +369,26 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
-go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
-go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
-go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
-go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
-go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
 go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
-go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
-go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -405,8 +405,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -415,8 +415,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -445,8 +445,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20260304144227-18da59047661 h1:AjMsnhVxCTd9HssPcdXPhNTQ0YazDt5+52EqmcWQ0PA=
 golang.org/x/telemetry v0.0.0-20260304144227-18da59047661/go.mod h1:NuITXsA9cTiqnXtVk+/wrBT2Ja4X5hsfGOYRJ6kgYjs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -454,18 +454,18 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

scripts/build-alb-image.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# build-alb-image.sh — Thin wrapper for backward compatibility
+# Delegates to the generic build-system-image.sh with the ALB manifest.
+exec "$(dirname "$0")/build-system-image.sh" "$(dirname "$0")/images/alb.conf" "$@"

scripts/build-system-image.sh

@@ -0,0 +1,275 @@
+#!/bin/bash
+set -euo pipefail
+
+# build-system-image.sh — Build a minimal Alpine AMI from a manifest
+#
+# Creates a pre-baked Alpine Linux image with custom packages, binaries,
+# and setup scripts installed, ready for import as a Spinifex AMI.
+#
+# Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
+# Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
+#   --import  Also import the image as an AMI via spx admin images import
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+usage() {
+    echo "Usage: $0 <manifest.conf> [--import]"
+    echo ""
+    echo "  manifest.conf  Path to image manifest (see scripts/images/ for examples)"
+    echo "  --import       Import the built image as an AMI"
+    exit 1
+}
+
+if [[ $# -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
+    usage
+fi
+
+MANIFEST="$1"
+shift
+
+if [[ ! -f "$MANIFEST" ]]; then
+    echo "ERROR: Manifest not found: $MANIFEST"
+    exit 1
+fi
+
+DO_IMPORT=false
+if [[ "${1:-}" == "--import" ]]; then
+    DO_IMPORT=true
+fi
+
+# Source the manifest
+# shellcheck source=/dev/null
+source "$MANIFEST"
+
+# Validate required manifest fields
+for field in IMAGE_NAME ALPINE_VERSION IMAGE_SIZE; do
+    if [[ -z "${!field:-}" ]]; then
+        echo "ERROR: Manifest missing required field: $field"
+        exit 1
+    fi
+done
+
+# Derived paths
+BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
+ALPINE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
+ALPINE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${ALPINE_IMAGE}"
+OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
+OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
+NBD_DEV="/dev/nbd0"
+MOUNT_DIR="${BUILD_DIR}/mnt"
+
+cleanup() {
+    echo "Cleaning up..."
+    sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+    sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+    echo "Done."
+}
+trap cleanup EXIT
+
+echo "=== System Image Builder ==="
+echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
+echo "Alpine:  ${ALPINE_VERSION}"
+echo "Size:    ${IMAGE_SIZE}"
+echo "Build:   ${BUILD_DIR}"
+echo ""
+
+# Step 0: Check prerequisites
+if ! command -v qemu-nbd &>/dev/null; then
+    echo "ERROR: qemu-nbd not found. Install qemu-utils."
+    exit 1
+fi
+
+if ! command -v qemu-img &>/dev/null; then
+    echo "ERROR: qemu-img not found. Install qemu-utils."
+    exit 1
+fi
+
+# Build binaries if BUILD_COMMANDS is set
+if [[ -n "${BUILD_COMMANDS:-}" ]]; then
+    echo "Building binaries: ${BUILD_COMMANDS}"
+    if ! (cd "$PROJECT_DIR" && eval "$BUILD_COMMANDS"); then
+        echo "ERROR: BUILD_COMMANDS failed: ${BUILD_COMMANDS}"
+        exit 1
+    fi
+fi
+
+# Verify binaries exist and are statically linked
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        src_path="${PROJECT_DIR}/${src}"
+        if [[ ! -f "$src_path" ]]; then
+            echo "ERROR: Binary not found: $src_path"
+            exit 1
+        fi
+        if ! file "$src_path" | grep -q "statically linked"; then
+            echo "ERROR: $src_path is not statically linked (Alpine uses musl — dynamic glibc binaries will fail)"
+            echo "  Rebuild with: CGO_ENABLED=0 go build ..."
+            exit 1
+        fi
+    done
+fi
+
+mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
+
+# Step 1: Download Alpine cloud image
+if [[ -f "${BUILD_DIR}/${ALPINE_IMAGE}" ]]; then
+    echo "Alpine image already downloaded."
+else
+    echo "Downloading Alpine ${ALPINE_VERSION} cloud image..."
+    if ! curl --fail -L -o "${BUILD_DIR}/${ALPINE_IMAGE}" "$ALPINE_URL"; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Failed to download Alpine image from $ALPINE_URL"
+        exit 1
+    fi
+    # Verify the download is a valid qcow2 image
+    if ! qemu-img info "${BUILD_DIR}/${ALPINE_IMAGE}" &>/dev/null; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Downloaded file is not a valid qcow2 image"
+        exit 1
+    fi
+fi
+
+# Step 2: Copy image for customization
+echo "Copying image for customization..."
+cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
+
+# Resize the image (Alpine cloud images are ~200MB, need room for packages)
+qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
+
+# Step 3: Connect via qemu-nbd
+echo "Connecting image via qemu-nbd..."
+sudo modprobe nbd max_part=4 2>/dev/null || true
+if [[ ! -e "${NBD_DEV}" ]]; then
+    echo "ERROR: ${NBD_DEV} does not exist. Is the nbd kernel module loaded? Try: sudo modprobe nbd"
+    exit 1
+fi
+sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
+sleep 1
+
+# Alpine cloud images have ext4 directly on the block device (no partition table).
+# Resize the filesystem to fill the resized image.
+echo "Checking filesystem..."
+sudo e2fsck -f -y "${NBD_DEV}" || {
+    ec=$?
+    if [[ $ec -gt 1 ]]; then
+        echo "ERROR: e2fsck failed with exit code $ec on ${NBD_DEV}"
+        exit 1
+    fi
+}
+
+echo "Resizing filesystem..."
+if ! sudo resize2fs "${NBD_DEV}"; then
+    echo "ERROR: resize2fs failed on ${NBD_DEV}"
+    exit 1
+fi
+
+# Step 4: Mount and customize
+echo "Mounting root filesystem..."
+sudo mount "${NBD_DEV}" "$MOUNT_DIR"
+
+# Set up resolv.conf for DNS inside chroot
+sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
+
+# Install packages
+if [[ -n "${APK_PACKAGES:-}" ]]; then
+    echo "Installing packages: ${APK_PACKAGES}..."
+    sudo chroot "$MOUNT_DIR" /bin/sh -c "
+set -e
+# Enable community repo
+sed -i 's|^#\(.*community\)|\1|' /etc/apk/repositories 2>/dev/null || true
+
+# Ensure community repo is present
+if ! grep -q community /etc/apk/repositories; then
+    MIRROR=\$(grep main /etc/apk/repositories | head -1 | sed 's|/main|/community|')
+    echo \"\$MIRROR\" >> /etc/apk/repositories
+fi
+
+apk update
+apk add ${APK_PACKAGES}
+"
+fi
+
+# Enable OpenRC services
+if [[ -n "${ENABLE_SERVICES:-}" ]]; then
+    echo "Enabling services: ${ENABLE_SERVICES}..."
+    IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
+    for svc in "${SERVICES[@]}"; do
+        if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
+            echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
+            exit 1
+        fi
+    done
+fi
+
+# Step 5: Copy binaries into the image (before setup script, which may reference them)
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    echo "Installing binaries..."
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        dst="${pair#*:}"
+        src_path="${PROJECT_DIR}/${src}"
+        echo "  ${src} -> ${dst}"
+        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
+        sudo chmod 755 "${MOUNT_DIR}${dst}"
+    done
+fi
+
+# Run custom setup script inside chroot (after binaries are installed)
+if [[ -n "${SETUP_SCRIPT:-}" ]]; then
+    setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
+    if [[ ! -f "$setup_path" ]]; then
+        echo "ERROR: Setup script not found: $setup_path"
+        exit 1
+    fi
+    echo "Running setup script: ${SETUP_SCRIPT}..."
+    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
+fi
+
+# Step 6: Clean up and unmount
+echo "Cleaning up image..."
+sudo chroot "$MOUNT_DIR" /bin/sh -c '
+apk cache clean 2>/dev/null || true
+rm -rf /var/cache/apk/* /tmp/*
+'
+
+# Restore original resolv.conf (cloud-init will set it on boot)
+sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
+
+echo "Unmounting..."
+sudo umount "$MOUNT_DIR"
+sudo qemu-nbd --disconnect "${NBD_DEV}"
+
+# Step 7: Convert to raw for import
+echo "Converting to raw format..."
+qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
+
+echo ""
+echo "=== Build complete ==="
+echo "  Image: ${IMAGE_NAME}"
+echo "  qcow2: $OUTPUT_IMAGE ($(du -h "$OUTPUT_IMAGE" | cut -f1))"
+echo "  raw:   $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
+echo ""
+
+if [[ "$DO_IMPORT" == true ]]; then
+    echo "Importing as AMI..."
+    (cd "$PROJECT_DIR" && ./bin/spx admin images import \
+        --file "$OUTPUT_RAW" \
+        --distro alpine \
+        --version "${ALPINE_VERSION}-${IMAGE_NAME}" \
+        --arch x86_64 \
+        --config "$HOME/spinifex/config/spinifex.toml")
+else
+    echo "To import as AMI, run:"
+    echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
+    echo "    --file $OUTPUT_RAW \\"
+    echo "    --distro alpine --version ${ALPINE_VERSION}-${IMAGE_NAME} --arch x86_64 \\"
+    echo "    --config \$HOME/spinifex/config/spinifex.toml"
+fi

scripts/clone-deps.sh

@@ -3,7 +3,6 @@
 # Spinifex Development Dependencies Setup Script
 # This script clones Viperblock and Predastore repositories for cross-repo development
 
-# TODO: Improve, remove CLAUDE readme reference, simplify next-steps
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

scripts/coverage.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Run Go tests with coverage and report total code coverage.
-
-COVERPROFILE=$(mktemp /tmp/spinifex-coverage-XXXXXX.out)
-trap 'rm -f "$COVERPROFILE"' EXIT
-
-echo "Running tests with coverage..."
-echo ""
-LOG_IGNORE=1 go test -timeout 120s -coverprofile="$COVERPROFILE" -covermode=atomic ./spinifex/... 2>&1
-
-if [[ ! -s "$COVERPROFILE" ]]; then
-    echo "No coverage data collected."
-    exit 1
-fi
-
-echo ""
-echo "=== Total ==="
-go tool cover -func="$COVERPROFILE" | awk '/^total:/ { print $NF }'

scripts/dev-setup.sh

@@ -172,14 +172,10 @@ echo ""
 echo "   1. Setup OVN networking:"
 echo "      ./scripts/setup-ovn.sh --management"
 echo ""
-echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, and certs):"
-echo "      ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
+echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):"
+echo "      sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
 echo ""
-echo "   3. Trust the CA certificate:"
-echo "      sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt"
-echo "      sudo update-ca-certificates"
-echo ""
-echo "   4. Start services:"
+echo "   3. Start services:"
 echo "      ./scripts/start-dev.sh"
 echo ""
 echo "🔧 Development endpoints:"

scripts/diff-coverage.sh

@@ -7,13 +7,12 @@ set -euo pipefail
 # Usage: scripts/diff-coverage.sh <coverprofile> [--base <ref>] [--threshold <pct>]
 #
 # Base ref auto-detection:
-#   dev branch    → origin/main
 #   main branch   → HEAD~1
-#   other branch  → origin/dev
+#   other branch  → origin/main
 
 PROFILE=""
 BASE_REF=""
-THRESHOLD=60
+THRESHOLD=65
 QUIET="${QUIET:-}"
 
 while [[ $# -gt 0 ]]; do
@@ -45,8 +44,7 @@ if [[ -z "$BASE_REF" ]]; then
     BRANCH="${GITHUB_REF_NAME:-$(git rev-parse --abbrev-ref HEAD)}"
     case "$BRANCH" in
         main) BASE_REF="HEAD~1" ;;
-        dev)  BASE_REF="origin/main" ;;
-        *)    BASE_REF="origin/dev" ;;
+        *)    BASE_REF="origin/main" ;;
     esac
     [[ -z "$QUIET" ]] && echo "Base ref: $BASE_REF (branch: $BRANCH)"
 fi

scripts/iac/aws/basic/main.tf

@@ -0,0 +1,307 @@
+# Spinifex IaC Compatibility Test — "basic" suite
+#
+# Exercises every AWS resource Spinifex currently supports via tofu apply,
+# WITHOUT route tables, NACLs, or other unimplemented networking.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/basic
+#   tofu init
+#   tofu plan
+#   tofu apply        # full round-trip against a running Spinifex cluster
+#   tofu destroy      # clean teardown
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "mulgaos_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.mulgaos_endpoint
+    s3  = var.mulgaos_endpoint
+    iam = var.mulgaos_endpoint
+    sts = var.mulgaos_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 1. VPC + Subnet (no route table needed)
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "test" {
+  cidr_block           = "10.99.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "spx-basic-test-vpc"
+  }
+}
+
+resource "aws_subnet" "test" {
+  vpc_id                  = aws_vpc.test.id
+  cidr_block              = "10.99.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "spx-basic-test-subnet"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 2. Internet Gateway (attach to VPC, but no route table referencing it)
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "test" {
+  vpc_id = aws_vpc.test.id
+
+  tags = {
+    Name = "spx-basic-test-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 3. Security Group — inbound SSH + HTTP, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "test" {
+  name        = "spx-basic-test-sg"
+  description = "Basic test: SSH + HTTP inbound, all outbound"
+  vpc_id      = aws_vpc.test.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "spx-basic-test-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 4. Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "test" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "test" {
+  key_name   = "spx-basic-test-key"
+  public_key = tls_private_key.test.public_key_openssh
+}
+
+resource "local_file" "test_pem" {
+  filename        = "${path.module}/spx-basic-test.pem"
+  content         = tls_private_key.test.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# 5. Placement Group
+# ---------------------------------------------------------------------------
+
+resource "aws_placement_group" "test" {
+  name     = "spx-basic-test-spread"
+  strategy = "spread"
+}
+
+# ---------------------------------------------------------------------------
+# 6. EC2 Instance
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "test" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.test.id
+  vpc_security_group_ids = [aws_security_group.test.id]
+  key_name               = aws_key_pair.test.key_name
+  placement_group        = aws_placement_group.test.name
+
+  associate_public_ip_address = true
+
+  tags = {
+    Name        = "spx-basic-test-vm"
+    Environment = "test"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 7. EBS Volume + Attachment
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_volume" "data" {
+  availability_zone = data.aws_availability_zones.available.names[0]
+  size              = 10
+  type              = "gp3"
+
+  tags = {
+    Name = "spx-basic-test-data-vol"
+  }
+}
+
+
+# ---------------------------------------------------------------------------
+# 8. EBS Snapshot (of the standalone volume)
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_snapshot" "data" {
+  volume_id = aws_ebs_volume.data.id
+
+  tags = {
+    Name = "spx-basic-test-snapshot"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 9. Elastic IP + Association
+#    Requires external IPAM (pool mode) — uncomment when available.
+# ---------------------------------------------------------------------------
+
+# resource "aws_eip" "test" {
+#   domain = "vpc"
+#
+#   tags = {
+#     Name = "spx-basic-test-eip"
+#   }
+# }
+#
+# resource "aws_eip_association" "test" {
+#   allocation_id = aws_eip.test.id
+#   instance_id   = aws_instance.test.id
+# }
+
+# ---------------------------------------------------------------------------
+# 10. Standalone Network Interface (not attached — just tests CRUD)
+# ---------------------------------------------------------------------------
+
+resource "aws_network_interface" "extra" {
+  subnet_id       = aws_subnet.test.id
+  security_groups = [aws_security_group.test.id]
+
+  tags = {
+    Name = "spx-basic-test-eni"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "vpc_id" {
+  value = aws_vpc.test.id
+}
+
+output "instance_id" {
+  value = aws_instance.test.id
+}
+
+output "instance_public_ip" {
+  value = aws_instance.test.public_ip
+}
+
+# output "eip_public_ip" {
+#   value = aws_eip.test.public_ip
+# }
+
+output "volume_id" {
+  value = aws_ebs_volume.data.id
+}
+
+output "snapshot_id" {
+  value = aws_ebs_snapshot.data.id
+}
+
+output "ssh_command" {
+  value = "ssh -i spx-basic-test.pem admin@${aws_instance.test.public_ip}"
+}

scripts/iac/aws/main.tf

@@ -26,10 +26,10 @@ variable "region" {
 # Set these environment variables or use terraform.tfvars:
 #   export AWS_ACCESS_KEY_ID=your_access_key
 #   export AWS_SECRET_ACCESS_KEY=your_secret_key
-#   export TF_VAR_mulgaos_endpoint=http://localhost:8080
+#   export TF_VAR_mulgaos_endpoint=https://localhost:9999
 variable "mulgaos_endpoint" {
   type        = string
-  default     = "http://localhost:8080"
+  default     = "https://localhost:9999"
   description = "MulgaOS API endpoint (awsd gateway)"
 }
 
@@ -57,15 +57,15 @@ data "aws_availability_zones" "available" {
   state = "available"
 }
 
-# Debian 12 AMI (official Debian account)
-# For MulgaOS, you may need to import a local AMI first
+# Debian 12 AMI
+# For MulgaOS, need to import a local AMI first
 data "aws_ami" "debian12" {
   most_recent = true
-  owners      = ["136693071363"] # Debian official
+  owners      = ["000000000000"] # Spinifex system images (on AWS use "136693071363" for Debian official)
 
   filter {
     name   = "name"
-    values = ["debian-12-amd64-*"]
+    values = ["*debian-12*"]
   }
 
   filter {
@@ -96,9 +96,9 @@ resource "local_file" "hive_test_pem" {
   file_permission = "0600"
 }
 
-# 1: VPC 10.1.1.0/16
+# 1: VPC 10.1.0.0/16
 resource "aws_vpc" "main" {
-  cidr_block           = "10.1.1.0/16"
+  cidr_block           = "10.1.0.0/16"
   enable_dns_hostnames = true
   enable_dns_support   = true
 
@@ -189,12 +189,6 @@ resource "aws_security_group" "web_ssh" {
   }
 }
 
-# 4: Ensure 3 servers on different underlying hardware (spread placement group)
-resource "aws_placement_group" "spread" {
-  name     = "spinifex-test-spread"
-  strategy = "spread"
-}
-
 resource "aws_instance" "vm" {
   count         = 3
   ami           = data.aws_ami.debian12.id
@@ -205,7 +199,6 @@ resource "aws_instance" "vm" {
   key_name               = aws_key_pair.hive_test.key_name
 
   associate_public_ip_address = true
-  placement_group             = aws_placement_group.spread.name
 
   tags = {
     Name = "spinifex-test-vm-${count.index + 1}"

scripts/iac/configure-cluster.sh

@@ -110,7 +110,7 @@ INIT_PID=$!
 # Wait for formation server to be ready
 echo "    Waiting for formation server on $NODE1_IP..."
 attempts=0
-while ! curl -s --connect-timeout 3 "http://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
+while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
     attempts=$((attempts + 1))
     if [ $attempts -ge 60 ]; then
         echo "Error: Formation server did not start on $NODE1_IP after 5 minutes"

scripts/images/alb-setup.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+# alb-setup.sh — ALB-specific chroot setup
+#
+# Runs inside the Alpine image chroot after packages are installed.
+# Sets up HAProxy placeholder config and alb-agent OpenRC init script.
+
+set -e
+
+# Create haproxy config directory and placeholder config
+mkdir -p /etc/haproxy
+cat > /etc/haproxy/haproxy.cfg <<'EOF'
+# Placeholder config — replaced by alb-agent on first config push
+global
+    daemon
+    maxconn 256
+
+defaults
+    mode http
+    timeout connect 5s
+    timeout client 30s
+    timeout server 30s
+
+frontend health
+    bind *:8405
+    http-request return status 200 content-type text/plain string "ok"
+EOF
+
+# Create alb-agent OpenRC init script
+mkdir -p /etc/init.d
+cat > /etc/init.d/alb-agent <<'INITSCRIPT'
+#!/sbin/openrc-run
+
+description="ALB NATS Config Agent"
+command="/usr/local/bin/alb-agent"
+command_args="--lb-id=${ALB_LB_ID:-unknown} --nats=${ALB_NATS_URL:-nats://10.0.2.2:4222}"
+command_background=true
+pidfile="/run/alb-agent.pid"
+output_log="/var/log/alb-agent.log"
+error_log="/var/log/alb-agent.log"
+
+depend() {
+    need net
+    after firewall
+}
+INITSCRIPT
+chmod 755 /etc/init.d/alb-agent

scripts/images/alb.conf

@@ -0,0 +1,22 @@
+# ALB system image manifest
+# Used by build-system-image.sh to create an Alpine AMI with HAProxy + alb-agent
+
+IMAGE_NAME="alb"
+IMAGE_DESCRIPTION="ALB VM with HAProxy and alb-agent"
+ALPINE_VERSION="3.21.6"
+IMAGE_SIZE="512M"
+
+# Packages to install via apk
+APK_PACKAGES="haproxy curl"
+
+# Binaries to copy into the image (source:destination)
+INSTALL_BINARIES="bin/alb-agent:/usr/local/bin/alb-agent"
+
+# Build command for binaries (run before image build)
+BUILD_COMMANDS="make build-alb-agent"
+
+# OpenRC services to enable at boot
+ENABLE_SERVICES="haproxy"
+
+# Custom setup script (runs inside chroot after packages are installed)
+SETUP_SCRIPT="scripts/images/alb-setup.sh"

scripts/reset-dev-env.sh

@@ -1,9 +1,34 @@
 #!/bin/bash
 
 # Reset dev environment (will purge all data)
+# This script is for single-node dev environments only.
 
 set -euo pipefail
 
+CONFIG_FILE="$HOME/spinifex/config/spinifex.toml"
+
+# --- Guard: refuse to run on multi-node clusters ---
+if [ -f "$CONFIG_FILE" ]; then
+    NODE_COUNT=$(grep -cE '^\[nodes\.[^.]+\]' "$CONFIG_FILE")
+    if [ "$NODE_COUNT" -gt 1 ]; then
+        echo "❌ Multi-node cluster detected ($NODE_COUNT nodes in $CONFIG_FILE)."
+        echo "   This script only supports single-node dev environments."
+        echo "   Reset each node individually or use 'spx admin cluster shutdown'."
+        exit 1
+    fi
+fi
+
+# Save region from existing config before we delete everything
+REGION="ap-southeast-2"
+if [ -f "$CONFIG_FILE" ]; then
+    SAVED_REGION=$(grep -E '^\s*region\s*=' "$CONFIG_FILE" | tail -1 | sed 's/.*=\s*"\(.*\)"/\1/')
+    if [ -n "$SAVED_REGION" ]; then
+        REGION="$SAVED_REGION"
+    fi
+fi
+echo "Using region: $REGION"
+
+# --- Shutdown services ---
 echo "Shutting down services"
 
 if ! ./scripts/stop-dev.sh; then
@@ -11,47 +36,266 @@ if ! ./scripts/stop-dev.sh; then
     exit 1
 fi
 
-echo "Removing data"
+# Verify no spinifex-related processes are still running
+PROCESS_PATTERN='(bin/spx|spinifex-ui|nats-server|predastore|viperblock|vpcd)'
+remaining=$(pgrep -af "$PROCESS_PATTERN" | grep -v "reset-dev-env.sh" || true)
+if [ -n "$remaining" ]; then
+    echo "Waiting for remaining processes to exit..."
+    timeout=30
+    elapsed=0
+    while pgrep -af "$PROCESS_PATTERN" | grep -qv "reset-dev-env.sh" 2>/dev/null; do
+        if [ $elapsed -ge $timeout ]; then
+            echo "❌ Processes still running after ${timeout}s:"
+            pgrep -af "$PROCESS_PATTERN" | grep -v "reset-dev-env.sh" || true
+            echo "   Kill them manually and re-run this script."
+            exit 1
+        fi
+        sleep 1
+        elapsed=$((elapsed + 1))
+    done
+fi
 
-rm -rf ~/spinifex/predastore/
+# Verify no QEMU VMs are running
+if pgrep -x qemu-system-x86_64 > /dev/null 2>&1; then
+    echo "❌ QEMU instances still running. Cannot reset while VMs are active."
+    echo "   Run './scripts/stop-dev.sh' or kill them manually."
+    exit 1
+fi
 
-rm -rf ~/spinifex/spinifex/*
+echo "All services confirmed stopped"
 
-rm -rf ~/spinifex/viperblock/*
+# --- Clean OVS/OVN ---
+echo "Removing OVS bridges and config"
 
-rm -rf ~/spinifex/nats/*
+if command -v ovs-vsctl >/dev/null 2>&1; then
+    # Ensure OVS is running so we can clean up
+    sudo systemctl start openvswitch-switch 2>/dev/null || true
+    sleep 1
 
-rm -rf ~/spinifex/images/*
+    # Delete all OVS bridges (br-int, br-external, etc.)
+    for br in $(sudo ovs-vsctl list-br 2>/dev/null); do
+        echo "  Deleting bridge: $br"
+        sudo ovs-vsctl --if-exists del-br "$br"
+    done
+
+    # Clear OVN external_ids
+    sudo ovs-vsctl --if-exists clear Open_vSwitch . external_ids 2>/dev/null || true
+    echo "  Cleared OVS external_ids"
+
+    # Stop OVS again after cleanup
+    sudo systemctl stop openvswitch-switch 2>/dev/null || true
+fi
+
+# Clean OVN databases (both Northbound and Southbound)
+# Delete the DB files outright — setup-ovn.sh will restart ovn-central with fresh
+# empty databases. This eliminates stale SB state (chassis entries, port bindings,
+# datapath bindings) that accumulates across resets and causes ovn-controller to
+# enter a commit failure loop ("OVNSB commit failed, force recompute").
+echo "Removing OVN database files"
+sudo systemctl stop ovn-central 2>/dev/null || true
+sudo systemctl stop ovn-controller 2>/dev/null || true
+if [ -d /var/lib/ovn ]; then
+    sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
+    echo "  Deleted /var/lib/ovn/ovn{nb,sb}_db.db"
+else
+    echo "  /var/lib/ovn not found, skipping OVN DB cleanup"
+fi
+
+# Remove veth pair created by setup-ovn.sh (veth mode — Linux bridge ↔ OVS bridge)
+if ip link show veth-wan-br >/dev/null 2>&1; then
+    echo "  Deleting veth pair: veth-wan-br ↔ veth-wan-ovs"
+    sudo ip link del veth-wan-br 2>/dev/null || true
+fi
+
+# Remove macvlan interfaces created by setup-ovn.sh
+for iface in $(ip -o link show type macvlan 2>/dev/null | awk -F': ' '{print $2}' | grep '^spx-ext-'); do
+    echo "  Deleting macvlan: $iface"
+    sudo ip link del "$iface" 2>/dev/null || true
+done
+
+# --- Wipe data ---
+echo "Removing ~/spinifex"
+rm -rf ~/spinifex
+
+# --- Re-initialize ---
+# Detect WAN interface (default route).
+WAN_IFACE=$(ip -4 route show default | awk '{print $5}' | head -1)
+WAN_GW=$(ip -4 route show default | awk '{print $3}' | head -1)
+
+echo "Detected WAN interface: ${WAN_IFACE:-none}, gateway: ${WAN_GW:-none}"
+
+# Check if WAN interface is already a bridge (tofu-cluster / production setup).
+# If so, setup-ovn.sh auto-detects it. If it's a physical NIC, we need to
+# decide between direct bridge and macvlan based on SSH safety.
+WAN_IS_BRIDGE=false
+if [ -n "$WAN_IFACE" ]; then
+    if ip -d link show "$WAN_IFACE" 2>/dev/null | grep -q "bridge"; then
+        WAN_IS_BRIDGE=true
+    fi
+fi
+
+# Build setup-ovn.sh flags based on detected topology
+SETUP_OVN_FLAGS=""
+if [ "$WAN_IS_BRIDGE" = true ]; then
+    # WAN is already a bridge (e.g. br-wan from cloud-init) — setup-ovn.sh
+    # auto-detects it, no explicit flags needed.
+    echo "  WAN is a bridge: $WAN_IFACE (auto-detected)"
+elif [ -n "$WAN_IFACE" ]; then
+    # WAN is a physical NIC. Detect SSH NIC to decide bridge strategy.
+    SSH_NIC=""
+    if [ -n "${SSH_CONNECTION:-}" ]; then
+        SSH_IP=$(echo "$SSH_CONNECTION" | awk '{print $3}')
+        SSH_NIC=$(ip -o -4 addr show | awk -v ip="$SSH_IP" '$0 ~ ip"/" {print $2}' | head -1)
+    fi
+    if [ -z "$SSH_NIC" ]; then
+        SSH_NIC="$WAN_IFACE"
+    fi
+
+    if [ "$WAN_IFACE" != "$SSH_NIC" ]; then
+        SETUP_OVN_FLAGS="--wan-bridge=br-wan --wan-iface=$WAN_IFACE"
+        echo "  Bridge mode: direct (WAN=$WAN_IFACE != SSH=$SSH_NIC)"
+    else
+        SETUP_OVN_FLAGS="--macvlan --wan-iface=$WAN_IFACE"
+        echo "  Bridge mode: macvlan (WAN=$WAN_IFACE == SSH=$SSH_NIC)"
+    fi
+fi
+
+# Chassis ID — let setup-ovn.sh auto-detect from hostname. This ensures the
+# system-id matches what ovn-controller registers in the SBDB, which is what
+# vpcd discovers at startup for gateway scheduling.
+
+echo "Re-initializing OVN"
+./scripts/setup-ovn.sh --management $SETUP_OVN_FLAGS
+
+echo "Initializing platform"
+ADMIN_INIT_ARGS="--region $REGION --az ${REGION}a --node node1 --nodes 1"
+
+# External networking mode: set EXTERNAL_MODE=nat for NAT/DHCP mode (outbound-only),
+# or leave unset / EXTERNAL_MODE=pool for pool mode (per-VM public IPs).
+EXTERNAL_MODE="${EXTERNAL_MODE:-pool}"
+
+if [ -n "$WAN_IFACE" ] && [ -n "$WAN_GW" ]; then
+    WAN_IP=$(ip -4 -o addr show "$WAN_IFACE" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    WAN_PREFIX=$(ip -4 -o addr show "$WAN_IFACE" 2>/dev/null | awk '{print $4}' | cut -d/ -f2 | head -1)
+    if [ -z "$WAN_PREFIX" ]; then WAN_PREFIX=24; fi
+
+    if [ "$EXTERNAL_MODE" = "nat" ]; then
+        # NAT mode: outbound-only via shared SNAT. Use a single IP from the
+        # high end of the WAN subnet as the gateway IP. No per-VM public IPs.
+        IFS='.' read -r o1 o2 o3 o4 <<< "$WAN_GW"
+        GATEWAY_IP="${GATEWAY_IP:-${o1}.${o2}.${o3}.200}"
+        echo "  External mode: nat (gateway IP: $GATEWAY_IP, WAN gateway: $WAN_GW)"
+        ADMIN_INIT_ARGS="$ADMIN_INIT_ARGS --external-mode=nat --gateway-ip=${GATEWAY_IP} --external-gateway=${WAN_GW} --external-prefix-len=${WAN_PREFIX}"
+    else
+        # Pool mode: per-VM public IPs from a static range
+        IFS='.' read -r o1 o2 o3 o4 <<< "$WAN_GW"
+        POOL_START="${o1}.${o2}.${o3}.200"
+        POOL_END="${o1}.${o2}.${o3}.250"
+        echo "  External pool: $POOL_START - $POOL_END (gateway: $WAN_GW)"
+        ADMIN_INIT_ARGS="$ADMIN_INIT_ARGS --external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=${WAN_GW} --external-prefix-len=${WAN_PREFIX}"
+    fi
+fi
+
+# Re-initialize with --force to regenerate credentials, certs, config, and
+# update ~/.aws/credentials. Must carry the same args so external networking
+# config isn't lost.
+echo "Re-initializing platform..."
+./bin/spx admin init --force $ADMIN_INIT_ARGS
+
+# Generate SSH key if it doesn't exist
+if [ ! -f ~/.ssh/spinifex-key.pub ]; then
+    echo "Generating SSH key pair..."
+    ssh-keygen -t ed25519 -f ~/.ssh/spinifex-key -N ""
+fi
+
+# Trust the Spinifex CA certificate so TLS connections to predastore/awsgw work
+if [ -f "$HOME/spinifex/config/ca.pem" ]; then
+    echo "Adding Spinifex CA certificate to system trust store..."
+    sudo cp "$HOME/spinifex/config/ca.pem" /usr/local/share/ca-certificates/spinifex-ca.crt
+    sudo update-ca-certificates
+fi
 
 # Enable pprof for development
-PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
-
-# Follow INSTALL.md steps
-echo "Importing AMI"
+#PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
+./scripts/start-dev.sh --build
 
 export AWS_PROFILE=spinifex
 
-# Step 1 SSH key
+# Import SSH key
+echo "Importing SSH key"
 aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/spinifex-key.pub
-
-echo "Validating key pair imported"
-
 aws ec2 describe-key-pairs
 
-# Detect architecture
-ARCH=$(uname -m)
-if [ "$ARCH" = "x86_64" ]; then
-    IMAGE_NAME="ubuntu-24.04-x86_64"
-elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
-    IMAGE_NAME="ubuntu-24.04-arm64"
-else
-    echo "Warning: Unknown architecture $ARCH, defaulting to x86_64"
-    IMAGE_NAME="ubuntu-24.04-x86_64"
-fi
+# Import AMI
+echo "Importing AMI"
 
-echo "Detected architecture: $ARCH, using image: $IMAGE_NAME"
-./bin/spx admin images import --name "$IMAGE_NAME"
+LOCAL_IMAGE="$HOME/images/ubuntu-24.04.img"
+if [ -f "$LOCAL_IMAGE" ]; then
+    echo "Using local image: $LOCAL_IMAGE"
+    ARCH=$(uname -m)
+    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+        IMG_ARCH="arm64"
+    else
+        IMG_ARCH="x86_64"
+    fi
+    ./bin/spx admin images import --file "$LOCAL_IMAGE" --distro ubuntu --version 24.04 --arch "$IMG_ARCH"
+else
+    # Download by name based on architecture
+    ARCH=$(uname -m)
+    if [ "$ARCH" = "x86_64" ]; then
+        IMAGE_NAME="ubuntu-24.04-x86_64"
+    elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+        IMAGE_NAME="ubuntu-24.04-arm64"
+    else
+        echo "Warning: Unknown architecture $ARCH, defaulting to x86_64"
+        IMAGE_NAME="ubuntu-24.04-x86_64"
+    fi
+    echo "Downloading image: $IMAGE_NAME"
+    ./bin/spx admin images import --name "$IMAGE_NAME"
+fi
 
 aws ec2 describe-images
 
-echo "Reset successful, fresh AMI imported, proceed to creating instances"
+# --- Launch a smoke-test instance ---
+echo "Launching smoke-test instance..."
+
+# Pick instance type based on CPU vendor
+if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+    INSTANCE_TYPE="t3a.small"
+else
+    INSTANCE_TYPE="t3.small"
+fi
+
+# Get the AMI we just imported
+AMI_ID=$(aws ec2 describe-images --query "Images[0].ImageId" --output text)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "❌ No AMI found, skipping instance launch"
+    exit 0
+fi
+
+# Find a public subnet (first available)
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --filters "Name=map-public-ip-on-launch,Values=true" \
+    --query "Subnets[0].SubnetId" --output text 2>/dev/null)
+# Fallback: just grab the first subnet if no public-tagged one exists
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    SUBNET_ID=$(aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text)
+fi
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "❌ No subnet found, skipping instance launch"
+    exit 0
+fi
+
+echo "  AMI: $AMI_ID"
+echo "  Instance type: $INSTANCE_TYPE"
+echo "  Subnet: $SUBNET_ID"
+
+INSTANCE_ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+
+echo "✅ Reset complete — instance $INSTANCE_ID launched ($INSTANCE_TYPE)"

scripts/setup-ovn.sh

@@ -6,34 +6,55 @@
 #   1. Installs OVN/OVS packages (if not present)
 #   2. Enables required services (openvswitch-switch, ovn-controller)
 #   3. Creates br-int with secure fail-mode
-#   4. Configures OVS external_ids for OVN chassis identity
-#   5. Applies sysctl tuning for overlay networking
+#   4. Configures WAN bridge for public subnet uplink (auto-detected or manual)
+#   5. Configures OVS external_ids for OVN chassis identity
+#   6. Applies sysctl tuning for overlay networking
 #
 # Usage:
 #   ./scripts/setup-ovn.sh [options]
 #
 # Options:
-#   --management     Also start OVN central services (NB DB, SB DB, ovn-northd)
-#   --ovn-remote     OVN SB DB address (default: tcp:127.0.0.1:6642)
-#   --encap-ip       Geneve tunnel endpoint IP (default: auto-detect)
-#   --chassis-id     OVN chassis identifier (default: hostname)
+#   --management         Also start OVN central services (NB DB, SB DB, ovn-northd)
+#   --wan-bridge=NAME    OVS bridge for WAN traffic (default: auto-detect from default route)
+#   --wan-iface=NAME     Physical NIC to add to the WAN bridge (use with --wan-bridge or --macvlan)
+#   --macvlan            Create macvlan off --wan-iface instead of moving NIC directly.
+#                        SSH-safe for single-NIC hosts where WAN NIC carries SSH.
+#   --dhcp               Obtain gateway IP via DHCP on the WAN bridge interface
+#   --ovn-remote=ADDR    OVN SB DB address (default: tcp:127.0.0.1:6642)
+#   --encap-ip=IP        Geneve tunnel endpoint IP (default: auto-detect)
+#   --chassis-id=ID      OVN chassis identifier (default: chassis-$(hostname -s))
+#
+# WAN Bridge Auto-Detection:
+#   When no --wan-bridge is given, the script checks the default route interface:
+#   - If it's an OVS bridge → use it directly for bridge-mappings
+#   - If it's a Linux bridge → create OVS br-ext + veth pair to link them
+#     (non-destructive, Linux bridge keeps IP/routes, no interruption)
+#   - If it's a physical NIC → stop and print guidance (cannot safely move NIC)
 #
 # Examples:
-#   # Single-node development (management + compute on same host):
+#   # WAN is already on a bridge (tofu-cluster, production):
 #   ./scripts/setup-ovn.sh --management
 #
+#   # Dedicated WAN NIC (not your SSH NIC — you take responsibility):
+#   ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
+#
+#   # Single-NIC host (SSH-safe macvlan):
+#   ./scripts/setup-ovn.sh --management --macvlan --wan-iface=eth0
+#
 #   # Compute node joining an existing cluster:
 #   ./scripts/setup-ovn.sh --ovn-remote=tcp:10.0.0.1:6642 --encap-ip=10.0.0.2
 #
-#   # Multi-node dev cluster (node1 = management):
-#   Node 1: ./scripts/setup-ovn.sh --management --encap-ip=127.0.0.1
-#   Node 2: ./scripts/setup-ovn.sh --ovn-remote=tcp:127.0.0.1:6642 --encap-ip=127.0.0.2
-#   Node 3: ./scripts/setup-ovn.sh --ovn-remote=tcp:127.0.0.1:6642 --encap-ip=127.0.0.3
+#   # No WAN bridge (overlay-only, no public subnet):
+#   ./scripts/setup-ovn.sh --management --encap-ip=10.0.0.1
 
 set -e
 
 # Defaults
 MANAGEMENT=false
+WAN_BRIDGE=""
+WAN_IFACE=""
+MACVLAN_MODE=false
+EXTERNAL_DHCP=false
 OVN_REMOTE="tcp:127.0.0.1:6642"
 ENCAP_IP=""
 CHASSIS_ID=""
@@ -41,12 +62,16 @@ CHASSIS_ID=""
 # Parse arguments
 for arg in "$@"; do
     case "$arg" in
-        --management)     MANAGEMENT=true ;;
-        --ovn-remote=*)   OVN_REMOTE="${arg#*=}" ;;
-        --encap-ip=*)     ENCAP_IP="${arg#*=}" ;;
-        --chassis-id=*)   CHASSIS_ID="${arg#*=}" ;;
+        --management)       MANAGEMENT=true ;;
+        --macvlan)          MACVLAN_MODE=true ;;
+        --dhcp)             EXTERNAL_DHCP=true ;;
+        --wan-bridge=*)     WAN_BRIDGE="${arg#*=}" ;;
+        --wan-iface=*)      WAN_IFACE="${arg#*=}" ;;
+        --ovn-remote=*)     OVN_REMOTE="${arg#*=}" ;;
+        --encap-ip=*)       ENCAP_IP="${arg#*=}" ;;
+        --chassis-id=*)     CHASSIS_ID="${arg#*=}" ;;
         --help|-h)
-            head -30 "$0" | tail -28
+            head -44 "$0" | tail -42
             exit 0
             ;;
         *)
@@ -56,13 +81,121 @@ for arg in "$@"; do
     esac
 done
 
+# --- WAN bridge auto-detection ---
+# Determine the WAN bridge name and how to set it up.
+WAN_BRIDGE_MODE=""  # "existing", "veth", "direct", "macvlan", or ""
+LINUX_BRIDGE=""     # Set when WAN_BRIDGE_MODE="veth" — the Linux bridge behind the veth pair
+
+detect_wan_bridge() {
+    # If --wan-bridge was explicitly given, use it
+    if [ -n "$WAN_BRIDGE" ]; then
+        if [ "$MACVLAN_MODE" = true ] && [ -n "$WAN_IFACE" ]; then
+            WAN_BRIDGE_MODE="macvlan"
+        elif [ -n "$WAN_IFACE" ]; then
+            WAN_BRIDGE_MODE="direct"
+        elif sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
+            WAN_BRIDGE_MODE="existing"
+        else
+            # Bridge doesn't exist yet and no --wan-iface — create empty OVS bridge
+            WAN_BRIDGE_MODE="existing"
+        fi
+        return
+    fi
+
+    # If --macvlan was given without --wan-bridge, we need --wan-iface
+    if [ "$MACVLAN_MODE" = true ]; then
+        if [ -z "$WAN_IFACE" ]; then
+            echo "ERROR: --macvlan requires --wan-iface=<NIC>"
+            exit 1
+        fi
+        WAN_BRIDGE="br-wan"
+        WAN_BRIDGE_MODE="macvlan"
+        return
+    fi
+
+    # Auto-detect: find the default route interface
+    local default_dev
+    default_dev=$(ip -4 route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev") print $(i+1)}' | head -1)
+
+    if [ -z "$default_dev" ]; then
+        echo "  No default route found — no WAN bridge configured"
+        echo "  (VMs will not have external connectivity)"
+        return
+    fi
+
+    # Check if the default route device is a bridge (Linux or OVS)
+    local is_bridge=false
+    if ip -d link show "$default_dev" 2>/dev/null | grep -q "bridge"; then
+        is_bridge=true
+    fi
+    if sudo ovs-vsctl br-exists "$default_dev" 2>/dev/null; then
+        is_bridge=true
+    fi
+
+    if [ "$is_bridge" = true ]; then
+        if sudo ovs-vsctl br-exists "$default_dev" 2>/dev/null; then
+            # Already an OVS bridge — use it directly for bridge-mappings
+            WAN_BRIDGE="$default_dev"
+            WAN_BRIDGE_MODE="existing"
+            echo "  Auto-detected WAN bridge: $WAN_BRIDGE (OVS bridge, default route)"
+        else
+            # Linux bridge — link to OVS via veth pair (non-destructive)
+            LINUX_BRIDGE="$default_dev"
+            WAN_BRIDGE="br-ext"
+            WAN_BRIDGE_MODE="veth"
+            echo "  Auto-detected Linux bridge: $LINUX_BRIDGE (default route)"
+            echo "  Will create OVS bridge br-ext + veth pair to link them"
+        fi
+        return
+    fi
+
+    # Default route is a physical NIC — cannot safely move it to OVS
+    # because it might be carrying SSH.
+    local wan_ip
+    wan_ip=$(ip -4 -o addr show "$default_dev" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+
+    echo ""
+    echo "============================================================"
+    echo "  WAN interface '$default_dev' ($wan_ip) is a physical NIC."
+    echo "  Cannot auto-create a bridge — this may drop your connection."
+    echo ""
+    echo "  Options:"
+    echo ""
+    echo "  1. Create a WAN bridge first (e.g. via netplan), then re-run:"
+    echo "     ./scripts/setup-ovn.sh --management"
+    echo ""
+    echo "  2. Dedicated WAN NIC (NOT your SSH connection):"
+    echo "     ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=$default_dev"
+    echo ""
+    echo "  3. Single-NIC host (SSH-safe macvlan):"
+    echo "     ./scripts/setup-ovn.sh --management --macvlan --wan-iface=$default_dev"
+    echo ""
+    echo "  4. No external networking (overlay-only):"
+    echo "     ./scripts/setup-ovn.sh --management --encap-ip=$wan_ip"
+    echo "============================================================"
+    echo ""
+    exit 1
+}
+
+detect_wan_bridge
+
 # Auto-detect encap IP if not specified
 if [ -z "$ENCAP_IP" ]; then
-    ENCAP_IP=$(ip -4 route get 8.8.8.8 2>/dev/null | awk '/src/{print $7}' | head -1)
-    if [ -z "$ENCAP_IP" ]; then
-        ENCAP_IP="127.0.0.1"
+    # Prefer br-vpc IP if it exists (dedicated VPC data plane)
+    if ip -4 addr show br-vpc >/dev/null 2>&1; then
+        ENCAP_IP=$(ip -4 -o addr show br-vpc 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+        if [ -n "$ENCAP_IP" ]; then
+            echo "Auto-detected encap IP from br-vpc: $ENCAP_IP"
+        fi
+    fi
+    # Fall back to default route source IP
+    if [ -z "$ENCAP_IP" ]; then
+        ENCAP_IP=$(ip -4 route get 8.8.8.8 2>/dev/null | awk '/src/{print $7}' | head -1)
+        if [ -z "$ENCAP_IP" ]; then
+            ENCAP_IP="127.0.0.1"
+        fi
+        echo "Auto-detected encap IP: $ENCAP_IP"
     fi
-    echo "Auto-detected encap IP: $ENCAP_IP"
 fi
 
 # Auto-detect chassis ID if not specified
@@ -72,10 +205,21 @@ if [ -z "$CHASSIS_ID" ]; then
 fi
 
 echo "=== Spinifex OVN Compute Node Setup ==="
-echo "  Management node: $MANAGEMENT"
-echo "  OVN Remote (SB): $OVN_REMOTE"
-echo "  Encap IP:        $ENCAP_IP"
-echo "  Chassis ID:      $CHASSIS_ID"
+echo "  Management node:  $MANAGEMENT"
+if [ -n "$WAN_BRIDGE" ]; then
+    echo "  WAN bridge:       $WAN_BRIDGE ($WAN_BRIDGE_MODE)"
+    if [ -n "$LINUX_BRIDGE" ]; then
+        echo "  Linux bridge:     $LINUX_BRIDGE (linked via veth pair)"
+    fi
+    if [ -n "$WAN_IFACE" ]; then
+        echo "  WAN interface:    $WAN_IFACE"
+    fi
+else
+    echo "  WAN bridge:       none (overlay-only)"
+fi
+echo "  OVN Remote (SB):  $OVN_REMOTE"
+echo "  Encap IP:         $ENCAP_IP"
+echo "  Chassis ID:       $CHASSIS_ID"
 echo ""
 
 # --- Step 1: Install packages ---
@@ -118,6 +262,15 @@ if [ "$MANAGEMENT" = true ]; then
     sudo systemctl start ovn-central
     echo "  ovn-central: started (NB DB + SB DB + ovn-northd)"
 
+    # Wait for OVN NB DB socket to become available
+    for i in $(seq 1 15); do
+        if sudo ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+            break
+        fi
+        echo "  Waiting for OVN NB DB... ($i/15)"
+        sleep 1
+    done
+
     # Allow remote connections to NB and SB databases
     sudo ovn-nbctl set-connection ptcp:6641
     sudo ovn-sbctl set-connection ptcp:6642
@@ -135,15 +288,189 @@ sudo ovs-vsctl set Bridge br-int other-config:disable-in-band=true
 sudo ip link set br-int up
 echo "  br-int: created, fail-mode=secure, up"
 
+# --- Step 3b: Configure WAN bridge for public subnet uplink ---
+if [ -n "$WAN_BRIDGE" ]; then
+    echo ""
+    echo "Step 3b: Configuring WAN bridge ($WAN_BRIDGE) for public subnet uplink..."
+
+    case "$WAN_BRIDGE_MODE" in
+        existing)
+            # Already an OVS bridge (from a previous run or explicit --wan-bridge).
+            if ! sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
+                sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
+                echo "  created OVS bridge: $WAN_BRIDGE"
+            fi
+            sudo ip link set "$WAN_BRIDGE" up
+            echo "  $WAN_BRIDGE: OVS bridge, up"
+            ;;
+
+        veth)
+            # Linux bridge detected (e.g. br-wan from cloud-init/netplan).
+            # OVN bridge-mappings require an OVS bridge. Rather than converting
+            # the Linux bridge (destructive, causes WAN interruption), we create
+            # a separate OVS bridge and link them with a veth pair:
+            #
+            #   br-wan (Linux, keeps IP/routes) ←→ veth pair ←→ br-ext (OVS, for OVN)
+            #
+            # No network interruption. The Linux bridge is untouched.
+
+            # Create OVS bridge
+            if ! sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
+                sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
+                echo "  created OVS bridge: $WAN_BRIDGE"
+            fi
+            sudo ip link set "$WAN_BRIDGE" up
+
+            # Create veth pair (idempotent)
+            if ! ip link show veth-wan-br >/dev/null 2>&1; then
+                sudo ip link add veth-wan-br type veth peer name veth-wan-ovs
+                echo "  created veth pair: veth-wan-br ↔ veth-wan-ovs"
+            else
+                echo "  veth pair already exists: veth-wan-br ↔ veth-wan-ovs"
+            fi
+
+            # Enslave veth-wan-br to the Linux bridge
+            if ! ip link show veth-wan-br 2>/dev/null | grep -q "master $LINUX_BRIDGE"; then
+                sudo ip link set veth-wan-br master "$LINUX_BRIDGE"
+                echo "  veth-wan-br → $LINUX_BRIDGE (Linux bridge)"
+            fi
+            sudo ip link set veth-wan-br up
+
+            # Add veth-wan-ovs to the OVS bridge
+            if ! sudo ovs-vsctl port-to-br veth-wan-ovs >/dev/null 2>&1; then
+                sudo ovs-vsctl --may-exist add-port "$WAN_BRIDGE" veth-wan-ovs
+                echo "  veth-wan-ovs → $WAN_BRIDGE (OVS bridge)"
+            fi
+            sudo ip link set veth-wan-ovs up
+
+            echo "  $LINUX_BRIDGE (Linux) ↔ veth pair ↔ $WAN_BRIDGE (OVS)"
+            echo "  $LINUX_BRIDGE keeps its IP and routes — no interruption"
+            ;;
+
+        direct)
+            # Add WAN NIC directly to OVS bridge. The NIC becomes an OVS slave —
+            # its IP (if any) is no longer reachable from the host. The user has
+            # confirmed this NIC is NOT their SSH connection.
+            if ! ip link show "$WAN_IFACE" >/dev/null 2>&1; then
+                echo "  ERROR: interface $WAN_IFACE does not exist"
+                echo "  Available interfaces:"
+                ip -o link show | awk -F': ' '{print "    " $2}'
+                exit 1
+            fi
+
+            sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
+            sudo ip link set "$WAN_BRIDGE" up
+
+            if sudo ovs-vsctl port-to-br "$WAN_IFACE" >/dev/null 2>&1; then
+                echo "  $WAN_IFACE already on $(sudo ovs-vsctl port-to-br "$WAN_IFACE")"
+            else
+                sudo ovs-vsctl --may-exist add-port "$WAN_BRIDGE" "$WAN_IFACE"
+                echo "  added $WAN_IFACE directly to $WAN_BRIDGE"
+            fi
+            sudo ip link set "$WAN_IFACE" up
+            echo "  $WAN_BRIDGE: direct bridge on $WAN_IFACE"
+            echo "  NOTE: $WAN_IFACE is now an OVS port — no host IP on this NIC"
+            ;;
+
+        macvlan)
+            # Create a macvlan sub-interface in bridge mode off the WAN NIC.
+            # The host keeps its IP on the parent NIC — SSH-safe. OVN localnet
+            # traffic flows through the macvlan to the physical wire.
+            if ! ip link show "$WAN_IFACE" >/dev/null 2>&1; then
+                echo "  ERROR: interface $WAN_IFACE does not exist"
+                echo "  Available interfaces:"
+                ip -o link show | awk -F': ' '{print "    " $2}'
+                exit 1
+            fi
+
+            MACVLAN_NAME="spx-ext-${WAN_IFACE}"
+
+            sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
+            sudo ip link set "$WAN_BRIDGE" up
+
+            if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then
+                echo "  macvlan $MACVLAN_NAME already exists"
+            else
+                sudo ip link add "$MACVLAN_NAME" link "$WAN_IFACE" type macvlan mode bridge
+                echo "  created macvlan: $MACVLAN_NAME (bridge mode) on $WAN_IFACE"
+            fi
+
+            sudo ip link set "$MACVLAN_NAME" up
+            sudo ovs-vsctl --may-exist add-port "$WAN_BRIDGE" "$MACVLAN_NAME"
+            echo "  $WAN_BRIDGE: macvlan port $MACVLAN_NAME on $WAN_IFACE"
+            echo "  NOTE: host keeps its IP on $WAN_IFACE (SSH-safe)"
+            echo "  QUIRK: host cannot reach VMs at their public IPs (macvlan isolation)"
+            ;;
+    esac
+
+    # --- DHCP: obtain gateway IP for OVN SNAT ---
+    if [ "$EXTERNAL_DHCP" = true ]; then
+        echo ""
+        echo "Step 3c: Obtaining external gateway IP via DHCP..."
+
+        # For macvlan mode, DHCP on the macvlan interface (it has L2 access to WAN).
+        # For direct/existing bridge, DHCP on the bridge itself.
+        if [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then
+            DHCP_IFACE="spx-ext-${WAN_IFACE}"
+        else
+            DHCP_IFACE="$WAN_BRIDGE"
+        fi
+
+        # Run DHCP client to get a lease
+        if command -v dhcpcd >/dev/null 2>&1; then
+            sudo dhcpcd --waitip=4 --timeout 15 "$DHCP_IFACE" 2>/dev/null || true
+        elif command -v dhclient >/dev/null 2>&1; then
+            sudo dhclient -1 -timeout 15 "$DHCP_IFACE" 2>/dev/null || true
+        else
+            echo "  WARNING: no DHCP client found (dhcpcd or dhclient)"
+            echo "  Install dhcpcd-base or isc-dhcp-client, or set gateway_ip manually"
+        fi
+
+        # Read the obtained IP
+        DHCP_IP=$(ip -4 addr show dev "$DHCP_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1 | cut -d/ -f1)
+        if [ -n "$DHCP_IP" ]; then
+            echo "  DHCP obtained: $DHCP_IP on $DHCP_IFACE"
+
+            # Write the gateway IP to the spinifex config so vpcd can use it
+            CONFIG_DIR="${CONFIG_DIR:-$HOME/spinifex/config}"
+            CONFIG_FILE="$CONFIG_DIR/spinifex.toml"
+            if [ -f "$CONFIG_FILE" ]; then
+                if grep -q "gateway_ip" "$CONFIG_FILE"; then
+                    sed -i "s/gateway_ip.*/gateway_ip = \"$DHCP_IP\"/" "$CONFIG_FILE"
+                else
+                    sed -i "/^gateway *=.*/a gateway_ip  = \"$DHCP_IP\"" "$CONFIG_FILE"
+                fi
+                echo "  Updated $CONFIG_FILE with gateway_ip = $DHCP_IP"
+            else
+                echo "  WARNING: $CONFIG_FILE not found — set gateway_ip manually"
+            fi
+        else
+            echo "  WARNING: DHCP failed to obtain IP on $DHCP_IFACE"
+            echo "  VMs will not have external connectivity until gateway_ip is configured"
+        fi
+    fi
+fi
+
 # --- Step 4: Configure OVN external_ids ---
 echo ""
 echo "Step 4: Setting OVS external_ids for OVN..."
 
-sudo ovs-vsctl set Open_vSwitch . \
-    external_ids:system-id="$CHASSIS_ID" \
-    external_ids:ovn-remote="$OVN_REMOTE" \
-    external_ids:ovn-encap-ip="$ENCAP_IP" \
-    external_ids:ovn-encap-type="geneve"
+if [ -n "$WAN_BRIDGE" ]; then
+    BRIDGE_MAPPINGS="external:${WAN_BRIDGE}"
+    sudo ovs-vsctl set Open_vSwitch . \
+        external_ids:system-id="$CHASSIS_ID" \
+        external_ids:ovn-remote="$OVN_REMOTE" \
+        external_ids:ovn-encap-ip="$ENCAP_IP" \
+        external_ids:ovn-encap-type="geneve" \
+        external_ids:ovn-bridge-mappings="$BRIDGE_MAPPINGS"
+    echo "  ovn-bridge-mappings: $BRIDGE_MAPPINGS"
+else
+    sudo ovs-vsctl set Open_vSwitch . \
+        external_ids:system-id="$CHASSIS_ID" \
+        external_ids:ovn-remote="$OVN_REMOTE" \
+        external_ids:ovn-encap-ip="$ENCAP_IP" \
+        external_ids:ovn-encap-type="geneve"
+fi
 
 echo "  system-id:      $CHASSIS_ID"
 echo "  ovn-remote:     $OVN_REMOTE"
@@ -287,15 +614,16 @@ sudo tee "$LOGROTATE_FILE" >/dev/null <<'LOGROTATE'
 LOGROTATE
 echo "  logrotate config: $LOGROTATE_FILE (daily, 100M max, 3 rotations, explicit filenames)"
 
-# --- Step 10: Disable auto-start on boot ---
-# start-dev.sh / stop-dev.sh manage the OVN lifecycle. Without spinifex services
-# running, ovn-controller spins in a tight reconnect loop burning CPU.
+# --- Step 10: Enable auto-start on boot ---
+# OVN services should start with the system in production. ovn-controller uses
+# exponential backoff if the SB DB isn't ready yet, so CPU impact is negligible.
+# In dev, stop-dev.sh explicitly stops OVN when not needed.
 echo ""
-echo "Step 10: Disabling OVN auto-start on boot (start-dev.sh will manage lifecycle)..."
-sudo systemctl disable openvswitch-switch 2>/dev/null || true
-sudo systemctl disable ovn-controller 2>/dev/null || true
-echo "  openvswitch-switch: disabled on boot"
-echo "  ovn-controller: disabled on boot"
+echo "Step 10: Enabling OVN auto-start on boot..."
+sudo systemctl enable openvswitch-switch 2>/dev/null || true
+sudo systemctl enable ovn-controller 2>/dev/null || true
+echo "  openvswitch-switch: enabled on boot"
+echo "  ovn-controller: enabled on boot"
 
 # --- Step 11: Health check ---
 echo ""
@@ -311,6 +639,40 @@ else
     OK=false
 fi
 
+# Check WAN bridge (only if configured)
+if [ -n "$WAN_BRIDGE" ]; then
+    if sudo ovs-vsctl br-exists "$WAN_BRIDGE"; then
+        echo "  $WAN_BRIDGE:$(printf '%*s' $((15 - ${#WAN_BRIDGE})) '') OK"
+        if [ "$WAN_BRIDGE_MODE" = "veth" ]; then
+            if ip link show veth-wan-br >/dev/null 2>&1 && ip link show veth-wan-ovs >/dev/null 2>&1; then
+                echo "  veth pair:       OK (veth-wan-br ↔ veth-wan-ovs)"
+                echo "  linux bridge:    $LINUX_BRIDGE (untouched)"
+            else
+                echo "  veth pair:       FAILED (veth-wan-br/veth-wan-ovs not found)"
+                OK=false
+            fi
+        elif [ "$WAN_BRIDGE_MODE" = "direct" ]; then
+            if sudo ovs-vsctl port-to-br "$WAN_IFACE" >/dev/null 2>&1; then
+                echo "  direct bridge:   OK ($WAN_IFACE on $WAN_BRIDGE)"
+            else
+                echo "  direct bridge:   FAILED ($WAN_IFACE not on $WAN_BRIDGE)"
+                OK=false
+            fi
+        elif [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then
+            MACVLAN_NAME="spx-ext-${WAN_IFACE}"
+            if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then
+                echo "  macvlan:         OK ($MACVLAN_NAME)"
+            else
+                echo "  macvlan:         FAILED ($MACVLAN_NAME not found)"
+                OK=false
+            fi
+        fi
+    else
+        echo "  $WAN_BRIDGE:$(printf '%*s' $((15 - ${#WAN_BRIDGE})) '') FAILED"
+        OK=false
+    fi
+fi
+
 # Check ovn-controller
 if sudo ovs-appctl -t ovn-controller version >/dev/null 2>&1 || systemctl is-active --quiet ovn-controller 2>/dev/null; then
     echo "  ovn-controller:  OK"

scripts/setup.sh

@@ -127,7 +127,7 @@ install_apt_deps() {
         $QEMU_PACKAGES qemu-utils qemu-kvm \
         libvirt-daemon-system libvirt-clients \
         jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
-        ovn-central ovn-host openvswitch-switch \
+        ovn-central ovn-host openvswitch-switch dhcpcd-base \
         > /dev/null
 
     info "System dependencies installed"
@@ -196,7 +196,8 @@ download_spinifex() {
         fi
         info "Checksum verified"
     else
-        warn "Checksum not available, skipping verification"
+        rm -rf "$SPINIFEX_TMPDIR"
+        fatal "Checksum not available at $CHECKSUM_URL. Cannot verify download integrity."
     fi
 
     # Extract
@@ -233,6 +234,7 @@ install_files() {
         $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
         info "  /usr/local/share/spinifex/setup-ovn.sh"
     fi
+
 }
 
 # --- Create directories ---
@@ -272,6 +274,15 @@ SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
 EOF
     $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
     info "Generated /etc/spinifex/systemd.env"
+
+    # Service helper scripts
+    if [ -d "$EXTRACT_DIR/scripts" ]; then
+        for script in "$EXTRACT_DIR"/scripts/*.sh; do
+            $SUDO install -o "$SPINIFEX_USER" -g "$SPINIFEX_GROUP" -m 0755 \
+                "$script" "/var/lib/spinifex/$(basename "$script")"
+            info "  /var/lib/spinifex/$(basename "$script")"
+        done
+    fi
 }
 
 # --- Install systemd units ---
@@ -341,19 +352,22 @@ print_summary() {
     echo "  Next steps:"
     echo ""
     echo "  1. Setup OVN networking:"
-    echo "     sudo /usr/local/share/spinifex/setup-ovn.sh --management"
+    echo "     If your WAN interface is already a bridge (e.g. br-wan):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management"
     echo ""
-    echo "  2. Initialize a region:"
-    echo "     sudo spx admin init --region <region> --az <az> --node node1 --nodes 1"
+    echo "     If your WAN is a physical NIC:"
+    echo "       # Dedicated WAN NIC (not your SSH connection):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
+    echo "       # Single-NIC host (SSH-safe macvlan):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --macvlan --wan-iface=enp0s3"
     echo ""
-    echo "  3. Trust the CA certificate:"
-    echo "     sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt"
-    echo "     sudo update-ca-certificates"
+    echo "  2. Initialize:"
+    echo "     sudo spx admin init --node node1 --nodes 1"
     echo ""
-    echo "  4. Start services:"
+    echo "  3. Start services:"
     echo "     sudo systemctl start spinifex.target"
     echo ""
-    echo "  5. Verify:"
+    echo "  4. Verify:"
     echo "     export AWS_PROFILE=spinifex"
     echo "     aws ec2 describe-instance-types"
     echo ""

scripts/start-multi-nats.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-SPINIFEX_CONFIG_PATH=~/node1/config/nats/nats.conf ./bin/spx service nats start

scripts/stop-dev.sh

@@ -150,13 +150,43 @@ has_service "predastore" && stop_service "predastore" "$PID_DIR"
 has_service "nats" && stop_service "nats" "$PID_DIR"
 
 # Stop OVN networking (prevents idle CPU burn when spinifex isn't running)
+# CRITICAL: Do NOT stop openvswitch-switch if OVS carries WAN traffic.
+# This can happen when an OVS bridge has a WAN IP (direct bridge mode) or
+# when a veth pair links an OVS bridge to a Linux WAN bridge (veth mode).
+# Stopping OVS would destroy the bridge and break WAN connectivity.
+# OVS is infrastructure — only stop ovn-controller and ovn-central.
 if pidof systemd >/dev/null 2>&1; then
     echo "🛑 Stopping OVN networking..."
     sudo systemctl stop ovn-controller 2>/dev/null && echo "✅ ovn-controller stopped" || true
     if systemctl is-active --quiet ovn-central 2>/dev/null; then
         sudo systemctl stop ovn-central 2>/dev/null && echo "✅ ovn-central stopped" || true
     fi
-    sudo systemctl stop openvswitch-switch 2>/dev/null && echo "✅ openvswitch-switch stopped" || true
+
+    # Only stop OVS if tearing it down won't kill WAN connectivity.
+    # Two cases where OVS is load-bearing for WAN:
+    #   1. An OVS bridge has a WAN IP directly (direct bridge mode)
+    #   2. A veth pair links a Linux bridge to an OVS bridge (veth mode) —
+    #      stopping OVS destroys the OVS bridge end, breaking the link
+    WAN_BRIDGE_HAS_IP=false
+    VETH_LINKS_WAN=false
+    for br in $(sudo ovs-vsctl list-br 2>/dev/null); do
+        if [ "$br" = "br-int" ]; then continue; fi
+        if ip -4 addr show "$br" 2>/dev/null | grep -q "inet "; then
+            WAN_BRIDGE_HAS_IP=true
+            break
+        fi
+    done
+    # Check if a veth pair links an OVS bridge to a Linux bridge with a WAN IP
+    if ip link show veth-wan-br >/dev/null 2>&1; then
+        VETH_LINKS_WAN=true
+    fi
+    if [ "$WAN_BRIDGE_HAS_IP" = true ]; then
+        echo "⚠️  Skipping openvswitch-switch stop ($br has WAN IP — stopping would kill connectivity)"
+    elif [ "$VETH_LINKS_WAN" = true ]; then
+        echo "⚠️  Skipping openvswitch-switch stop (veth pair links OVS to WAN bridge)"
+    else
+        sudo systemctl stop openvswitch-switch 2>/dev/null && echo "✅ openvswitch-switch stopped" || true
+    fi
 fi
 
 echo ""

spinifex/admin/admin.go

@@ -64,8 +64,31 @@ type ConfigSettings struct {
 	OVNNBAddr string
 	OVNSBAddr string
 
+	// External networking for public subnets
+	ExternalMode   string   // "pool" or "" (disabled)
+	ExternalIface  string   // WAN NIC name (e.g., "eth0", "eth1")
+	WanBridge      string   // OVS bridge for WAN traffic (default "br-wan")
+	ExternalDHCP   bool     // Obtain gateway IP via DHCP on macvlan/bridge
+	PoolName       string   // External pool name (e.g., "wan")
+	PoolSource     string   // IP source: "static" (default) or "dhcp" (from router DHCP)
+	PoolStart      string   // First IP in external pool range (static source only)
+	PoolEnd        string   // Last IP in external pool range (static source only)
+	PoolGateway    string   // WAN gateway IP
+	PoolGatewayIP  string   // Explicit SNAT IP (for nat mode without DHCP)
+	PoolPrefixLen  int      // Subnet prefix length (default 24)
+	PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host)
+
 	// Other nodes in the cluster (for config source of truth)
 	RemoteNodes []RemoteNode
+
+	// Bootstrap: pre-generated default VPC IDs for vpcd reconciliation.
+	// Written by admin init so [bootstrap] exists before services start.
+	BootstrapAccountId  string
+	BootstrapVpcId      string
+	BootstrapSubnetId   string
+	BootstrapIgwId      string
+	BootstrapCidr       string
+	BootstrapSubnetCidr string
 }
 
 // PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
@@ -147,11 +170,13 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (
 		fmt.Printf("   Certificate: %s\n", serverCertPath)
 		fmt.Printf("   Key: %s\n", serverKeyPath)
 
-		// Print instructions for adding CA to system trust store
-		fmt.Println("\n📋 To trust the Spinifex CA system-wide (recommended):")
-		fmt.Printf("   sudo cp %s /usr/local/share/ca-certificates/spinifex-ca.crt\n", caCertPath)
-		fmt.Println("   sudo update-ca-certificates")
-		fmt.Println("\n   This allows AWS CLI and other tools to trust Spinifex services automatically.")
+		// Print manual instructions only when not root (root gets auto-install)
+		if os.Getuid() != 0 {
+			fmt.Println("\n📋 To trust the Spinifex CA system-wide (recommended):")
+			fmt.Printf("   sudo cp %s /usr/local/share/ca-certificates/spinifex-ca.crt\n", caCertPath)
+			fmt.Println("   sudo update-ca-certificates")
+			fmt.Println("\n   This allows AWS CLI and other tools to trust Spinifex services automatically.")
+		}
 	} else {
 		fmt.Println("\n✅ CA and SSL certificates already exist")
 	}
@@ -402,53 +427,109 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
 	return nil
 }
 
+// DiscoverLocalIPs enumerates all non-loopback network interface addresses on
+// this machine and returns them as strings. Link-local addresses are excluded.
+func DiscoverLocalIPs() []string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		slog.Warn("failed to enumerate network interfaces", "error", err)
+		return nil
+	}
+
+	seen := make(map[string]struct{})
+	var ips []string
+
+	for _, iface := range ifaces {
+		// Skip loopback and down interfaces.
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			slog.Debug("failed to get addresses for interface", "iface", iface.Name, "error", err)
+			continue
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+				continue
+			}
+			s := ip.String()
+			if _, ok := seen[s]; !ok {
+				seen[s] = struct{}{}
+				ips = append(ips, s)
+			}
+		}
+	}
+	return ips
+}
+
+// DiscoverHostname returns the system hostname suitable for use as a DNS SAN.
+// Returns an empty string if the hostname cannot be determined or is "localhost".
+func DiscoverHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		slog.Warn("failed to determine system hostname", "error", err)
+		return ""
+	}
+	if hostname == "" || hostname == "localhost" {
+		return ""
+	}
+	return hostname
+}
+
 // GenerateSignedCert generates a server certificate signed by the CA.
 // extraIPs are additional IP addresses to include in the certificate's SANs.
+// All non-loopback interface IPs on the local machine are automatically included.
 func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
-	// Load CA certificate
+	return GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, nil)
+}
+
+// GenerateSignedCertWithDNS generates a server certificate signed by the CA.
+// All non-loopback interface IPs and the machine hostname are automatically
+// included. extraIPs and extraDNS allow adding additional SANs.
+func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, extraIPs, extraDNS []string) error {
 	caCertPEM, err := os.ReadFile(caCertPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA cert: %w", err)
 	}
-
 	caCertBlock, _ := pem.Decode(caCertPEM)
 	if caCertBlock == nil {
 		return fmt.Errorf("failed to decode CA cert PEM")
 	}
-
 	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA cert: %w", err)
 	}
 
-	// Load CA private key
 	caKeyPEM, err := os.ReadFile(caKeyPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA key: %w", err)
 	}
-
 	caKeyBlock, _ := pem.Decode(caKeyPEM)
 	if caKeyBlock == nil {
 		return fmt.Errorf("failed to decode CA key PEM")
 	}
-
 	caPrivateKey, err := x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA private key: %w", err)
 	}
-
 	caRSAKey, ok := caPrivateKey.(*rsa.PrivateKey)
 	if !ok {
 		return fmt.Errorf("CA key is not RSA")
 	}
 
-	// Generate server private key
 	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
 		return fmt.Errorf("failed to generate server private key: %w", err)
 	}
 
-	// Create server certificate template
 	notBefore := time.Now()
 	notAfter := notBefore.Add(365 * 24 * time.Hour) // 1 year for server certs
 
@@ -457,20 +538,51 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}
 
-	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	// Build IP list: localhost IPs + auto-discovered interface IPs + explicit extras.
+	seen := map[string]struct{}{"127.0.0.1": {}, "::1": {}}
 	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	addIP := func(s string) {
+		if s == "" || s == "0.0.0.0" {
+			return
+		}
+		if _, ok := seen[s]; ok {
+			return
+		}
+		if parsed := net.ParseIP(s); parsed != nil {
+			seen[s] = struct{}{}
+			ipAddresses = append(ipAddresses, parsed)
+		}
+	}
+	for _, ip := range DiscoverLocalIPs() {
+		addIP(ip)
+	}
 	for _, ip := range extraIPs {
-		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
-			if parsed := net.ParseIP(ip); parsed != nil {
-				ipAddresses = append(ipAddresses, parsed)
-			}
+		addIP(ip)
+	}
+
+	// Build DNS names: always include localhost, plus hostname if available.
+	dnsSeen := map[string]struct{}{"localhost": {}}
+	dnsNames := []string{"localhost"}
+	if hostname := DiscoverHostname(); hostname != "" {
+		if _, ok := dnsSeen[hostname]; !ok {
+			dnsSeen[hostname] = struct{}{}
+			dnsNames = append(dnsNames, hostname)
+		}
+	}
+	for _, dns := range extraDNS {
+		if dns == "" {
+			continue
+		}
+		if _, ok := dnsSeen[dns]; !ok {
+			dnsSeen[dns] = struct{}{}
+			dnsNames = append(dnsNames, dns)
 		}
 	}
 
 	serverTemplate := x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
-			CommonName:   "localhost",
+			CommonName:   "Spinifex Server",
 			Organization: []string{"Spinifex Platform"},
 		},
 		NotBefore:             notBefore,
@@ -478,39 +590,33 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
-		DNSNames:              []string{"localhost"},
+		DNSNames:              dnsNames,
 		IPAddresses:           ipAddresses,
 	}
 
-	// Sign the server certificate with the CA
 	serverDerBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, caCert, &serverPrivateKey.PublicKey, caRSAKey)
 	if err != nil {
 		return fmt.Errorf("failed to create server certificate: %w", err)
 	}
 
-	// Write server certificate to file
 	certOut, err := os.Create(certPath)
 	if err != nil {
 		return fmt.Errorf("failed to create cert file: %w", err)
 	}
 	defer certOut.Close()
-
 	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverDerBytes}); err != nil {
 		return fmt.Errorf("failed to write cert: %w", err)
 	}
 
-	// Write server private key to file
 	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to create key file: %w", err)
 	}
 	defer keyOut.Close()
-
 	privBytes, err := x509.MarshalPKCS8PrivateKey(serverPrivateKey)
 	if err != nil {
 		return fmt.Errorf("failed to marshal private key: %w", err)
 	}
-
 	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
 		return fmt.Errorf("failed to write key: %w", err)
 	}

spinifex/admin/admin_test.go

@@ -335,7 +335,7 @@ func TestGenerateSignedCert(t *testing.T) {
 		cert, err := x509.ParseCertificate(block.Bytes)
 		require.NoError(t, err)
 		assert.False(t, cert.IsCA)
-		assert.Equal(t, "localhost", cert.Subject.CommonName)
+		assert.Equal(t, "Spinifex Server", cert.Subject.CommonName)
 		assert.Contains(t, cert.DNSNames, "localhost")
 
 		hasLoopback := false
@@ -346,6 +346,9 @@ func TestGenerateSignedCert(t *testing.T) {
 		}
 		assert.True(t, hasLoopback)
 
+		// Auto-discovered IPs should be present (at least loopback + any interface IPs).
+		assert.GreaterOrEqual(t, len(cert.IPAddresses), 2)
+
 		// Verify against CA
 		caCertPEM, _ := os.ReadFile(caCertPath)
 		caBlock, _ := pem.Decode(caCertPEM)
@@ -392,7 +395,19 @@ func TestGenerateSignedCert(t *testing.T) {
 		block, _ := pem.Decode(certPEM)
 		cert, _ := x509.ParseCertificate(block.Bytes)
 
-		assert.Len(t, cert.IPAddresses, 2)
+		// Should have loopback (127.0.0.1, ::1) + auto-discovered interface IPs.
+		// Duplicates (127.0.0.1, ::1) and specials (0.0.0.0, "") must not add extras.
+		// Count unique IPs from a clean run without extras.
+		baseDir := t.TempDir()
+		baseCert := filepath.Join(baseDir, "server.pem")
+		baseKey := filepath.Join(baseDir, "server.key")
+		require.NoError(t, GenerateSignedCert(baseCert, baseKey, caCertPath, caKeyPath))
+		basePEM, _ := os.ReadFile(baseCert)
+		baseBlock, _ := pem.Decode(basePEM)
+		baseParsed, _ := x509.ParseCertificate(baseBlock.Bytes)
+
+		assert.Equal(t, len(baseParsed.IPAddresses), len(cert.IPAddresses),
+			"passing duplicate/special IPs should not add extra entries")
 	})
 
 	t.Run("InvalidCACert", func(t *testing.T) {
@@ -406,6 +421,178 @@ func TestGenerateSignedCert(t *testing.T) {
 	})
 }
 
+func TestDiscoverLocalIPs(t *testing.T) {
+	t.Parallel()
+	ips := DiscoverLocalIPs()
+	// Should find at least one non-loopback IP on any machine with a network interface.
+	assert.NotEmpty(t, ips, "expected at least one non-loopback IP")
+
+	for _, ip := range ips {
+		parsed := net.ParseIP(ip)
+		require.NotNil(t, parsed, "returned IP should be parseable: %s", ip)
+		assert.False(t, parsed.IsLoopback(), "should not include loopback: %s", ip)
+		assert.False(t, parsed.IsLinkLocalUnicast(), "should not include link-local: %s", ip)
+	}
+
+	// No duplicates.
+	seen := make(map[string]struct{})
+	for _, ip := range ips {
+		_, dup := seen[ip]
+		assert.False(t, dup, "duplicate IP: %s", ip)
+		seen[ip] = struct{}{}
+	}
+}
+
+func TestDiscoverHostname(t *testing.T) {
+	t.Parallel()
+	hostname := DiscoverHostname()
+	// We can't assert a specific value, but if non-empty it shouldn't be "localhost".
+	if hostname != "" {
+		assert.NotEqual(t, "localhost", hostname)
+	}
+}
+
+func TestGenerateSignedCertWithDNS_DedupDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Pass "localhost" as extra DNS — should not duplicate the built-in entry.
+	// Also pass empty strings and duplicates — should be ignored.
+	extraDNS := []string{"localhost", "", "example.local", "example.local", ""}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	// Count occurrences of "localhost" — must be exactly 1.
+	localhostCount := 0
+	exampleCount := 0
+	for _, dns := range cert.DNSNames {
+		if dns == "localhost" {
+			localhostCount++
+		}
+		if dns == "example.local" {
+			exampleCount++
+		}
+	}
+	assert.Equal(t, 1, localhostCount, "localhost should appear exactly once")
+	assert.Equal(t, 1, exampleCount, "example.local should appear exactly once")
+	assert.Contains(t, cert.DNSNames, "example.local")
+}
+
+func TestGenerateSignedCertWithDNS_NilSlices(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Both slices nil — should still produce a valid cert with defaults.
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, nil))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.GreaterOrEqual(t, len(cert.IPAddresses), 2, "should have at least loopback IPs")
+}
+
+func TestGenerateSignedCertWithDNS_InvalidCA(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// Missing CA cert file.
+	err := GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "missing-ca.pem"), filepath.Join(dir, "missing-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to read CA cert")
+
+	// Corrupt CA cert.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.pem"), []byte("not-pem"), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.key"), []byte("not-pem"), 0600))
+	err = GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "bad-ca.pem"), filepath.Join(dir, "bad-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+}
+
+func TestGenerateSignedCertWithDNS_ExtraDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	extraIPs := []string{"10.0.0.42"}
+	extraDNS := []string{"spinifex.local", "node1.spinifex.local"}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.Contains(t, cert.DNSNames, "spinifex.local")
+	assert.Contains(t, cert.DNSNames, "node1.spinifex.local")
+
+	hasExtraIP := false
+	for _, ip := range cert.IPAddresses {
+		if ip.Equal(net.ParseIP("10.0.0.42")) {
+			hasExtraIP = true
+		}
+	}
+	assert.True(t, hasExtraIP, "cert should contain extra IP 10.0.0.42")
+
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+func TestGenerateSignedCert_IncludesHostname(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+	require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	hostname := DiscoverHostname()
+	if hostname != "" {
+		assert.Contains(t, cert.DNSNames, hostname,
+			"cert DNS SANs should include the machine hostname")
+	}
+}
+
 func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
 	t.Parallel()
 	dir := t.TempDir()

spinifex/admin/network_detect.go

@@ -0,0 +1,205 @@
+package admin
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/utils"
+)
+
+// DetectedInterface represents a network interface discovered on the host.
+type DetectedInterface struct {
+	Name      string // e.g., "enp0s3"
+	IP        string // e.g., "192.168.1.21"
+	Subnet    string // e.g., "192.168.0.0/23"
+	PrefixLen int    // e.g., 23
+	Gateway   string // non-empty only for WAN (default route)
+	Role      string // "wan", "lan"
+}
+
+// DetectedNetwork holds the auto-detected network topology.
+type DetectedNetwork struct {
+	Interfaces []DetectedInterface
+	WAN        *DetectedInterface // The interface with the default route
+	LANCount   int                // Number of non-WAN interfaces
+}
+
+// DetectNetwork auto-detects the host's network topology from routing table.
+// It identifies the WAN interface (default route) and LAN interfaces.
+func DetectNetwork() (*DetectedNetwork, error) {
+	// Get default route: "default via 192.168.1.1 dev enp0s3 ..."
+	out, err := exec.Command("ip", "-4", "route", "show", "default").Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get default route: %w", err)
+	}
+	defaultRoute := strings.TrimSpace(string(out))
+	if defaultRoute == "" {
+		return nil, fmt.Errorf("no default route found — host has no internet connectivity")
+	}
+
+	// Parse: "default via 192.168.1.1 dev enp0s3 ..."
+	var wanGateway, wanIface string
+	fields := strings.Fields(defaultRoute)
+	for i, f := range fields {
+		if f == "via" && i+1 < len(fields) {
+			wanGateway = fields[i+1]
+		}
+		if f == "dev" && i+1 < len(fields) {
+			wanIface = fields[i+1]
+		}
+	}
+	if wanIface == "" || wanGateway == "" {
+		return nil, fmt.Errorf("could not parse default route: %s", defaultRoute)
+	}
+
+	// Get all routes to find interfaces and their subnets
+	allRoutes, err := exec.Command("ip", "-4", "route", "show", "proto", "kernel", "scope", "link").Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get routes: %w", err)
+	}
+
+	// Parse routes: "192.168.0.0/23 dev enp0s3 ... src 192.168.1.21"
+	seen := make(map[string]bool)
+	var interfaces []DetectedInterface
+
+	for line := range strings.SplitSeq(strings.TrimSpace(string(allRoutes)), "\n") {
+		if line == "" {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			continue
+		}
+
+		subnet := fields[0]
+		var ifaceName, srcIP string
+		for i, f := range fields {
+			if f == "dev" && i+1 < len(fields) {
+				ifaceName = fields[i+1]
+			}
+			if f == "src" && i+1 < len(fields) {
+				srcIP = fields[i+1]
+			}
+		}
+		if ifaceName == "" || srcIP == "" {
+			continue
+		}
+
+		// Skip virtual/docker/ovs interfaces (but allow the default route device)
+		if isVirtualInterface(ifaceName, wanIface) {
+			continue
+		}
+
+		// Skip duplicates (same interface may have multiple routes)
+		if seen[ifaceName] {
+			continue
+		}
+		seen[ifaceName] = true
+
+		prefixLen := 24 // default
+		if _, ipNet, err := net.ParseCIDR(subnet); err == nil {
+			ones, _ := ipNet.Mask.Size()
+			prefixLen = ones
+		}
+
+		iface := DetectedInterface{
+			Name:      ifaceName,
+			IP:        srcIP,
+			Subnet:    subnet,
+			PrefixLen: prefixLen,
+		}
+
+		if ifaceName == wanIface {
+			iface.Gateway = wanGateway
+			iface.Role = "wan"
+		} else {
+			iface.Role = "lan"
+		}
+
+		interfaces = append(interfaces, iface)
+	}
+
+	if len(interfaces) == 0 {
+		return nil, fmt.Errorf("no usable network interfaces found")
+	}
+
+	result := &DetectedNetwork{
+		Interfaces: interfaces,
+	}
+
+	for i := range interfaces {
+		if interfaces[i].Role == "wan" {
+			result.WAN = &interfaces[i]
+		} else {
+			result.LANCount++
+		}
+	}
+
+	return result, nil
+}
+
+// SuggestPoolRange suggests a default external IP pool range based on the
+// WAN subnet. Places the pool at the high end of the subnet to avoid
+// collisions with typical DHCP ranges (which start low).
+func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
+	_, ipNet, err := net.ParseCIDR(wan.Subnet)
+	if err != nil {
+		return "", ""
+	}
+
+	// Calculate broadcast address
+	ip := ipNet.IP.To4()
+	mask := ipNet.Mask
+	broadcast := make(net.IP, 4)
+	for i := range ip {
+		broadcast[i] = ip[i] | ^mask[i]
+	}
+
+	// Pool: broadcast - 100 to broadcast - 5
+	// e.g., for 192.168.0.0/23 (broadcast 192.168.1.255):
+	//   start = 192.168.1.155, end = 192.168.1.250
+	endIP := make(net.IP, 4)
+	copy(endIP, broadcast)
+	endIP[3] -= 5 // Leave some room at top
+
+	startIP := make(net.IP, 4)
+	copy(startIP, endIP)
+	// Go back 100 IPs
+	carry := 100
+	for i := 3; i >= 0 && carry > 0; i-- {
+		val := int(startIP[i]) - carry
+		if val < 0 {
+			startIP[i] = utils.SafeIntToUint8(256 + val)
+			carry = 1
+		} else {
+			startIP[i] = utils.SafeIntToUint8(val)
+			carry = 0
+		}
+	}
+
+	return startIP.String(), endIP.String()
+}
+
+// isVirtualInterface returns true for interfaces that shouldn't be
+// considered as physical NICs. The defaultRouteDev parameter is the
+// interface carrying the default route — it's always considered physical
+// even if its name matches a virtual prefix (e.g., br-wan).
+func isVirtualInterface(name string, defaultRouteDev string) bool {
+	// The default route interface is always considered physical — it's the
+	// WAN uplink, even if it's a bridge (br-wan, br-external, etc.).
+	if defaultRouteDev != "" && name == defaultRouteDev {
+		return false
+	}
+	prefixes := []string{
+		"docker", "br-", "veth", "ovs", "virbr",
+		"tun", "tap", "tailscale", "wg", "lo",
+	}
+	for _, p := range prefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}

spinifex/admin/telemetry.go

@@ -0,0 +1,114 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+)
+
+const defaultTelemetryURL = "https://install.mulgadc.com/install"
+
+// TelemetryPayload is the JSON body sent to the telemetry endpoint.
+type TelemetryPayload struct {
+	MachineID    string `json:"machine_id"`
+	Event        string `json:"event"` // "init" or "join"
+	Region       string `json:"region"`
+	AZ           string `json:"az"`
+	Node         string `json:"node"`
+	Nodes        int    `json:"nodes"`
+	BindIP       string `json:"bind_ip"`
+	Version      string `json:"version"`
+	Arch         string `json:"arch"`
+	OS           string `json:"os"`
+	ExternalMode string `json:"external_mode,omitempty"`
+	Timestamp    string `json:"timestamp"`
+
+	// URL overrides the telemetry endpoint (for testing only). Not serialized.
+	URL string `json:"-"`
+}
+
+// SendTelemetry posts the payload to the telemetry endpoint.
+// It respects the context deadline and logs at Debug level only.
+func SendTelemetry(ctx context.Context, payload TelemetryPayload) {
+	if payload.Arch == "" {
+		payload.Arch = runtime.GOARCH
+	}
+	if payload.OS == "" {
+		payload.OS = runtime.GOOS
+	}
+	if payload.Timestamp == "" {
+		payload.Timestamp = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	url := payload.URL
+	if url == "" {
+		url = defaultTelemetryURL
+	}
+
+	body, err := json.Marshal(payload)
+	if err != nil {
+		slog.Debug("telemetry: failed to marshal payload", "error", err)
+		return
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		slog.Debug("telemetry: failed to create request", "error", err)
+		return
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		slog.Debug("telemetry: request failed", "error", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	slog.Debug("telemetry: sent", "status", resp.StatusCode, "event", payload.Event)
+}
+
+// ReadMachineID returns a stable anonymous identifier for this machine.
+// Reads /etc/machine-id (standard Linux UUID). Falls back to a SHA-256
+// hash of hostname + first non-loopback MAC address.
+func ReadMachineID() string {
+	data, err := os.ReadFile("/etc/machine-id")
+	if err == nil {
+		id := strings.TrimSpace(string(data))
+		if id != "" {
+			return id
+		}
+	}
+
+	// Fallback: hash hostname + MAC
+	hostname, _ := os.Hostname()
+	mac := firstMAC()
+	hash := sha256.Sum256([]byte(hostname + mac))
+	return fmt.Sprintf("%x", hash[:16])
+}
+
+// firstMAC returns the MAC address of the first non-loopback, up interface.
+func firstMAC() string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return ""
+	}
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		if mac := iface.HardwareAddr.String(); mac != "" {
+			return mac
+		}
+	}
+	return ""
+}

spinifex/admin/telemetry_test.go

@@ -0,0 +1,107 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadMachineID_ReturnsNonEmpty(t *testing.T) {
+	t.Parallel()
+	id := ReadMachineID()
+	assert.NotEmpty(t, id, "machine ID should never be empty")
+	assert.Greater(t, len(id), 8, "machine ID should be a reasonable length")
+}
+
+func TestSendTelemetry_PostsCorrectPayload(t *testing.T) {
+	t.Parallel()
+	var received TelemetryPayload
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodPost, r.Method)
+		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+		require.NoError(t, json.Unmarshal(body, &received))
+
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	SendTelemetry(ctx, TelemetryPayload{
+		MachineID: "test-machine-123",
+		Event:     "init",
+		Region:    "ap-southeast-2",
+		AZ:        "ap-southeast-2a",
+		Node:      "node1",
+		Nodes:     3,
+		BindIP:    "10.11.12.1",
+		Version:   "v0.5.0",
+		URL:       server.URL,
+	})
+
+	assert.Equal(t, "test-machine-123", received.MachineID)
+	assert.Equal(t, "init", received.Event)
+	assert.Equal(t, "ap-southeast-2", received.Region)
+	assert.Equal(t, 3, received.Nodes)
+	assert.Equal(t, "v0.5.0", received.Version)
+	assert.NotEmpty(t, received.Arch, "arch should be auto-filled")
+	assert.NotEmpty(t, received.OS, "os should be auto-filled")
+	assert.NotEmpty(t, received.Timestamp, "timestamp should be auto-filled")
+}
+
+func TestSendTelemetry_RespectsTimeout(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(10 * time.Second)
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "timeout-test", URL: server.URL})
+	elapsed := time.Since(start)
+
+	assert.Less(t, elapsed, 2*time.Second, "should respect context timeout")
+}
+
+func TestSendTelemetry_HandlesServerError(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "error-test", URL: server.URL})
+}
+
+func TestSendTelemetry_HandlesUnreachableServer(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	// URL pointing to a port that's not listening
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "unreachable-test", URL: "http://127.0.0.1:1"})
+}
+
+func TestFirstMAC_ReturnsNonEmpty(t *testing.T) {
+	t.Parallel()
+	mac := firstMAC()
+	assert.NotEmpty(t, mac, "expected at least one MAC address")
+}

spinifex/albagent/agent.go

@@ -0,0 +1,305 @@
+// Package albagent implements the NATS config agent that runs inside ALB VMs.
+// It subscribes to config updates, writes HAProxy configuration, and reloads
+// the HAProxy process.
+package albagent
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+const (
+	// Default paths for HAProxy config and PID file.
+	DefaultConfigPath = "/etc/haproxy/haproxy.cfg"
+	DefaultPIDPath    = "/run/haproxy.pid"
+
+	// NATS topic patterns.
+	configTopicPrefix = "elbv2.alb."
+	configTopicSuffix = ".config"
+	pingTopicSuffix   = ".ping"
+)
+
+// Agent manages HAProxy configuration inside an ALB VM.
+type Agent struct {
+	lbID       string
+	natsURL    string
+	configPath string
+	pidPath    string
+
+	nc   *nats.Conn
+	subs []*nats.Subscription
+	mu   sync.Mutex
+
+	// For testing: override the reload function.
+	reloadFn func(configPath, pidPath string) error
+}
+
+// New creates a new ALB agent for the given load balancer.
+func New(lbID, natsURL string) (*Agent, error) {
+	if lbID == "" {
+		return nil, fmt.Errorf("lbID is required")
+	}
+	if natsURL == "" {
+		natsURL = "nats://127.0.0.1:4222"
+	}
+
+	return &Agent{
+		lbID:       lbID,
+		natsURL:    natsURL,
+		configPath: DefaultConfigPath,
+		pidPath:    DefaultPIDPath,
+		reloadFn:   reloadHAProxy,
+	}, nil
+}
+
+// ConfigTopic returns the NATS topic for config updates.
+func ConfigTopic(lbID string) string {
+	return configTopicPrefix + lbID + configTopicSuffix
+}
+
+// PingTopic returns the NATS topic for health pings.
+func PingTopic(lbID string) string {
+	return configTopicPrefix + lbID + pingTopicSuffix
+}
+
+// Start connects to NATS and subscribes to config and ping topics.
+func (a *Agent) Start() error {
+	nc, err := nats.Connect(a.natsURL,
+		nats.Name("alb-agent-"+a.lbID),
+		nats.MaxReconnects(-1),
+		nats.ReconnectWait(2*time.Second),
+		nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
+			slog.Warn("NATS disconnected", "err", err)
+		}),
+		nats.ReconnectHandler(func(_ *nats.Conn) {
+			slog.Info("NATS reconnected")
+		}),
+	)
+	if err != nil {
+		return fmt.Errorf("connect to NATS at %s: %w", a.natsURL, err)
+	}
+	a.nc = nc
+
+	// Subscribe to config updates
+	configSub, err := nc.Subscribe(ConfigTopic(a.lbID), a.handleConfig)
+	if err != nil {
+		nc.Close()
+		return fmt.Errorf("subscribe to config topic: %w", err)
+	}
+
+	// Subscribe to health pings (request/reply)
+	pingSub, err := nc.Subscribe(PingTopic(a.lbID), a.handlePing)
+	if err != nil {
+		nc.Close()
+		return fmt.Errorf("subscribe to ping topic: %w", err)
+	}
+
+	a.mu.Lock()
+	a.subs = append(a.subs, configSub, pingSub)
+	a.mu.Unlock()
+
+	// Flush to ensure the server has processed our subscriptions before
+	// we report that the agent is started and ready.
+	if err := nc.Flush(); err != nil {
+		nc.Close()
+		return fmt.Errorf("flush subscriptions: %w", err)
+	}
+
+	slog.Info("Agent started",
+		"lbId", a.lbID,
+		"configTopic", ConfigTopic(a.lbID),
+		"pingTopic", PingTopic(a.lbID),
+	)
+	return nil
+}
+
+// Stop gracefully shuts down the agent.
+func (a *Agent) Stop() {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	for _, sub := range a.subs {
+		if err := sub.Unsubscribe(); err != nil {
+			slog.Warn("Failed to unsubscribe", "err", err)
+		}
+	}
+	a.subs = nil
+
+	if a.nc != nil {
+		a.nc.Close()
+		a.nc = nil
+	}
+
+	slog.Info("Agent stopped", "lbId", a.lbID)
+}
+
+// handleConfig processes a config update message. The message payload is the
+// raw HAProxy config text. It writes the config to disk and reloads HAProxy.
+func (a *Agent) handleConfig(msg *nats.Msg) {
+	config := string(msg.Data)
+	if config == "" {
+		slog.Warn("Received empty config, ignoring")
+		a.respond(msg, "error", "empty config")
+		return
+	}
+
+	slog.Info("Received config update", "lbId", a.lbID, "size", len(config))
+
+	// Write config to disk
+	if err := WriteConfig(a.configPath, config); err != nil {
+		slog.Error("Failed to write config", "path", a.configPath, "err", err)
+		a.respond(msg, "error", err.Error())
+		return
+	}
+
+	// Reload HAProxy
+	if err := a.reloadFn(a.configPath, a.pidPath); err != nil {
+		slog.Error("Failed to reload HAProxy", "err", err)
+		a.respond(msg, "error", err.Error())
+		return
+	}
+
+	slog.Info("Config applied and HAProxy reloaded", "lbId", a.lbID)
+	a.respond(msg, "ok", "")
+}
+
+// PingResponse is the JSON response to a health ping.
+type PingResponse struct {
+	Status    string `json:"status"`
+	LBID      string `json:"lb_id"`
+	HAProxy   bool   `json:"haproxy_running"`
+	ConfigAge int64  `json:"config_age_seconds"`
+}
+
+// handlePing responds to health check requests.
+func (a *Agent) handlePing(msg *nats.Msg) {
+	haproxyRunning := isHAProxyRunning(a.pidPath)
+	configAge := configAgeSeconds(a.configPath)
+
+	resp := PingResponse{
+		Status:    "ok",
+		LBID:      a.lbID,
+		HAProxy:   haproxyRunning,
+		ConfigAge: configAge,
+	}
+
+	data, err := json.Marshal(resp)
+	if err != nil {
+		slog.Error("Failed to marshal ping response", "err", err)
+		return
+	}
+	if err := msg.Respond(data); err != nil {
+		slog.Warn("Failed to respond to ping", "err", err)
+	}
+}
+
+// respond sends a JSON reply to a NATS message if it has a reply subject.
+func (a *Agent) respond(msg *nats.Msg, status, errMsg string) {
+	if msg.Reply == "" {
+		return
+	}
+	resp := map[string]string{"status": status}
+	if errMsg != "" {
+		resp["error"] = errMsg
+	}
+	data, err := json.Marshal(resp)
+	if err != nil {
+		slog.Error("Failed to marshal response", "err", err)
+		return
+	}
+	if err := msg.Respond(data); err != nil {
+		slog.Warn("Failed to respond", "err", err)
+	}
+}
+
+// WriteConfig atomically writes an HAProxy config file.
+// It writes to a temp file first, then renames for atomicity.
+func WriteConfig(path, content string) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("create config dir: %w", err)
+	}
+
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
+		return fmt.Errorf("write temp config: %w", err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("rename config: %w", err)
+	}
+	return nil
+}
+
+// reloadHAProxy starts or reloads the HAProxy process.
+// If HAProxy is running (PID file exists and process alive), it does a
+// graceful reload with -sf. Otherwise it starts a fresh instance.
+func reloadHAProxy(configPath, pidPath string) error {
+	// Ensure the stats socket directory exists (the config may reference
+	// /tmp/spinifex-haproxy/ which doesn't exist on fresh Alpine VMs).
+	_ = os.MkdirAll("/tmp/spinifex-haproxy", 0o750)
+
+	oldPID := readPID(pidPath)
+
+	var cmd *exec.Cmd
+	if oldPID > 0 {
+		// Graceful reload: new worker starts, old workers finish in-flight requests
+		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D", "-sf", strconv.Itoa(oldPID))
+	} else {
+		// Fresh start
+		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D")
+	}
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("haproxy: %s: %w", strings.TrimSpace(string(out)), err)
+	}
+	return nil
+}
+
+// readPID reads the HAProxy PID from the PID file. Returns 0 if unavailable.
+func readPID(pidPath string) int {
+	data, err := os.ReadFile(pidPath)
+	if err != nil {
+		return 0
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(data)))
+	if err != nil {
+		return 0
+	}
+	// Check if process is still alive
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return 0
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return 0
+	}
+	return pid
+}
+
+// isHAProxyRunning checks if HAProxy is running via its PID file.
+func isHAProxyRunning(pidPath string) bool {
+	return readPID(pidPath) > 0
+}
+
+// configAgeSeconds returns how many seconds since the config file was last modified.
+// Returns -1 if the file doesn't exist.
+func configAgeSeconds(configPath string) int64 {
+	info, err := os.Stat(configPath)
+	if err != nil {
+		return -1
+	}
+	return int64(time.Since(info.ModTime()).Seconds())
+}

spinifex/albagent/agent_test.go

@@ -0,0 +1,342 @@
+package albagent
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	natstest "github.com/nats-io/nats-server/v2/test"
+	"github.com/nats-io/nats.go"
+)
+
+func startTestNATS(t *testing.T) *natsserver.Server {
+	t.Helper()
+	opts := natstest.DefaultTestOptions
+	opts.Port = -1 // random port
+	ns := natstest.RunServer(&opts)
+	t.Cleanup(ns.Shutdown)
+	return ns
+}
+
+func TestNew(t *testing.T) {
+	agent, err := New("lb-test123", "nats://localhost:4222")
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if agent.lbID != "lb-test123" {
+		t.Errorf("lbID = %q, want %q", agent.lbID, "lb-test123")
+	}
+}
+
+func TestNew_EmptyLBID(t *testing.T) {
+	_, err := New("", "nats://localhost:4222")
+	if err == nil {
+		t.Fatal("expected error for empty lbID")
+	}
+}
+
+func TestNew_DefaultNATSURL(t *testing.T) {
+	agent, err := New("lb-test", "")
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if agent.natsURL != "nats://127.0.0.1:4222" {
+		t.Errorf("natsURL = %q, want default", agent.natsURL)
+	}
+}
+
+func TestConfigTopic(t *testing.T) {
+	topic := ConfigTopic("lb-abc123")
+	if topic != "elbv2.alb.lb-abc123.config" {
+		t.Errorf("ConfigTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.config")
+	}
+}
+
+func TestPingTopic(t *testing.T) {
+	topic := PingTopic("lb-abc123")
+	if topic != "elbv2.alb.lb-abc123.ping" {
+		t.Errorf("PingTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.ping")
+	}
+}
+
+func TestWriteConfig(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "haproxy.cfg")
+
+	content := "global\n  log stdout\n"
+	if err := WriteConfig(path, content); err != nil {
+		t.Fatalf("WriteConfig: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(data) != content {
+		t.Errorf("config = %q, want %q", string(data), content)
+	}
+}
+
+func TestWriteConfig_CreatesDir(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "subdir", "haproxy.cfg")
+
+	if err := WriteConfig(path, "test"); err != nil {
+		t.Fatalf("WriteConfig: %v", err)
+	}
+
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("config file not created: %v", err)
+	}
+}
+
+func TestWriteConfig_Atomic(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "haproxy.cfg")
+
+	// Write initial config
+	if err := WriteConfig(path, "initial"); err != nil {
+		t.Fatalf("WriteConfig initial: %v", err)
+	}
+
+	// Overwrite with new config
+	if err := WriteConfig(path, "updated"); err != nil {
+		t.Fatalf("WriteConfig updated: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	if string(data) != "updated" {
+		t.Errorf("config = %q, want %q", string(data), "updated")
+	}
+
+	// No temp file left behind
+	if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) {
+		t.Error("temp file should not exist after successful write")
+	}
+}
+
+func TestConfigAgeSeconds(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.cfg")
+
+	// Non-existent file
+	if age := configAgeSeconds(path); age != -1 {
+		t.Errorf("age of non-existent file = %d, want -1", age)
+	}
+
+	// Write file and check age
+	os.WriteFile(path, []byte("test"), 0o644)
+	age := configAgeSeconds(path)
+	if age < 0 || age > 2 {
+		t.Errorf("age = %d, expected 0-2 seconds", age)
+	}
+}
+
+func TestIsHAProxyRunning_NoPIDFile(t *testing.T) {
+	if isHAProxyRunning("/nonexistent/haproxy.pid") {
+		t.Error("expected false for non-existent PID file")
+	}
+}
+
+func TestReadPID_InvalidContent(t *testing.T) {
+	dir := t.TempDir()
+	pidFile := filepath.Join(dir, "haproxy.pid")
+
+	os.WriteFile(pidFile, []byte("not-a-number\n"), 0o644)
+	if pid := readPID(pidFile); pid != 0 {
+		t.Errorf("readPID = %d, want 0 for invalid content", pid)
+	}
+}
+
+func TestReadPID_DeadProcess(t *testing.T) {
+	dir := t.TempDir()
+	pidFile := filepath.Join(dir, "haproxy.pid")
+
+	// Use a very high PID that's unlikely to be running
+	os.WriteFile(pidFile, []byte("999999999\n"), 0o644)
+	if pid := readPID(pidFile); pid != 0 {
+		t.Errorf("readPID = %d, want 0 for dead process", pid)
+	}
+}
+
+func TestAgent_StartStop(t *testing.T) {
+	ns := startTestNATS(t)
+
+	agent, err := New("lb-test123", ns.ClientURL())
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+
+	// Verify agent is connected
+	if agent.nc == nil || !agent.nc.IsConnected() {
+		t.Fatal("agent not connected to NATS")
+	}
+
+	agent.Stop()
+
+	if agent.nc != nil {
+		t.Error("nc should be nil after Stop")
+	}
+}
+
+func TestAgent_HandleConfig(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-cfg123", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+
+	// Mock reload to avoid needing real HAProxy
+	reloadCalled := false
+	agent.reloadFn = func(configPath, pidPath string) error {
+		reloadCalled = true
+		return nil
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	// Send a config update via a separate NATS connection
+	nc, err := nats.Connect(ns.ClientURL())
+	if err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	defer nc.Close()
+
+	haproxyConfig := "global\n  log stdout\ndefaults\n  mode http\n"
+	reply, err := nc.Request(ConfigTopic("lb-cfg123"), []byte(haproxyConfig), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	// Check response
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "ok" {
+		t.Errorf("response status = %q, want %q", resp["status"], "ok")
+	}
+
+	// Verify config was written
+	data, err := os.ReadFile(agent.configPath)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(data) != haproxyConfig {
+		t.Errorf("written config = %q, want %q", string(data), haproxyConfig)
+	}
+
+	if !reloadCalled {
+		t.Error("reload function was not called")
+	}
+}
+
+func TestAgent_HandleConfig_Empty(t *testing.T) {
+	ns := startTestNATS(t)
+
+	agent, _ := New("lb-empty", ns.ClientURL())
+	agent.reloadFn = func(_, _ string) error { return nil }
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(ConfigTopic("lb-empty"), []byte(""), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "error" {
+		t.Errorf("response status = %q, want %q for empty config", resp["status"], "error")
+	}
+}
+
+func TestAgent_HandlePing(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-ping123", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(PingTopic("lb-ping123"), nil, 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp PingResponse
+	if err := json.Unmarshal(reply.Data, &resp); err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+
+	if resp.Status != "ok" {
+		t.Errorf("status = %q, want %q", resp.Status, "ok")
+	}
+	if resp.LBID != "lb-ping123" {
+		t.Errorf("lb_id = %q, want %q", resp.LBID, "lb-ping123")
+	}
+	if resp.HAProxy {
+		t.Error("haproxy_running should be false (no PID file)")
+	}
+	if resp.ConfigAge != -1 {
+		t.Errorf("config_age = %d, want -1 (no config file)", resp.ConfigAge)
+	}
+}
+
+func TestAgent_HandleConfig_ReloadError(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-err", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+	agent.reloadFn = func(_, _ string) error {
+		return fmt.Errorf("haproxy binary not found")
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(ConfigTopic("lb-err"), []byte("some config"), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "error" {
+		t.Errorf("response status = %q, want %q", resp["status"], "error")
+	}
+	if resp["error"] == "" {
+		t.Error("expected error message in response")
+	}
+}

spinifex/awsec2query/parser.go

@@ -186,10 +186,20 @@ func setFieldValue(field reflect.Value, value string) error {
 }
 
 func setSliceField(field reflect.Value, params map[string]string, prefix string) error {
-	// Find all indexed items for this slice
+	// Find all indexed items for this slice.
+	// Supports both EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N) formats.
 	indices := make(map[int]bool)
+	useMember := false
 	for key := range params {
-		if strings.HasPrefix(key, prefix+".") {
+		if strings.HasPrefix(key, prefix+".member.") {
+			parts := strings.Split(key[len(prefix)+len(".member."):], ".")
+			if len(parts) > 0 {
+				if idx, err := strconv.Atoi(parts[0]); err == nil {
+					indices[idx] = true
+					useMember = true
+				}
+			}
+		} else if strings.HasPrefix(key, prefix+".") {
 			parts := strings.Split(key[len(prefix)+1:], ".")
 			if len(parts) > 0 {
 				if idx, err := strconv.Atoi(parts[0]); err == nil {
@@ -221,7 +231,12 @@ func setSliceField(field reflect.Value, params map[string]string, prefix string)
 		}
 
 		elem := slice.Index(idx - 1)
-		indexPrefix := fmt.Sprintf("%s.%d", prefix, idx)
+		var indexPrefix string
+		if useMember {
+			indexPrefix = fmt.Sprintf("%s.member.%d", prefix, idx)
+		} else {
+			indexPrefix = fmt.Sprintf("%s.%d", prefix, idx)
+		}
 
 		// Handle different element types
 		switch elemType.Kind() {

spinifex/awsec2query/parser_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -509,3 +510,81 @@ func TestQueryParamsToStruct_BlockDeviceMappingsWithIOPS(t *testing.T) {
 	assert.Equal(t, int64(500), aws.Int64Value(bdm.Ebs.Throughput))
 	assert.Equal(t, "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", aws.StringValue(bdm.Ebs.KmsKeyId))
 }
+
+// --- ELBv2 member-style list tests ---
+
+func TestQueryParamsToStruct_ELBv2RegisterTargets(t *testing.T) {
+	// ELBv2 uses Targets.member.N.Id format (IAM-style query protocol)
+	args := map[string]string{
+		"Action":              "RegisterTargets",
+		"TargetGroupArn":      "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+		"Targets.member.1.Id": "i-1234567890abcdef0",
+		"Targets.member.2.Id": "i-0987654321fedcba0",
+	}
+
+	input := &elbv2.RegisterTargetsInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123", aws.StringValue(input.TargetGroupArn))
+	assert.Len(t, input.Targets, 2)
+	assert.Equal(t, "i-1234567890abcdef0", aws.StringValue(input.Targets[0].Id))
+	assert.Equal(t, "i-0987654321fedcba0", aws.StringValue(input.Targets[1].Id))
+}
+
+func TestQueryParamsToStruct_ELBv2RegisterTargetsWithPort(t *testing.T) {
+	args := map[string]string{
+		"Action":                "RegisterTargets",
+		"TargetGroupArn":        "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+		"Targets.member.1.Id":   "i-1234567890abcdef0",
+		"Targets.member.1.Port": "8080",
+	}
+
+	input := &elbv2.RegisterTargetsInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Len(t, input.Targets, 1)
+	assert.Equal(t, "i-1234567890abcdef0", aws.StringValue(input.Targets[0].Id))
+	assert.Equal(t, int64(8080), aws.Int64Value(input.Targets[0].Port))
+}
+
+func TestQueryParamsToStruct_ELBv2CreateLoadBalancerMemberSubnets(t *testing.T) {
+	// ELBv2 uses Subnets.member.N format for string slices
+	args := map[string]string{
+		"Action":           "CreateLoadBalancer",
+		"Name":             "my-alb",
+		"Subnets.member.1": "subnet-abc123",
+		"Subnets.member.2": "subnet-def456",
+	}
+
+	input := &elbv2.CreateLoadBalancerInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "my-alb", aws.StringValue(input.Name))
+	assert.Len(t, input.Subnets, 2)
+	assert.Equal(t, "subnet-abc123", aws.StringValue(input.Subnets[0]))
+	assert.Equal(t, "subnet-def456", aws.StringValue(input.Subnets[1]))
+}
+
+func TestQueryParamsToStruct_ELBv2CreateListenerWithActions(t *testing.T) {
+	args := map[string]string{
+		"Action":                                 "CreateListener",
+		"LoadBalancerArn":                        "arn:aws:elasticloadbalancing:us-east-1:000000000001:loadbalancer/app/my-alb/lb-abc123",
+		"Protocol":                               "HTTP",
+		"Port":                                   "80",
+		"DefaultActions.member.1.Type":           "forward",
+		"DefaultActions.member.1.TargetGroupArn": "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+	}
+
+	input := &elbv2.CreateListenerInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "HTTP", aws.StringValue(input.Protocol))
+	assert.Equal(t, int64(80), aws.Int64Value(input.Port))
+	assert.Len(t, input.DefaultActions, 1)
+	assert.Equal(t, "forward", aws.StringValue(input.DefaultActions[0].Type))
+	assert.Equal(t, "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123", aws.StringValue(input.DefaultActions[0].TargetGroupArn))
+}

spinifex/awserrors/awserrors.go

@@ -343,6 +343,7 @@ var (
 	ErrorPendingVpcPeeringConnectionLimitExceeded              = "PendingVpcPeeringConnectionLimitExceeded"
 	ErrorPlacementGroupLimitExceeded                           = "PlacementGroupLimitExceeded"
 	ErrorPrivateIpAddressLimitExceeded                         = "PrivateIpAddressLimitExceeded"
+	ErrorRequestEntityTooLarge                                 = "RequestEntityTooLarge"
 	ErrorRequestExpired                                        = "RequestExpired"
 	ErrorRequestLimitExceeded                                  = "RequestLimitExceeded"
 	ErrorRequestResourceCountExceeded                          = "RequestResourceCountExceeded"
@@ -432,6 +433,24 @@ var (
 	ErrorIAMInvalidInput            = ErrorInvalidInput // alias for IAM usage
 	ErrorIAMMalformedPolicyDocument = "MalformedPolicyDocument"
 	ErrorAccessDenied               = "AccessDenied"
+
+	// ELBv2-specific error codes
+	ErrorELBv2LoadBalancerNotFound         = "LoadBalancerNotFound"
+	ErrorELBv2TargetGroupNotFound          = "TargetGroupNotFound"
+	ErrorELBv2ListenerNotFound             = "ListenerNotFound"
+	ErrorELBv2DuplicateLoadBalancer        = "DuplicateLoadBalancerName"
+	ErrorELBv2DuplicateTargetGroup         = "DuplicateTargetGroupName"
+	ErrorELBv2DuplicateListener            = "DuplicateListener"
+	ErrorELBv2TooManyLoadBalancers         = "TooManyLoadBalancers"
+	ErrorELBv2TooManyTargetGroups          = "TooManyTargetGroups"
+	ErrorELBv2TooManyListeners             = "TooManyListeners"
+	ErrorELBv2TooManyTargets               = "TooManyRegistrationsForTargetId"
+	ErrorELBv2InvalidTarget                = "InvalidTarget"
+	ErrorELBv2TargetGroupInUse             = "ResourceInUse"
+	ErrorELBv2InvalidSecurityGroup         = "InvalidSecurityGroup"
+	ErrorELBv2InvalidScheme                = "InvalidScheme"
+	ErrorELBv2SubnetNotFound               = "SubnetNotFound"
+	ErrorELBv2AvailabilityZoneNotSupported = "AvailabilityZoneNotSupported"
 )
 
 // ValidErrorCode returns the error code if it exists in ErrorLookup,
@@ -782,6 +801,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorPendingVpcPeeringConnectionLimitExceeded:              {HTTPCode: 409, Message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
 	ErrorPlacementGroupLimitExceeded:                           {HTTPCode: 400, Message: "You've reached the limit on the number of placement groups that you can have."},
 	ErrorPrivateIpAddressLimitExceeded:                         {HTTPCode: 400, Message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
+	ErrorRequestEntityTooLarge:                                 {HTTPCode: 413, Message: "Request body exceeds the maximum allowed size."},
 	ErrorRequestExpired:                                        {HTTPCode: 403, Message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
 	ErrorRequestLimitExceeded:                                  {HTTPCode: 500, Message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
 	ErrorRequestResourceCountExceeded:                          {HTTPCode: 409, Message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
@@ -870,4 +890,22 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorIAMLimitExceeded:           {HTTPCode: 409, Message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
 	ErrorIAMMalformedPolicyDocument: {HTTPCode: 400, Message: "The policy document is malformed."},
 	ErrorAccessDenied:               {HTTPCode: 403, Message: "User is not authorized to perform this action."},
+
+	// ELBv2 error codes
+	ErrorELBv2LoadBalancerNotFound:         {HTTPCode: 400, Message: "One or more load balancers not found."},
+	ErrorELBv2TargetGroupNotFound:          {HTTPCode: 400, Message: "One or more target groups not found."},
+	ErrorELBv2ListenerNotFound:             {HTTPCode: 400, Message: "One or more listeners not found."},
+	ErrorELBv2DuplicateLoadBalancer:        {HTTPCode: 400, Message: "A load balancer with the same name already exists."},
+	ErrorELBv2DuplicateTargetGroup:         {HTTPCode: 400, Message: "A target group with the same name already exists."},
+	ErrorELBv2DuplicateListener:            {HTTPCode: 400, Message: "A listener with the specified port already exists on this load balancer."},
+	ErrorELBv2TooManyLoadBalancers:         {HTTPCode: 400, Message: "You've reached the limit on the number of load balancers for your account."},
+	ErrorELBv2TooManyTargetGroups:          {HTTPCode: 400, Message: "You've reached the limit on the number of target groups for your account."},
+	ErrorELBv2TooManyListeners:             {HTTPCode: 400, Message: "You've reached the limit on the number of listeners per load balancer."},
+	ErrorELBv2TooManyTargets:               {HTTPCode: 400, Message: "You've reached the limit on the number of times a target can be registered with a target group."},
+	ErrorELBv2InvalidTarget:                {HTTPCode: 400, Message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
+	ErrorELBv2TargetGroupInUse:             {HTTPCode: 400, Message: "The target group is currently in use by a listener or rule."},
+	ErrorELBv2InvalidSecurityGroup:         {HTTPCode: 400, Message: "The specified security group does not exist."},
+	ErrorELBv2InvalidScheme:                {HTTPCode: 400, Message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
+	ErrorELBv2SubnetNotFound:               {HTTPCode: 400, Message: "The specified subnet does not exist."},
+	ErrorELBv2AvailabilityZoneNotSupported: {HTTPCode: 400, Message: "The specified Availability Zone is not supported."},
 }

spinifex/awserrors/awserrors_test.go

@@ -345,6 +345,7 @@ func TestErrorLookup(t *testing.T) {
 		{code: "PendingVpcPeeringConnectionLimitExceeded", http: 409, message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
 		{code: "PlacementGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of placement groups that you can have."},
 		{code: "PrivateIpAddressLimitExceeded", http: 400, message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
+		{code: "RequestEntityTooLarge", http: 413, message: "Request body exceeds the maximum allowed size."},
 		{code: "RequestExpired", http: 403, message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
 		{code: "RequestLimitExceeded", http: 500, message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
 		{code: "RequestResourceCountExceeded", http: 409, message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
@@ -433,6 +434,23 @@ func TestErrorLookup(t *testing.T) {
 		{code: "LimitExceeded", http: 409, message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
 		{code: "MalformedPolicyDocument", http: 400, message: "The policy document is malformed."},
 		{code: "AccessDenied", http: 403, message: "User is not authorized to perform this action."},
+		// ELBv2 error codes
+		{code: "LoadBalancerNotFound", http: 400, message: "One or more load balancers not found."},
+		{code: "TargetGroupNotFound", http: 400, message: "One or more target groups not found."},
+		{code: "ListenerNotFound", http: 400, message: "One or more listeners not found."},
+		{code: "DuplicateLoadBalancerName", http: 400, message: "A load balancer with the same name already exists."},
+		{code: "DuplicateTargetGroupName", http: 400, message: "A target group with the same name already exists."},
+		{code: "DuplicateListener", http: 400, message: "A listener with the specified port already exists on this load balancer."},
+		{code: "TooManyLoadBalancers", http: 400, message: "You've reached the limit on the number of load balancers for your account."},
+		{code: "TooManyTargetGroups", http: 400, message: "You've reached the limit on the number of target groups for your account."},
+		{code: "TooManyListeners", http: 400, message: "You've reached the limit on the number of listeners per load balancer."},
+		{code: "TooManyRegistrationsForTargetId", http: 400, message: "You've reached the limit on the number of times a target can be registered with a target group."},
+		{code: "InvalidTarget", http: 400, message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
+		{code: "ResourceInUse", http: 400, message: "The target group is currently in use by a listener or rule."},
+		{code: "InvalidSecurityGroup", http: 400, message: "The specified security group does not exist."},
+		{code: "InvalidScheme", http: 400, message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
+		{code: "SubnetNotFound", http: 400, message: "The specified subnet does not exist."},
+		{code: "AvailabilityZoneNotSupported", http: 400, message: "The specified Availability Zone is not supported."},
 	}
 
 	if len(ErrorLookup) != len(expected) {

spinifex/config/config.go

@@ -5,15 +5,51 @@ import (
 	"log/slog"
 	"os"
 	"slices"
+	"strings"
 
 	"github.com/spf13/viper"
 )
 
 type ClusterConfig struct {
-	Epoch   uint64            `mapstructure:"epoch"`   // bump when leader commits changes
-	Node    string            `mapstructure:"node"`    // my node name
-	Version string            `mapstructure:"version"` // spinifex version
-	Nodes   map[string]Config `mapstructure:"nodes"`   // full config for every node
+	Epoch     uint64            `mapstructure:"epoch"`     // bump when leader commits changes
+	Node      string            `mapstructure:"node"`      // my node name
+	Version   string            `mapstructure:"version"`   // spinifex version
+	Network   NetworkConfig     `mapstructure:"network"`   // cluster-wide external network settings
+	Bootstrap BootstrapConfig   `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
+	Nodes     map[string]Config `mapstructure:"nodes"`     // full config for every node
+}
+
+// ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
+type ExternalPool struct {
+	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
+	Source     string   `mapstructure:"source"`      // IP source: "static" (default) or "dhcp" (from router DHCP)
+	RangeStart string   `mapstructure:"range_start"` // First IP in range (static source only)
+	RangeEnd   string   `mapstructure:"range_end"`   // Last IP in range (static source only)
+	Gateway    string   `mapstructure:"gateway"`     // WAN default gateway (next hop for 0.0.0.0/0)
+	GatewayIP  string   `mapstructure:"gateway_ip"`  // OVN router external IP (override; defaults to first IP in range)
+	PrefixLen  int      `mapstructure:"prefix_len"`  // Subnet mask (default 24)
+	DNSServers []string `mapstructure:"dns_servers"` // DNS servers for VM DHCP (auto-detected from host; fallback: 8.8.8.8, 1.1.1.1)
+	Region     string   `mapstructure:"region"`      // Scope to region (optional — empty means any region)
+	AZ         string   `mapstructure:"az"`          // Scope to AZ (optional — empty means any AZ in region)
+}
+
+// NetworkConfig holds cluster-wide external network settings.
+type NetworkConfig struct {
+	ExternalMode  string         `mapstructure:"external_mode"`  // "pool" or "" (disabled)
+	ExternalDHCP  bool           `mapstructure:"external_dhcp"`  // Gateway IP obtained via DHCP (pool/dhcp source)
+	ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools
+}
+
+// BootstrapConfig holds the default VPC infrastructure IDs written by admin init.
+// vpcd reads this on startup to ensure OVN topology exists for the bootstrap VPC,
+// covering the case where admin init ran before services were started.
+type BootstrapConfig struct {
+	AccountID  string `mapstructure:"account_id"`
+	VpcId      string `mapstructure:"vpc_id"`
+	SubnetId   string `mapstructure:"subnet_id"`
+	IgwId      string `mapstructure:"igw_id"`
+	Cidr       string `mapstructure:"cidr"`
+	SubnetCidr string `mapstructure:"subnet_cidr"`
 }
 
 // Config holds all configuration for the application
@@ -53,8 +89,11 @@ type ViperblockConfig struct {
 
 // VPCDConfig holds the VPC daemon (vpcd) configuration.
 type VPCDConfig struct {
-	OVNNBAddr string `mapstructure:"ovn_nb_addr"` // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641")
-	OVNSBAddr string `mapstructure:"ovn_sb_addr"` // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642")
+	OVNNBAddr         string `mapstructure:"ovn_nb_addr"`        // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641")
+	OVNSBAddr         string `mapstructure:"ovn_sb_addr"`        // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642")
+	ExternalInterface string `mapstructure:"external_interface"` // WAN NIC name (e.g., "eth1", "enp0s3") — the physical NIC on the WAN bridge
+	WanBridge         string `mapstructure:"wan_bridge"`         // OVS bridge for WAN traffic (default "br-wan", maps to OVN "external" network)
+	BridgeMode        string `mapstructure:"bridge_mode"`        // "direct" or "macvlan" (auto-detected if empty)
 }
 
 type PredastoreConfig struct {
@@ -159,5 +198,15 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 		return nil, fmt.Errorf("error unmarshaling config: %w", err)
 	}
 
+	// Normalize the local node's bind address: 0.0.0.0 means "listen on all
+	// interfaces" but is not a valid connect address. Only rewrite for the
+	// local node — remote nodes use real IPs that must not be changed.
+	if local, ok := config.Nodes[config.Node]; ok {
+		if strings.HasPrefix(local.Predastore.Host, "0.0.0.0") {
+			local.Predastore.Host = strings.Replace(local.Predastore.Host, "0.0.0.0", "127.0.0.1", 1)
+			config.Nodes[config.Node] = local
+		}
+	}
+
 	return &config, nil
 }

spinifex/config/config_test.go

@@ -205,7 +205,7 @@ expected_nodes = 3
 	assert.Equal(t, "0.0.0.0:4222", n.NATS.Host)
 	assert.Equal(t, "secret-token", n.NATS.ACL.Token)
 	assert.Equal(t, "test-subject", n.NATS.Sub.Subject)
-	assert.Equal(t, "0.0.0.0:8443", n.Predastore.Host)
+	assert.Equal(t, "127.0.0.1:8443", n.Predastore.Host) // 0.0.0.0 normalized to loopback
 	assert.Equal(t, "mybucket", n.Predastore.Bucket)
 	assert.Equal(t, "AK", n.Predastore.AccessKey)
 	assert.Equal(t, "/data", n.Predastore.BaseDir)
@@ -300,6 +300,141 @@ func TestNodeBaseDir_EmptyBaseDir(t *testing.T) {
 	assert.Equal(t, "", cc.NodeBaseDir())
 }
 
+// Tests for NetworkConfig (external pools)
+
+func TestLoadConfig_NetworkExternalPools(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+
+	toml := `
+node = "n1"
+
+[network]
+external_mode = "pool"
+
+[[network.external_pools]]
+name = "wan"
+range_start = "192.168.1.150"
+range_end = "192.168.1.250"
+gateway = "192.168.1.1"
+prefix_len = 24
+
+[[network.external_pools]]
+name = "overflow"
+range_start = "10.0.0.2"
+range_end = "10.0.0.254"
+gateway = "10.0.0.1"
+prefix_len = 24
+region = "us-east-1"
+az = "us-east-1a"
+
+[nodes.n1]
+region = "us-east-1"
+
+[nodes.n1.vpcd]
+ovn_nb_addr = "tcp:127.0.0.1:6641"
+external_interface = "enp0s3"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	assert.Equal(t, "pool", cfg.Network.ExternalMode)
+	require.Len(t, cfg.Network.ExternalPools, 2)
+
+	wan := cfg.Network.ExternalPools[0]
+	assert.Equal(t, "wan", wan.Name)
+	assert.Equal(t, "192.168.1.150", wan.RangeStart)
+	assert.Equal(t, "192.168.1.250", wan.RangeEnd)
+	assert.Equal(t, "192.168.1.1", wan.Gateway)
+	assert.Equal(t, 24, wan.PrefixLen)
+	assert.Empty(t, wan.Region)
+	assert.Empty(t, wan.AZ)
+
+	overflow := cfg.Network.ExternalPools[1]
+	assert.Equal(t, "overflow", overflow.Name)
+	assert.Equal(t, "us-east-1", overflow.Region)
+	assert.Equal(t, "us-east-1a", overflow.AZ)
+
+	n := cfg.Nodes["n1"]
+	assert.Equal(t, "enp0s3", n.VPCD.ExternalInterface)
+}
+
+func TestLoadConfig_NetworkPoolDHCPMode(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+
+	toml := `
+node = "n1"
+
+[network]
+external_mode = "pool"
+
+[[network.external_pools]]
+name = "wan"
+source = "dhcp"
+gateway = "192.168.1.1"
+gateway_ip = "192.168.1.100"
+prefix_len = 24
+
+[nodes.n1]
+region = "us-east-1"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	assert.Equal(t, "pool", cfg.Network.ExternalMode)
+	require.Len(t, cfg.Network.ExternalPools, 1)
+	assert.Equal(t, "dhcp", cfg.Network.ExternalPools[0].Source)
+	assert.Equal(t, "192.168.1.100", cfg.Network.ExternalPools[0].GatewayIP)
+}
+
+func TestLoadConfig_NetworkDisabledByDefault(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+
+	require.NoError(t, os.WriteFile(path, []byte(`node = "n1"`), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	assert.Empty(t, cfg.Network.ExternalMode)
+	assert.Empty(t, cfg.Network.ExternalPools)
+}
+
+func TestLoadConfig_ExternalInterfacePerNode(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+
+	toml := `
+node = "n1"
+
+[nodes.n1.vpcd]
+external_interface = "eth1"
+
+[nodes.n2.vpcd]
+external_interface = "eno2"
+
+[nodes.n3.vpcd]
+external_interface = "enp3s0"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+
+	assert.Equal(t, "eth1", cfg.Nodes["n1"].VPCD.ExternalInterface)
+	assert.Equal(t, "eno2", cfg.Nodes["n2"].VPCD.ExternalInterface)
+	assert.Equal(t, "enp3s0", cfg.Nodes["n3"].VPCD.ExternalInterface)
+}
+
 // Tests for ViperblockConfig
 
 func TestLoadConfig_ViperblockShardWAL_Explicit(t *testing.T) {

spinifex/daemon/daemon.go

@@ -28,18 +28,24 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/go-chi/chi/v5"
+	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
 	handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
+	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
 	handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw"
 	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+	handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw"
+	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
+	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
@@ -86,28 +92,35 @@ type ResourceManager struct {
 	natsConn     *nats.Conn
 	instanceSubs map[string]*nats.Subscription
 	handler      nats.MsgHandler
+	nodeID       string // node identifier for node-specific topic subscriptions
 }
 
 // Daemon represents the main daemon service
 type Daemon struct {
-	node            string
-	clusterConfig   *config.ClusterConfig
-	config          *config.Config
-	natsConn        *nats.Conn
-	resourceMgr     *ResourceManager
-	instanceService *handlers_ec2_instance.InstanceServiceImpl
-	keyService      *handlers_ec2_key.KeyServiceImpl
-	imageService    *handlers_ec2_image.ImageServiceImpl
-	volumeService   *handlers_ec2_volume.VolumeServiceImpl
-	accountService  *handlers_ec2_account.AccountSettingsServiceImpl
-	snapshotService *handlers_ec2_snapshot.SnapshotServiceImpl
-	tagsService     *handlers_ec2_tags.TagsServiceImpl
-	eigwService     *handlers_ec2_eigw.EgressOnlyIGWServiceImpl
-	igwService      *handlers_ec2_igw.IGWServiceImpl
-	vpcService      *handlers_ec2_vpc.VPCServiceImpl
-	ctx             context.Context
-	cancel          context.CancelFunc
-	shutdownWg      sync.WaitGroup
+	node                  string
+	clusterConfig         *config.ClusterConfig
+	config                *config.Config
+	natsConn              *nats.Conn
+	resourceMgr           *ResourceManager
+	instanceService       *handlers_ec2_instance.InstanceServiceImpl
+	keyService            *handlers_ec2_key.KeyServiceImpl
+	imageService          *handlers_ec2_image.ImageServiceImpl
+	volumeService         *handlers_ec2_volume.VolumeServiceImpl
+	accountService        *handlers_ec2_account.AccountSettingsServiceImpl
+	snapshotService       *handlers_ec2_snapshot.SnapshotServiceImpl
+	tagsService           *handlers_ec2_tags.TagsServiceImpl
+	eigwService           *handlers_ec2_eigw.EgressOnlyIGWServiceImpl
+	igwService            *handlers_ec2_igw.IGWServiceImpl
+	placementGroupService *handlers_ec2_placementgroup.PlacementGroupServiceImpl
+	vpcService            *handlers_ec2_vpc.VPCServiceImpl
+	eipService            *handlers_ec2_eip.EIPServiceImpl
+	elbv2Service          *handlers_elbv2.ELBv2ServiceImpl
+	routeTableService     *handlers_ec2_routetable.RouteTableServiceImpl
+	natGatewayService     *handlers_ec2_natgw.NatGatewayServiceImpl
+	externalIPAM          *handlers_ec2_vpc.ExternalIPAM
+	ctx                   context.Context
+	cancel                context.CancelFunc
+	shutdownWg            sync.WaitGroup
 
 	// Local VM Instances
 	Instances vm.Instances
@@ -386,16 +399,49 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.DescribeInternetGateways", d.handleEC2DescribeInternetGateways, "spinifex-workers"},
 		{"ec2.AttachInternetGateway", d.handleEC2AttachInternetGateway, "spinifex-workers"},
 		{"ec2.DetachInternetGateway", d.handleEC2DetachInternetGateway, "spinifex-workers"},
+		{"ec2.CreatePlacementGroup", d.handleEC2CreatePlacementGroup, "spinifex-workers"},
+		{"ec2.DeletePlacementGroup", d.handleEC2DeletePlacementGroup, "spinifex-workers"},
+		{"ec2.DescribePlacementGroups", d.handleEC2DescribePlacementGroups, "spinifex-workers"},
+		{"ec2.ReserveSpreadNodes", d.handleEC2ReserveSpreadNodes, "spinifex-workers"},
+		{"ec2.FinalizeSpreadInstances", d.handleEC2FinalizeSpreadInstances, "spinifex-workers"},
+		{"ec2.ReleaseSpreadNodes", d.handleEC2ReleaseSpreadNodes, "spinifex-workers"},
+		{"ec2.RemoveInstanceFromPlacementGroup", d.handleEC2RemoveInstanceFromPlacementGroup, "spinifex-workers"},
+		{"ec2.ReserveClusterNode", d.handleEC2ReserveClusterNode, "spinifex-workers"},
+		{"ec2.FinalizeClusterInstances", d.handleEC2FinalizeClusterInstances, "spinifex-workers"},
+		{"ec2.CreateNatGateway", d.handleEC2CreateNatGateway, "spinifex-workers"},
+		{"ec2.DeleteNatGateway", d.handleEC2DeleteNatGateway, "spinifex-workers"},
+		{"ec2.DescribeNatGateways", d.handleEC2DescribeNatGateways, "spinifex-workers"},
+		{"ec2.CreateRouteTable", d.handleEC2CreateRouteTable, "spinifex-workers"},
+		{"ec2.DeleteRouteTable", d.handleEC2DeleteRouteTable, "spinifex-workers"},
+		{"ec2.DescribeRouteTables", d.handleEC2DescribeRouteTables, "spinifex-workers"},
+		{"ec2.CreateRoute", d.handleEC2CreateRoute, "spinifex-workers"},
+		{"ec2.DeleteRoute", d.handleEC2DeleteRoute, "spinifex-workers"},
+		{"ec2.ReplaceRoute", d.handleEC2ReplaceRoute, "spinifex-workers"},
+		{"ec2.AssociateRouteTable", d.handleEC2AssociateRouteTable, "spinifex-workers"},
+		{"ec2.DisassociateRouteTable", d.handleEC2DisassociateRouteTable, "spinifex-workers"},
+		{"ec2.ReplaceRouteTableAssociation", d.handleEC2ReplaceRouteTableAssociation, "spinifex-workers"},
 		{"ec2.CreateVpc", d.handleEC2CreateVpc, "spinifex-workers"},
 		{"ec2.DeleteVpc", d.handleEC2DeleteVpc, "spinifex-workers"},
 		{"ec2.DescribeVpcs", d.handleEC2DescribeVpcs, "spinifex-workers"},
 		{"ec2.CreateSubnet", d.handleEC2CreateSubnet, "spinifex-workers"},
 		{"ec2.DeleteSubnet", d.handleEC2DeleteSubnet, "spinifex-workers"},
 		{"ec2.DescribeSubnets", d.handleEC2DescribeSubnets, "spinifex-workers"},
+		{"ec2.ModifySubnetAttribute", d.handleEC2ModifySubnetAttribute, "spinifex-workers"},
+		{"ec2.ModifyVpcAttribute", d.handleEC2ModifyVpcAttribute, "spinifex-workers"},
+		{"ec2.DescribeVpcAttribute", d.handleEC2DescribeVpcAttribute, "spinifex-workers"},
 		{"ec2.CreateNetworkInterface", d.handleEC2CreateNetworkInterface, "spinifex-workers"},
 		{"ec2.DeleteNetworkInterface", d.handleEC2DeleteNetworkInterface, "spinifex-workers"},
 		{"ec2.DescribeNetworkInterfaces", d.handleEC2DescribeNetworkInterfaces, "spinifex-workers"},
+		{"ec2.ModifyNetworkInterfaceAttribute", d.handleEC2ModifyNetworkInterfaceAttribute, "spinifex-workers"},
+		{"ec2.CreateSecurityGroup", d.handleEC2CreateSecurityGroup, "spinifex-workers"},
+		{"ec2.DeleteSecurityGroup", d.handleEC2DeleteSecurityGroup, "spinifex-workers"},
+		{"ec2.DescribeSecurityGroups", d.handleEC2DescribeSecurityGroups, "spinifex-workers"},
+		{"ec2.AuthorizeSecurityGroupIngress", d.handleEC2AuthorizeSecurityGroupIngress, "spinifex-workers"},
+		{"ec2.AuthorizeSecurityGroupEgress", d.handleEC2AuthorizeSecurityGroupEgress, "spinifex-workers"},
+		{"ec2.RevokeSecurityGroupIngress", d.handleEC2RevokeSecurityGroupIngress, "spinifex-workers"},
+		{"ec2.RevokeSecurityGroupEgress", d.handleEC2RevokeSecurityGroupEgress, "spinifex-workers"},
 		{"ec2.ModifyInstanceAttribute", d.handleEC2ModifyInstanceAttribute, "spinifex-workers"},
+		{"ec2.DescribeInstanceAttribute", d.handleEC2DescribeInstanceAttribute, "spinifex-workers"},
 		{"ec2.start", d.handleEC2StartStoppedInstance, "spinifex-workers"},
 		{"ec2.terminate", d.handleEC2TerminateStoppedInstance, "spinifex-workers"},
 		{"ec2.DescribeStoppedInstances", d.handleEC2DescribeStoppedInstances, "spinifex-workers"},
@@ -409,10 +455,24 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.GetSerialConsoleAccessStatus", d.handleEC2GetSerialConsoleAccessStatus, "spinifex-workers"},
 		{"ec2.EnableSerialConsoleAccess", d.handleEC2EnableSerialConsoleAccess, "spinifex-workers"},
 		{"ec2.DisableSerialConsoleAccess", d.handleEC2DisableSerialConsoleAccess, "spinifex-workers"},
+		// ELBv2 operations
+		{"elbv2.CreateLoadBalancer", d.handleELBv2CreateLoadBalancer, "spinifex-workers"},
+		{"elbv2.DeleteLoadBalancer", d.handleELBv2DeleteLoadBalancer, "spinifex-workers"},
+		{"elbv2.DescribeLoadBalancers", d.handleELBv2DescribeLoadBalancers, "spinifex-workers"},
+		{"elbv2.CreateTargetGroup", d.handleELBv2CreateTargetGroup, "spinifex-workers"},
+		{"elbv2.DeleteTargetGroup", d.handleELBv2DeleteTargetGroup, "spinifex-workers"},
+		{"elbv2.DescribeTargetGroups", d.handleELBv2DescribeTargetGroups, "spinifex-workers"},
+		{"elbv2.RegisterTargets", d.handleELBv2RegisterTargets, "spinifex-workers"},
+		{"elbv2.DeregisterTargets", d.handleELBv2DeregisterTargets, "spinifex-workers"},
+		{"elbv2.DescribeTargetHealth", d.handleELBv2DescribeTargetHealth, "spinifex-workers"},
+		{"elbv2.CreateListener", d.handleELBv2CreateListener, "spinifex-workers"},
+		{"elbv2.DeleteListener", d.handleELBv2DeleteListener, "spinifex-workers"},
+		{"elbv2.DescribeListeners", d.handleELBv2DescribeListeners, "spinifex-workers"},
 		{fmt.Sprintf("spinifex.admin.%s.health", d.node), d.handleHealthCheck, ""},
 		{"spinifex.nodes.discover", d.handleNodeDiscover, ""},
 		{"spinifex.node.status", d.handleNodeStatus, ""},
 		{"spinifex.node.vms", d.handleNodeVMs, ""},
+		{"spinifex.storage.config", d.handleStorageConfig, ""},
 		// Account creation → create default VPC for new account
 		{"iam.account.created", d.handleAccountCreated, "spinifex-workers"},
 		// Coordinated cluster shutdown phases (fan-out, no queue group)
@@ -423,6 +483,19 @@ func (d *Daemon) subscribeAll() error {
 		{"spinifex.cluster.shutdown.infra", d.handleShutdownInfra, ""},
 	}
 
+	// EIP operations require external IPAM (pool mode). Only subscribe when available;
+	// without a subscriber the gateway returns a NATS timeout → clean error to the client.
+	if d.eipService != nil {
+		subs = append(subs,
+			natsSub{"ec2.AllocateAddress", d.handleEC2AllocateAddress, "spinifex-workers"},
+			natsSub{"ec2.ReleaseAddress", d.handleEC2ReleaseAddress, "spinifex-workers"},
+			natsSub{"ec2.AssociateAddress", d.handleEC2AssociateAddress, "spinifex-workers"},
+			natsSub{"ec2.DisassociateAddress", d.handleEC2DisassociateAddress, "spinifex-workers"},
+			natsSub{"ec2.DescribeAddresses", d.handleEC2DescribeAddresses, "spinifex-workers"},
+			natsSub{"ec2.DescribeAddressesAttribute", d.handleEC2DescribeAddressesAttribute, "spinifex-workers"},
+		)
+	}
+
 	for _, s := range subs {
 		var sub *nats.Subscription
 		var err error
@@ -504,6 +577,13 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize IGW service: %w", err)
 	}
 
+	d.placementGroupService, err = initServiceWithRetry("placement group service", func() (*handlers_ec2_placementgroup.PlacementGroupServiceImpl, error) {
+		return handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize placement group service: %w", err)
+	}
+
 	d.vpcService, err = initServiceWithRetry("VPC service", func() (*handlers_ec2_vpc.VPCServiceImpl, error) {
 		return handlers_ec2_vpc.NewVPCServiceImplWithNATS(d.config, d.natsConn)
 	})
@@ -511,6 +591,67 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize VPC service: %w", err)
 	}
 
+	d.routeTableService, err = initServiceWithRetry("RouteTable service", func() (*handlers_ec2_routetable.RouteTableServiceImpl, error) {
+		return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize RouteTable service: %w", err)
+	}
+
+	d.natGatewayService, err = initServiceWithRetry("NatGateway service", func() (*handlers_ec2_natgw.NatGatewayServiceImpl, error) {
+		return handlers_ec2_natgw.NewNatGatewayServiceImplWithNATS(d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize NatGateway service: %w", err)
+	}
+
+	// Initialize external IPAM if pool mode is configured (per-VM public IPs).
+	// NAT mode only uses SNAT via a shared gateway IP — no per-VM allocation needed.
+	if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode == "pool" {
+		js, jsErr := d.natsConn.JetStream()
+		if jsErr != nil {
+			slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr)
+		} else {
+			var pools []handlers_ec2_vpc.ExternalPoolConfig
+			// Resolve WAN bridge name for DHCP pools
+			wanBridge := ""
+			if node, ok := d.clusterConfig.Nodes[d.clusterConfig.Node]; ok {
+				wanBridge = node.VPCD.WanBridge
+			}
+			for _, p := range d.clusterConfig.Network.ExternalPools {
+				pools = append(pools, handlers_ec2_vpc.ExternalPoolConfig{
+					Name:       p.Name,
+					Source:     p.Source,
+					RangeStart: p.RangeStart,
+					RangeEnd:   p.RangeEnd,
+					Gateway:    p.Gateway,
+					GatewayIP:  p.GatewayIP,
+					PrefixLen:  p.PrefixLen,
+					Region:     p.Region,
+					AZ:         p.AZ,
+					WanBridge:  wanBridge,
+				})
+			}
+			d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools)
+			if err != nil {
+				slog.Warn("Failed to initialize external IPAM", "err", err)
+			} else {
+				slog.Info("External IPAM initialized", "mode", d.clusterConfig.Network.ExternalMode, "pools", len(pools))
+			}
+		}
+	}
+
+	// Initialize EIP service if external IPAM is available
+	if d.externalIPAM != nil && d.vpcService != nil {
+		eipSvc, eipErr := handlers_ec2_eip.NewEIPServiceImpl(d.natsConn, d.externalIPAM, d.vpcService)
+		if eipErr != nil {
+			slog.Warn("Failed to initialize EIP service", "err", eipErr)
+		} else {
+			d.eipService = eipSvc
+			slog.Info("EIP service initialized")
+		}
+	}
+
 	d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) {
 		return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.config, d.natsConn)
 	})
@@ -518,11 +659,64 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize account settings service: %w", err)
 	}
 
-	// Ensure default VPC exists (matches AWS: every account has a default VPC)
+	d.elbv2Service, err = initServiceWithRetry("ELBv2 service", func() (*handlers_elbv2.ELBv2ServiceImpl, error) {
+		return handlers_elbv2.NewELBv2ServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize ELBv2 service: %w", err)
+	}
 	if d.vpcService != nil {
-		if err := d.vpcService.EnsureDefaultVPC(utils.GlobalAccountID); err != nil {
-			slog.Error("Failed to ensure default VPC", "error", err)
+		d.elbv2Service.SetVPCService(d.vpcService)
+	}
+
+	// Wire ALB VM lifecycle: instance launcher + NATS URL for cloud-init.
+	// In dev mode (user-mode networking), VMs reach the host at 10.0.2.2
+	// (QEMU's default gateway). Replace 0.0.0.0 with this address.
+	d.elbv2Service.SetInstanceLauncher(d)
+	if d.config != nil && d.config.NATS.Host != "" {
+		natsHost := d.config.NATS.Host
+		if d.config.Daemon.DevNetworking && strings.HasPrefix(natsHost, "0.0.0.0") {
+			natsHost = strings.Replace(natsHost, "0.0.0.0", "10.0.2.2", 1)
 		}
+		d.elbv2Service.SetNATSURL("nats://" + natsHost)
+	}
+	// Discover system AMI for ALB VMs. Prefer an image with "alb" in the
+	// name (pre-baked Alpine with HAProxy), fall back to first available.
+	if d.imageService != nil {
+		imagesOut, imgErr := d.imageService.DescribeImages(&ec2.DescribeImagesInput{}, utils.GlobalAccountID)
+		if imgErr == nil && len(imagesOut.Images) > 0 {
+			systemAMI := aws.StringValue(imagesOut.Images[0].ImageId)
+			for _, img := range imagesOut.Images {
+				name := aws.StringValue(img.Name)
+				if strings.Contains(name, "alb") {
+					systemAMI = aws.StringValue(img.ImageId)
+					break
+				}
+			}
+			d.elbv2Service.SetSystemAMI(systemAMI)
+			slog.Info("System AMI set for ALB VMs", "amiId", systemAMI)
+		}
+	}
+
+	// Ensure default VPC exists for system and admin accounts
+	// (matches AWS: every account has a default VPC with IGW + default SG)
+	if d.vpcService != nil {
+		for _, accountID := range []string{utils.GlobalAccountID, admin.DefaultAccountID()} {
+			// Pass bootstrap IDs for the admin account so EnsureDefaultVPC uses
+			// the same IDs that admin init wrote to [bootstrap] in spinifex.toml.
+			var opts []handlers_ec2_vpc.BootstrapIDs
+			if accountID == admin.DefaultAccountID() && d.clusterConfig != nil && d.clusterConfig.Bootstrap.VpcId != "" {
+				opts = append(opts, handlers_ec2_vpc.BootstrapIDs{
+					VpcId:    d.clusterConfig.Bootstrap.VpcId,
+					SubnetId: d.clusterConfig.Bootstrap.SubnetId,
+				})
+			}
+			if _, err := d.vpcService.EnsureDefaultVPC(accountID, opts...); err != nil {
+				slog.Error("Failed to ensure default VPC", "accountID", accountID, "error", err)
+			}
+		}
+		// Ensure default VPC has an IGW and default security group
+		d.ensureDefaultVPCInfrastructure()
 	}
 
 	// Initialize network plumber for VPC tap device management
@@ -546,7 +740,7 @@ func (d *Daemon) Start() error {
 	// Initialize dynamic per-instance-type subscriptions for capacity-aware routing.
 	// Each instance type gets its own NATS topic (ec2.RunInstances.{type}) so requests
 	// are only routed to nodes with available capacity.
-	d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances)
+	d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances, d.node)
 
 	d.startHeartbeat()
 	d.startPendingWatchdog()
@@ -1471,8 +1665,11 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 				// Don't return - continue with cleanup
 			}
 
-			// Wait for PID file removal (or check if already gone)
-			err = utils.WaitForPidFileRemoval(instance.ID, 60*time.Second)
+			// Wait for PID file removal (or check if already gone).
+			// 20s is enough for a graceful ACPI shutdown — if the guest hasn't
+			// responded to system_powerdown by then, it won't (e.g. still booting,
+			// no ACPI handler). Force-kill at that point rather than wasting 60s.
+			err = utils.WaitForPidFileRemoval(instance.ID, 20*time.Second)
 			if err != nil {
 				slog.Warn("Timeout waiting for PID file removal", "id", instance.ID, "err", err)
 
@@ -1564,6 +1761,30 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 				}
 			}
 
+			// Release public IP before deleting ENI
+			if deleteVolume && instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
+				// Publish vpc.delete-nat to remove dnat_and_snat rule
+				portName := "port-" + instance.ENIId
+				vpcId := ""
+				logicalIP := ""
+				if instance.Instance != nil {
+					if instance.Instance.VpcId != nil {
+						vpcId = *instance.Instance.VpcId
+					}
+					if instance.Instance.PrivateIpAddress != nil {
+						logicalIP = *instance.Instance.PrivateIpAddress
+					}
+				}
+				d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, logicalIP, portName, "")
+
+				// Release IP back to pool
+				if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
+					slog.Warn("Failed to release public IP on termination", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
+				} else {
+					slog.Info("Released public IP on termination", "ip", instance.PublicIP, "instanceId", instance.ID)
+				}
+			}
+
 			// On termination, delete the auto-created ENI (releases IP back to IPAM,
 			// publishes vpc.delete-port for vpcd). On stop, ENI persists (AWS behavior).
 			if deleteVolume && instance.ENIId != "" && d.vpcService != nil {
@@ -1638,6 +1859,11 @@ func (d *Daemon) setupShutdown() {
 			}
 		}
 
+		// Stop ELBv2 background goroutines
+		if d.elbv2Service != nil {
+			d.elbv2Service.Close()
+		}
+
 		// Final cleanup
 		for _, sub := range d.natsSubscriptions {
 			// Unsubscribe from each subscription
@@ -1989,74 +2215,18 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 	consoleLogPath := filepath.Join(runtimeDir, fmt.Sprintf("console-%s.log", instance.ID))
 	serialSocket := filepath.Join(runtimeDir, fmt.Sprintf("serial-%s.sock", instance.ID))
 
-	instance.Config = vm.Config{
-		Name:           instance.ID,
-		PIDFile:        pidFile,
-		EnableKVM:      true, // If available, if kvm fails, will use cpu max
-		NoGraphic:      true,
-		MachineType:    "q35",
-		ConsoleLogPath: consoleLogPath,
-		SerialSocket:   serialSocket,
-		CPUType:        "host", // If available, if kvm fails, will use cpu max
-		Memory:         int(memoryMiB),
-		CPUCount:       vCPUs,
-		Architecture:   architecture,
-	}
+	instance.Config = buildBaseVMConfig(instance.ID, pidFile, consoleLogPath, serialSocket, architecture, vCPUs, int(memoryMiB))
 
-	// Add PCIe root ports for volume hotplug (Q35 requires explicit root ports).
-	// 11 ports for /dev/sd[f-p] hotplug slots, starting at chassis 1.
-	for i := 1; i <= 11; i++ {
-		instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-			Value: fmt.Sprintf("pcie-root-port,id=hotplug%d,chassis=%d,slot=0", i, i),
-		})
-	}
-
-	// Loop through each volume in volumes
+	// Build QEMU drives from EBS volume requests.
 	instance.EBSRequests.Mu.Lock()
-
-	for _, v := range instance.EBSRequests.Requests {
-		drive := vm.Drive{}
-
-		// Use the NBDURI from mount response - contains socket path or TCP address
-		// NBDURI format: "nbd:unix:/path/to/socket.sock" or "nbd://host:port"
-		if v.NBDURI == "" {
-			slog.Error("NBDURI not set for volume", "volume", v.Name)
-			return fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", v.Name)
-		}
-		drive.File = v.NBDURI
-		slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
-
-		if v.Boot {
-			drive.Format = "raw"
-			drive.If = "none"
-			drive.Media = "disk"
-			drive.ID = "os"
-			drive.Cache = "none"
-
-			iothreadID := "ioth-os"
-			instance.Config.IOThreads = append(instance.Config.IOThreads, vm.IOThread{ID: iothreadID})
-
-			instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-				Value: fmt.Sprintf("virtio-blk-pci,drive=%s,iothread=%s,num-queues=%d,bootindex=1",
-					drive.ID, iothreadID, instance.Config.CPUCount),
-			})
-		}
-
-		if v.CloudInit {
-			drive.Format = "raw"
-			drive.If = "virtio"
-			drive.Media = "cdrom"
-			drive.ID = "cloudinit"
-		}
-
-		// TODO: Add EFI support
-		if v.EFI {
-			continue
-		}
-
-		instance.Config.Drives = append(instance.Config.Drives, drive)
-	}
+	drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, vCPUs)
 	instance.EBSRequests.Mu.Unlock()
+	if err != nil {
+		return err
+	}
+	instance.Config.Drives = append(instance.Config.Drives, drives...)
+	instance.Config.IOThreads = append(instance.Config.IOThreads, iothreads...)
+	instance.Config.Devices = append(instance.Config.Devices, devices...)
 
 	// VPC tap networking vs user-mode fallback
 	if instance.ENIId != "" && d.networkPlumber != nil {
@@ -2090,8 +2260,32 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 					if bindIP == "" || bindIP == "0.0.0.0" {
 						bindIP = "127.0.0.1"
 					}
+					netdevVal := fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort)
+					// Add extra hostfwd rules for system instances (e.g. ALB VMs forwarding HTTP ports)
+					if instance.ExtraHostfwd != nil {
+						for guestPort := range instance.ExtraHostfwd {
+							fwdAddr, fwdErr := viperblock.FindFreePort()
+							if fwdErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to find free port for extra hostfwd", "guestPort", guestPort, "err", fwdErr)
+								continue
+							}
+							_, hostPort, splitErr := net.SplitHostPort(fwdAddr)
+							if splitErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to parse extra hostfwd address", "fwdAddr", fwdAddr, "err", splitErr)
+								continue
+							}
+							hostPortInt, convErr := strconv.Atoi(hostPort)
+							if convErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to convert extra hostfwd port", "hostPort", hostPort, "err", convErr)
+								continue
+							}
+							netdevVal += fmt.Sprintf(",hostfwd=tcp:%s:%s-:%d", bindIP, hostPort, guestPort)
+							instance.ExtraHostfwd[guestPort] = hostPortInt
+							slog.Info("DEV_NETWORKING: extra hostfwd", "guestPort", guestPort, "hostPort", hostPort, "instanceId", instance.ID)
+						}
+					}
 					instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-						Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
+						Value: netdevVal,
 					})
 					devMac := generateDevMAC(instance.ID)
 					instance.Config.Devices = append(instance.Config.Devices, vm.Device{
@@ -2273,6 +2467,83 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 	return nil
 }
 
+// buildBaseVMConfig creates a vm.Config with base QEMU settings and PCIe
+// hotplug root ports. Architecture, vCPU, and memory come from the caller
+// (resolved from instance type info).
+func buildBaseVMConfig(instanceID, pidFile, consoleLogPath, serialSocket, architecture string, vCPUs, memoryMiB int) vm.Config {
+	cfg := vm.Config{
+		Name:           instanceID,
+		PIDFile:        pidFile,
+		EnableKVM:      true,
+		NoGraphic:      true,
+		MachineType:    "q35",
+		ConsoleLogPath: consoleLogPath,
+		SerialSocket:   serialSocket,
+		CPUType:        "host",
+		Memory:         memoryMiB,
+		CPUCount:       vCPUs,
+		Architecture:   architecture,
+	}
+
+	// Add PCIe root ports for volume hotplug (Q35 requires explicit root ports).
+	// 11 ports for /dev/sd[f-p] hotplug slots, starting at chassis 1.
+	for i := 1; i <= 11; i++ {
+		cfg.Devices = append(cfg.Devices, vm.Device{
+			Value: fmt.Sprintf("pcie-root-port,id=hotplug%d,chassis=%d,slot=0", i, i),
+		})
+	}
+
+	return cfg
+}
+
+// buildDrives converts EBS volume requests into QEMU drive, iothread, and device
+// configurations. Returns an error if any non-EFI volume is missing its NBDURI.
+func buildDrives(requests []types.EBSRequest, cpuCount int) ([]vm.Drive, []vm.IOThread, []vm.Device, error) {
+	var drives []vm.Drive
+	var iothreads []vm.IOThread
+	var devices []vm.Device
+
+	for _, v := range requests {
+		// TODO: Add EFI support
+		if v.EFI {
+			continue
+		}
+
+		if v.NBDURI == "" {
+			return nil, nil, nil, fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", v.Name)
+		}
+
+		drive := vm.Drive{File: v.NBDURI}
+
+		if v.Boot {
+			drive.Format = "raw"
+			drive.If = "none"
+			drive.Media = "disk"
+			drive.ID = "os"
+			drive.Cache = "none"
+
+			iothreadID := "ioth-os"
+			iothreads = append(iothreads, vm.IOThread{ID: iothreadID})
+			devices = append(devices, vm.Device{
+				Value: fmt.Sprintf("virtio-blk-pci,drive=%s,iothread=%s,num-queues=%d,bootindex=1",
+					drive.ID, iothreadID, cpuCount),
+			})
+		}
+
+		if v.CloudInit {
+			drive.Format = "raw"
+			drive.If = "virtio"
+			drive.Media = "cdrom"
+			drive.ID = "cloudinit"
+		}
+
+		slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
+		drives = append(drives, drive)
+	}
+
+	return drives, iothreads, devices, nil
+}
+
 // ebsTopic returns a node-specific EBS NATS topic, e.g. "ebs.node1.mount".
 // This ensures mount/unmount requests are routed to the viperblock instance
 // running on the same node as the daemon (NBD sockets are local).
@@ -2464,17 +2735,21 @@ func (rm *ResourceManager) deallocate(instanceType *ec2.InstanceTypeInfo) {
 
 // initSubscriptions sets up dynamic per-instance-type NATS subscriptions.
 // Called once during daemon startup after NATS is connected.
-func (rm *ResourceManager) initSubscriptions(nc *nats.Conn, handler nats.MsgHandler) {
+func (rm *ResourceManager) initSubscriptions(nc *nats.Conn, handler nats.MsgHandler, nodeID string) {
 	rm.natsConn = nc
 	rm.handler = handler
+	rm.nodeID = nodeID
 	rm.instanceSubs = make(map[string]*nats.Subscription)
 	rm.updateInstanceSubscriptions()
 }
 
 // updateInstanceSubscriptions recalculates which instance types can fit on this
 // node and subscribes/unsubscribes from the corresponding NATS topics. Each type
-// gets its own topic (ec2.RunInstances.{type}) with the spinifex-workers queue group,
-// so NATS only routes requests to nodes that can actually serve them.
+// gets two topics:
+//   - ec2.RunInstances.{type} with spinifex-workers queue group (load-balanced, for single-instance launches)
+//   - ec2.RunInstances.{type}.{nodeId} without queue group (targeted, for multi-node distribution)
+//
+// Both use the same handler. NATS only routes requests to nodes with available capacity.
 func (rm *ResourceManager) updateInstanceSubscriptions() {
 	if rm.natsConn == nil {
 		return
@@ -2484,24 +2759,46 @@ func (rm *ResourceManager) updateInstanceSubscriptions() {
 	defer rm.subsMu.Unlock()
 
 	for typeName, typeInfo := range rm.instanceTypes {
-		topic := fmt.Sprintf("ec2.RunInstances.%s", typeName)
+		queueTopic := fmt.Sprintf("ec2.RunInstances.%s", typeName)
 		canFit := rm.canAllocate(typeInfo, 1) >= 1
 
-		_, subscribed := rm.instanceSubs[topic]
+		// Queue group subscription (load-balanced across nodes)
+		_, subscribed := rm.instanceSubs[queueTopic]
 		if canFit && !subscribed {
-			sub, err := rm.natsConn.QueueSubscribe(topic, "spinifex-workers", rm.handler)
+			sub, err := rm.natsConn.QueueSubscribe(queueTopic, "spinifex-workers", rm.handler)
 			if err != nil {
-				slog.Error("Failed to subscribe to instance type topic", "topic", topic, "err", err)
+				slog.Error("Failed to subscribe to instance type topic", "topic", queueTopic, "err", err)
 				continue
 			}
-			rm.instanceSubs[topic] = sub
-			slog.Debug("Subscribed to instance type", "topic", topic)
+			rm.instanceSubs[queueTopic] = sub
+			slog.Debug("Subscribed to instance type", "topic", queueTopic)
 		} else if !canFit && subscribed {
-			if err := rm.instanceSubs[topic].Unsubscribe(); err != nil {
-				slog.Error("Failed to unsubscribe from instance type topic", "topic", topic, "err", err)
+			if err := rm.instanceSubs[queueTopic].Unsubscribe(); err != nil {
+				slog.Error("Failed to unsubscribe from instance type topic", "topic", queueTopic, "err", err)
+			}
+			delete(rm.instanceSubs, queueTopic)
+			slog.Info("Unsubscribed from instance type (capacity full)", "topic", queueTopic)
+		}
+
+		// Node-specific subscription (targeted routing for multi-node distribution)
+		if rm.nodeID != "" {
+			nodeTopic := fmt.Sprintf("ec2.RunInstances.%s.%s", typeName, rm.nodeID)
+			_, nodeSubscribed := rm.instanceSubs[nodeTopic]
+			if canFit && !nodeSubscribed {
+				sub, err := rm.natsConn.Subscribe(nodeTopic, rm.handler)
+				if err != nil {
+					slog.Error("Failed to subscribe to node-specific topic", "topic", nodeTopic, "err", err)
+					continue
+				}
+				rm.instanceSubs[nodeTopic] = sub
+				slog.Debug("Subscribed to node-specific instance type", "topic", nodeTopic)
+			} else if !canFit && nodeSubscribed {
+				if err := rm.instanceSubs[nodeTopic].Unsubscribe(); err != nil {
+					slog.Error("Failed to unsubscribe from node-specific topic", "topic", nodeTopic, "err", err)
+				}
+				delete(rm.instanceSubs, nodeTopic)
+				slog.Info("Unsubscribed from node-specific instance type (capacity full)", "topic", nodeTopic)
 			}
-			delete(rm.instanceSubs, topic)
-			slog.Info("Unsubscribed from instance type (capacity full)", "topic", topic)
 		}
 	}
 }

spinifex/daemon/daemon_handlers.go

@@ -1,10 +1,13 @@
 package daemon
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net"
+	"net/http"
+	"sync"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
@@ -175,9 +178,109 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 		InstanceTypes: caps,
 	}
 
+	// Query service roles concurrently to halve worst-case latency (500ms vs 1s).
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() { defer wg.Done(); resp.NATSRole = d.queryNATSRole() }()
+	go func() { defer wg.Done(); resp.PredastoreRole = d.queryPredastoreRole() }()
+	wg.Wait()
+
 	respondWithJSON(msg, resp)
 }
 
+const (
+	roleLeader   = "leader"
+	roleFollower = "follower"
+
+	natsMonitorPort  = 8222
+	predastoreDBPort = 6660
+)
+
+// queryNATSRole queries the local NATS monitoring endpoint to determine this
+// node's JetStream meta-leader status. Returns "leader", "follower", or "".
+func (d *Daemon) queryNATSRole() string {
+	if !d.config.HasService("nats") {
+		return ""
+	}
+	url := fmt.Sprintf("http://%s:%d/varz", d.daemonIP(), natsMonitorPort)
+	return fetchNATSRole(url, roleHTTPClient)
+}
+
+// queryPredastoreRole queries the local Predastore DB status endpoint to
+// determine this node's Raft role. Returns "leader", "follower", or "".
+func (d *Daemon) queryPredastoreRole() string {
+	if !d.config.HasService("predastore") {
+		return ""
+	}
+	url := fmt.Sprintf("https://%s:%d/status", d.daemonIP(), predastoreDBPort)
+	return fetchPredastoreRole(url, roleTLSHTTPClient)
+}
+
+var roleHTTPClient = &http.Client{Timeout: 500 * time.Millisecond}
+
+var roleTLSHTTPClient = &http.Client{
+	Timeout: 500 * time.Millisecond,
+	Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // local self-signed cert
+	},
+}
+
+// fetchNATSRole queries a NATS /varz endpoint and returns "leader", "follower", or "".
+func fetchNATSRole(url string, client *http.Client) string {
+	resp, err := client.Get(url) //nolint:noctx // internal monitoring call
+	if err != nil {
+		slog.Debug("Failed to query NATS monitoring", "err", err)
+		return ""
+	}
+	defer resp.Body.Close()
+
+	var varz struct {
+		ServerName string `json:"server_name"`
+		JetStream  struct {
+			Meta struct {
+				Leader      string `json:"leader"`
+				ClusterSize int    `json:"cluster_size"`
+			} `json:"meta"`
+		} `json:"jetstream"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&varz); err != nil {
+		slog.Debug("Failed to decode NATS varz", "err", err)
+		return ""
+	}
+
+	// Single node or no meta cluster — this node is the leader by default
+	if varz.JetStream.Meta.ClusterSize <= 1 {
+		return roleLeader
+	}
+	if varz.JetStream.Meta.Leader == varz.ServerName {
+		return roleLeader
+	}
+	return roleFollower
+}
+
+// fetchPredastoreRole queries a Predastore /status endpoint and returns "leader", "follower", or "".
+func fetchPredastoreRole(url string, client *http.Client) string {
+	resp, err := client.Get(url) //nolint:noctx // internal monitoring call
+	if err != nil {
+		slog.Debug("Failed to query Predastore status", "err", err)
+		return ""
+	}
+	defer resp.Body.Close()
+
+	var status struct {
+		IsLeader bool `json:"is_leader"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
+		slog.Debug("Failed to decode Predastore status", "err", err)
+		return ""
+	}
+
+	if status.IsLeader {
+		return roleLeader
+	}
+	return roleFollower
+}
+
 // handleNodeVMs responds with the list of VMs running on this node.
 // Used by the CLI: spx get vms.
 func (d *Daemon) handleNodeVMs(msg *nats.Msg) {

spinifex/daemon/daemon_handlers_eip.go

@@ -0,0 +1,27 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleEC2AllocateAddress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.AllocateAddress)
+}
+
+func (d *Daemon) handleEC2ReleaseAddress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.ReleaseAddress)
+}
+
+func (d *Daemon) handleEC2AssociateAddress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.AssociateAddress)
+}
+
+func (d *Daemon) handleEC2DisassociateAddress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.DisassociateAddress)
+}
+
+func (d *Daemon) handleEC2DescribeAddresses(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.DescribeAddresses)
+}
+
+func (d *Daemon) handleEC2DescribeAddressesAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.DescribeAddressesAttribute)
+}

spinifex/daemon/daemon_handlers_elbv2.go

@@ -0,0 +1,51 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleELBv2CreateLoadBalancer(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateLoadBalancer)
+}
+
+func (d *Daemon) handleELBv2DeleteLoadBalancer(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteLoadBalancer)
+}
+
+func (d *Daemon) handleELBv2DescribeLoadBalancers(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeLoadBalancers)
+}
+
+func (d *Daemon) handleELBv2CreateTargetGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateTargetGroup)
+}
+
+func (d *Daemon) handleELBv2DeleteTargetGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteTargetGroup)
+}
+
+func (d *Daemon) handleELBv2DescribeTargetGroups(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTargetGroups)
+}
+
+func (d *Daemon) handleELBv2RegisterTargets(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.RegisterTargets)
+}
+
+func (d *Daemon) handleELBv2DeregisterTargets(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeregisterTargets)
+}
+
+func (d *Daemon) handleELBv2DescribeTargetHealth(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTargetHealth)
+}
+
+func (d *Daemon) handleELBv2CreateListener(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateListener)
+}
+
+func (d *Daemon) handleELBv2DeleteListener(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteListener)
+}
+
+func (d *Daemon) handleELBv2DescribeListeners(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeListeners)
+}

spinifex/daemon/daemon_handlers_instance.go

@@ -11,6 +11,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/filterutil"
+	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
@@ -156,6 +158,29 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 			continue
 		}
 
+		// When Terraform sets associate_public_ip_address, it sends the subnet
+		// and security groups inside NetworkInterfaces[0] instead of the top-level
+		// fields. Extract them so the rest of the handler works uniformly.
+		if (runInstancesInput.SubnetId == nil || *runInstancesInput.SubnetId == "") &&
+			len(runInstancesInput.NetworkInterfaces) > 0 && runInstancesInput.NetworkInterfaces[0] != nil {
+			nic := runInstancesInput.NetworkInterfaces[0]
+			if nic.SubnetId != nil && *nic.SubnetId != "" {
+				runInstancesInput.SubnetId = nic.SubnetId
+			}
+			if len(runInstancesInput.SecurityGroupIds) == 0 && len(nic.Groups) > 0 {
+				runInstancesInput.SecurityGroupIds = nic.Groups
+			}
+		}
+
+		// Resolve default subnet when none specified (matches AWS behavior)
+		if (runInstancesInput.SubnetId == nil || *runInstancesInput.SubnetId == "") && d.vpcService != nil {
+			defaultSubnet, dsErr := d.vpcService.GetDefaultSubnet(accountID)
+			if dsErr == nil {
+				runInstancesInput.SubnetId = aws.String(defaultSubnet.SubnetId)
+				slog.Info("Resolved default subnet for instance", "instanceId", instance.ID, "subnetId", defaultSubnet.SubnetId)
+			}
+		}
+
 		// Auto-create ENI when SubnetId is provided (matches AWS behavior)
 		if runInstancesInput.SubnetId != nil && *runInstancesInput.SubnetId != "" && d.vpcService != nil {
 			eniOut, eniErr := d.vpcService.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
@@ -172,6 +197,12 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 			eni := eniOut.NetworkInterface
 			instance.ENIId = *eni.NetworkInterfaceId
 			instance.ENIMac = *eni.MacAddress
+
+			// Mark ENI as attached to this instance so attachment.instance-id
+			// filter works (used by ELBv2 RegisterTargets to resolve target IPs).
+			if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+				slog.Error("Failed to attach ENI to instance record — ELBv2 target IP resolution will fail", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
+			}
 			ec2Instance.SetPrivateIpAddress(*eni.PrivateIpAddress)
 			ec2Instance.SetSubnetId(*runInstancesInput.SubnetId)
 			ec2Instance.SetVpcId(*eni.VpcId)
@@ -196,6 +227,41 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 				"privateIp", *eni.PrivateIpAddress,
 				"mac", instance.ENIMac,
 			)
+
+			// Auto-assign public IP if subnet has MapPublicIpOnLaunch and external IPAM is available
+			if d.externalIPAM != nil {
+				subnet, subErr := d.vpcService.GetSubnet(accountID, *runInstancesInput.SubnetId)
+				if subErr == nil && subnet.MapPublicIpOnLaunch {
+					region := ""
+					az := ""
+					if d.config != nil {
+						region = d.config.Region
+						az = d.config.AZ
+					}
+					publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", *eni.NetworkInterfaceId, instance.ID)
+					if allocErr != nil {
+						slog.Warn("Failed to allocate public IP for instance", "instanceId", instance.ID, "err", allocErr)
+					} else {
+						// Update ENI record with public IP
+						if updateErr := d.vpcService.UpdateENIPublicIP(accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil {
+							slog.Warn("Failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr)
+						}
+						// Publish vpc.add-nat for dnat_and_snat rule
+						portName := "port-" + *eni.NetworkInterfaceId
+						d.publishNATEvent("vpc.add-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
+						// Set on ec2Instance response
+						ec2Instance.PublicIpAddress = aws.String(publicIP)
+						instance.PublicIP = publicIP
+						instance.PublicIPPool = poolName
+						slog.Info("Auto-assigned public IP",
+							"instanceId", instance.ID,
+							"publicIp", publicIP,
+							"privateIp", *eni.PrivateIpAddress,
+							"pool", poolName,
+						)
+					}
+				}
+			}
 		}
 
 		instances = append(instances, instance)
@@ -227,10 +293,14 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 	reservation.SetOwnerId(accountID)
 	reservation.Instances = allEC2Instances
 
-	// Store reservation reference and account ID in all VMs
+	// Store reservation reference, account ID, and placement group in all VMs
 	for _, instance := range instances {
 		instance.Reservation = &reservation
 		instance.AccountID = accountID
+		if runInstancesInput.Placement != nil && runInstancesInput.Placement.GroupName != nil && *runInstancesInput.Placement.GroupName != "" {
+			instance.PlacementGroupName = *runInstancesInput.Placement.GroupName
+			instance.PlacementGroupNode = d.node
+		}
 	}
 
 	// Respond to NATS immediately with reservation (instances are provisioning)
@@ -484,6 +554,18 @@ func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command types.EC2I
 			}
 			slog.Info("Instance "+string(finalState), "id", inst.ID)
 
+			// Remove instance from placement group on terminate
+			if isTerminate && inst.PlacementGroupName != "" && d.placementGroupService != nil {
+				if _, pgErr := d.placementGroupService.RemoveInstance(&handlers_ec2_placementgroup.RemoveInstanceInput{
+					GroupName:  inst.PlacementGroupName,
+					NodeName:   inst.PlacementGroupNode,
+					InstanceID: inst.ID,
+				}, inst.AccountID); pgErr != nil {
+					slog.Error("Failed to remove instance from placement group",
+						"instanceId", inst.ID, "groupName", inst.PlacementGroupName, "err", pgErr)
+				}
+			}
+
 			if d.jsManager != nil {
 				if isTerminate {
 					// Write to terminated KV bucket (auto-expires after 1 hour via TTL).
@@ -551,6 +633,89 @@ func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command types.EC2I
 	}(instance, command.Attributes)
 }
 
+// describeInstancesValidFilters defines the set of filter names accepted by DescribeInstances.
+var describeInstancesValidFilters = map[string]bool{
+	"instance-state-name": true,
+	"instance-id":         true,
+	"instance-type":       true,
+	"vpc-id":              true,
+	"subnet-id":           true,
+	"tag-key":             true,
+	"tag-value":           true,
+}
+
+// instanceMatchesFilters checks whether a VM + its built ec2.Instance copy satisfy all parsed filters.
+func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]string) bool {
+	for name, values := range filters {
+		if strings.HasPrefix(name, "tag:") {
+			// tag:Key filters are handled after all field filters.
+			continue
+		}
+
+		var field string
+		switch name {
+		case "instance-state-name":
+			if ic.State != nil && ic.State.Name != nil {
+				field = *ic.State.Name
+			}
+		case "instance-id":
+			field = inst.ID
+		case "instance-type":
+			field = inst.InstanceType
+		case "vpc-id":
+			if ic.VpcId != nil {
+				field = *ic.VpcId
+			}
+		case "subnet-id":
+			if ic.SubnetId != nil {
+				field = *ic.SubnetId
+			}
+		case "tag-key":
+			if !matchTagKey(ic.Tags, values) {
+				return false
+			}
+			continue
+		case "tag-value":
+			if !matchTagValue(ic.Tags, values) {
+				return false
+			}
+			continue
+		default:
+			// Filter name passed ParseFilters but has no case — treat as non-match
+			// to avoid silently ignoring it.
+			return false
+		}
+
+		if !filterutil.MatchesAny(values, field) {
+			return false
+		}
+	}
+
+	// Check tag:Key filters via the instance's Tag slice.
+	tags := filterutil.EC2TagsToMap(ic.Tags)
+	return filterutil.MatchesTags(filters, tags)
+}
+
+// matchTagKey returns true if any tag key on the resource matches any of the filter values.
+func matchTagKey(tags []*ec2.Tag, values []string) bool {
+	for _, t := range tags {
+		if t.Key != nil && filterutil.MatchesAny(values, *t.Key) {
+			return true
+		}
+	}
+	return false
+}
+
+// matchTagValue returns true if any tag value on the resource matches any of the filter values.
+func matchTagValue(tags []*ec2.Tag, values []string) bool {
+	for _, t := range tags {
+		if t.Value != nil && filterutil.MatchesAny(values, *t.Value) {
+			return true
+		}
+	}
+	return false
+}
+
 // handleEC2DescribeInstances processes incoming EC2 DescribeInstances requests
 // This handler responds with instances running on this node owned by the caller's account
 func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
@@ -590,6 +755,14 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 		}
 	}
 
+	// Parse filters (returns error for unknown filter names)
+	parsedFilters, err := filterutil.ParseFilters(describeInstancesInput.Filters, describeInstancesValidFilters)
+	if err != nil {
+		slog.Warn("DescribeInstances: invalid filter", "err", err)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
 	// Group instances by reservation ID (AWS returns instances grouped by reservation)
 	reservationMap := make(map[string]*ec2.Reservation)
 
@@ -628,6 +801,11 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 			instanceCopy := *instance.Instance
 			instanceCopy.State = &ec2.InstanceState{}
 
+			// Populate PublicIpAddress from VM if stored
+			if instance.PublicIP != "" && instanceCopy.PublicIpAddress == nil {
+				instanceCopy.PublicIpAddress = aws.String(instance.PublicIP)
+			}
+
 			// Map internal status to EC2 state codes using the centralized mapping
 			if info, ok := vm.EC2StateCodes[instance.Status]; ok {
 				instanceCopy.State.SetCode(info.Code)
@@ -639,6 +817,19 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 				instanceCopy.State.SetName("pending")
 			}
 
+			// Populate Placement if instance belongs to a placement group
+			if instance.PlacementGroupName != "" {
+				instanceCopy.Placement = &ec2.Placement{
+					GroupName:        aws.String(instance.PlacementGroupName),
+					AvailabilityZone: aws.String(d.config.AZ),
+				}
+			}
+
+			// Apply filters against the fully-built instance copy
+			if len(parsedFilters) > 0 && !instanceMatchesFilters(instance, &instanceCopy, parsedFilters) {
+				continue
+			}
+
 			// Add instance to its reservation
 			reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
 		}
@@ -899,6 +1090,40 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
 	}
 	instance.EBSRequests.Mu.Unlock()
 
+	// Release public IP before termination
+	if instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
+		portName := "port-" + instance.ENIId
+		vpcId := ""
+		logicalIP := ""
+		if instance.Instance != nil {
+			if instance.Instance.VpcId != nil {
+				vpcId = *instance.Instance.VpcId
+			}
+			if instance.Instance.PrivateIpAddress != nil {
+				logicalIP = *instance.Instance.PrivateIpAddress
+			}
+		}
+		d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, logicalIP, portName, "")
+
+		if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
+			slog.Warn("handleEC2TerminateStoppedInstance: failed to release public IP", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
+		} else {
+			slog.Info("handleEC2TerminateStoppedInstance: released public IP", "ip", instance.PublicIP, "instanceId", req.InstanceID)
+		}
+	}
+
+	// Delete ENI if present
+	if instance.ENIId != "" && d.vpcService != nil {
+		_, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
+			NetworkInterfaceId: &instance.ENIId,
+		}, instance.AccountID)
+		if eniErr != nil {
+			slog.Error("handleEC2TerminateStoppedInstance: failed to delete ENI", "eni", instance.ENIId, "err", eniErr)
+		} else {
+			slog.Info("handleEC2TerminateStoppedInstance: deleted ENI", "eni", instance.ENIId, "instanceId", req.InstanceID)
+		}
+	}
+
 	// Write to terminated KV bucket FIRST so the instance is visible in DescribeInstances.
 	// If this fails, the instance remains in the stopped bucket (safe to retry).
 	instance.Status = vm.StateTerminated
@@ -963,6 +1188,13 @@ func (d *Daemon) describeInstancesFromKV(msg *nats.Msg, listFn func() ([]*vm.VM,
 		}
 	}
 
+	parsedFilters, filterErr := filterutil.ParseFilters(describeInput.Filters, describeInstancesValidFilters)
+	if filterErr != nil {
+		slog.Warn(handlerName+": invalid filter", "err", filterErr)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
 	instances, err := listFn()
 	if err != nil {
 		slog.Error(handlerName+": failed to list instances", "err", err)
@@ -1010,6 +1242,10 @@ func (d *Daemon) describeInstancesFromKV(msg *nats.Msg, listFn func() ([]*vm.VM,
 			instanceCopy.State.SetName(fallbackName)
 		}
 
+		if len(parsedFilters) > 0 && !instanceMatchesFilters(instance, &instanceCopy, parsedFilters) {
+			continue
+		}
+
 		reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
 	}
 
@@ -1040,6 +1276,18 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
 
 	instanceID := *input.InstanceId
 
+	// SourceDestCheck is a networking concept that doesn't apply to bare-metal VMs.
+	// Accept the call as a no-op so Terraform and the AWS CLI don't error out.
+	// Unlike InstanceType/UserData, AWS allows this on running instances, so handle
+	// it before the stopped-state gate.
+	if input.SourceDestCheck != nil {
+		slog.Info("handleEC2ModifyInstanceAttribute: accepting SourceDestCheck (no-op on bare metal)", "instanceId", instanceID)
+		if err := msg.Respond([]byte(`{}`)); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
+		return
+	}
+
 	if d.jsManager == nil {
 		slog.Error("handleEC2ModifyInstanceAttribute: JetStream not available")
 		respondWithError(msg, awserrors.ErrorServerInternal)
@@ -1115,3 +1363,142 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
 		slog.Error("Failed to respond to NATS request", "err", err)
 	}
 }
+
+// handleEC2DescribeInstanceAttribute returns a single requested attribute for an instance.
+// It checks running instances first (in-memory), then falls back to stopped instances in KV.
+func (d *Daemon) handleEC2DescribeInstanceAttribute(msg *nats.Msg) {
+	var input ec2.DescribeInstanceAttributeInput
+	if err := json.Unmarshal(msg.Data, &input); err != nil {
+		slog.Error("handleEC2DescribeInstanceAttribute: failed to unmarshal request", "err", err)
+		respondWithError(msg, awserrors.ErrorServerInternal)
+		return
+	}
+
+	if input.InstanceId == nil || *input.InstanceId == "" {
+		slog.Error("handleEC2DescribeInstanceAttribute: missing instance_id")
+		respondWithError(msg, awserrors.ErrorMissingParameter)
+		return
+	}
+	if input.Attribute == nil || *input.Attribute == "" {
+		slog.Error("handleEC2DescribeInstanceAttribute: missing attribute")
+		respondWithError(msg, awserrors.ErrorMissingParameter)
+		return
+	}
+
+	instanceID := *input.InstanceId
+	attribute := *input.Attribute
+	accountID := utils.AccountIDFromMsg(msg)
+
+	// Look up instance: running first, then stopped KV.
+	var instance *vm.VM
+
+	d.Instances.Mu.Lock()
+	if running, ok := d.Instances.VMS[instanceID]; ok {
+		instance = running
+	}
+	d.Instances.Mu.Unlock()
+
+	if instance == nil {
+		if d.jsManager == nil {
+			slog.Error("handleEC2DescribeInstanceAttribute: JetStream not available")
+			respondWithError(msg, awserrors.ErrorServerInternal)
+			return
+		}
+		stopped, err := d.jsManager.LoadStoppedInstance(instanceID)
+		if err != nil {
+			slog.Error("handleEC2DescribeInstanceAttribute: failed to load stopped instance",
+				"instanceId", instanceID, "err", err)
+			respondWithError(msg, awserrors.ErrorServerInternal)
+			return
+		}
+		instance = stopped
+	}
+
+	if instance == nil {
+		slog.Warn("handleEC2DescribeInstanceAttribute: instance not found",
+			"instanceId", instanceID)
+		respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
+		return
+	}
+
+	if !checkInstanceOwnership(msg, instanceID, instance.AccountID) {
+		return
+	}
+
+	output := &ec2.DescribeInstanceAttributeOutput{
+		InstanceId: &instanceID,
+	}
+
+	switch attribute {
+	case ec2.InstanceAttributeNameInstanceType:
+		val := instance.InstanceType
+		output.InstanceType = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameUserData:
+		val := instance.UserData
+		output.UserData = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameDisableApiTermination:
+		val := false
+		output.DisableApiTermination = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameDisableApiStop:
+		val := false
+		output.DisableApiStop = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameInstanceInitiatedShutdownBehavior:
+		val := ec2.ShutdownBehaviorStop
+		output.InstanceInitiatedShutdownBehavior = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameEbsOptimized:
+		val := false
+		output.EbsOptimized = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameEnaSupport:
+		val := true
+		output.EnaSupport = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameSourceDestCheck:
+		val := true
+		output.SourceDestCheck = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameGroupSet:
+		if instance.Instance != nil && len(instance.Instance.SecurityGroups) > 0 {
+			output.Groups = instance.Instance.SecurityGroups
+		} else {
+			output.Groups = []*ec2.GroupIdentifier{}
+		}
+
+	default:
+		slog.Warn("handleEC2DescribeInstanceAttribute: unsupported attribute",
+			"instanceId", instanceID, "attribute", attribute)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
+	slog.Info("handleEC2DescribeInstanceAttribute: completed",
+		"instanceId", instanceID, "attribute", attribute, "accountID", accountID)
+	respondWithJSON(msg, output)
+}
+
+// publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
+func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) {
+	if d.natsConn == nil {
+		return
+	}
+	evt := struct {
+		VpcId      string `json:"vpc_id"`
+		ExternalIP string `json:"external_ip"`
+		LogicalIP  string `json:"logical_ip"`
+		PortName   string `json:"port_name"`
+		MAC        string `json:"mac"`
+	}{VpcId: vpcId, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac}
+	data, err := json.Marshal(evt)
+	if err != nil {
+		slog.Error("Failed to marshal NAT event", "topic", topic, "err", err)
+		return
+	}
+	if err := d.natsConn.Publish(topic, data); err != nil {
+		slog.Error("Failed to publish NAT event", "topic", topic, "err", err)
+	}
+}

spinifex/daemon/daemon_handlers_natgw.go

@@ -0,0 +1,17 @@
+package daemon
+
+import (
+	"github.com/nats-io/nats.go"
+)
+
+func (d *Daemon) handleEC2CreateNatGateway(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.CreateNatGateway)
+}
+
+func (d *Daemon) handleEC2DeleteNatGateway(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.DeleteNatGateway)
+}
+
+func (d *Daemon) handleEC2DescribeNatGateways(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.DescribeNatGateways)
+}

spinifex/daemon/daemon_handlers_placementgroup.go

@@ -0,0 +1,39 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleEC2CreatePlacementGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.CreatePlacementGroup)
+}
+
+func (d *Daemon) handleEC2DeletePlacementGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.DeletePlacementGroup)
+}
+
+func (d *Daemon) handleEC2DescribePlacementGroups(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.DescribePlacementGroups)
+}
+
+func (d *Daemon) handleEC2ReserveSpreadNodes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.ReserveSpreadNodes)
+}
+
+func (d *Daemon) handleEC2FinalizeSpreadInstances(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.FinalizeSpreadInstances)
+}
+
+func (d *Daemon) handleEC2ReleaseSpreadNodes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.ReleaseSpreadNodes)
+}
+
+func (d *Daemon) handleEC2RemoveInstanceFromPlacementGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.RemoveInstance)
+}
+
+func (d *Daemon) handleEC2ReserveClusterNode(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.ReserveClusterNode)
+}
+
+func (d *Daemon) handleEC2FinalizeClusterInstances(msg *nats.Msg) {
+	handleNATSRequest(msg, d.placementGroupService.FinalizeClusterInstances)
+}

spinifex/daemon/daemon_handlers_routetable.go

@@ -0,0 +1,41 @@
+package daemon
+
+import (
+	"github.com/nats-io/nats.go"
+)
+
+func (d *Daemon) handleEC2CreateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.CreateRouteTable)
+}
+
+func (d *Daemon) handleEC2DeleteRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DeleteRouteTable)
+}
+
+func (d *Daemon) handleEC2DescribeRouteTables(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DescribeRouteTables)
+}
+
+func (d *Daemon) handleEC2CreateRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.CreateRoute)
+}
+
+func (d *Daemon) handleEC2DeleteRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DeleteRoute)
+}
+
+func (d *Daemon) handleEC2ReplaceRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.ReplaceRoute)
+}
+
+func (d *Daemon) handleEC2AssociateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.AssociateRouteTable)
+}
+
+func (d *Daemon) handleEC2DisassociateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DisassociateRouteTable)
+}
+
+func (d *Daemon) handleEC2ReplaceRouteTableAssociation(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.ReplaceRouteTableAssociation)
+}

spinifex/daemon/daemon_handlers_sg.go

@@ -0,0 +1,31 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleEC2CreateSecurityGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.CreateSecurityGroup)
+}
+
+func (d *Daemon) handleEC2DeleteSecurityGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.DeleteSecurityGroup)
+}
+
+func (d *Daemon) handleEC2DescribeSecurityGroups(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.DescribeSecurityGroups)
+}
+
+func (d *Daemon) handleEC2AuthorizeSecurityGroupIngress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.AuthorizeSecurityGroupIngress)
+}
+
+func (d *Daemon) handleEC2AuthorizeSecurityGroupEgress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.AuthorizeSecurityGroupEgress)
+}
+
+func (d *Daemon) handleEC2RevokeSecurityGroupIngress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.RevokeSecurityGroupIngress)
+}
+
+func (d *Daemon) handleEC2RevokeSecurityGroupEgress(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.RevokeSecurityGroupEgress)
+}

spinifex/daemon/daemon_handlers_storage.go

@@ -0,0 +1,106 @@
+package daemon
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/nats-io/nats.go"
+	toml "github.com/pelletier/go-toml/v2"
+)
+
+// predastoreTOML is a minimal representation of the predastore config file,
+// containing only the fields needed for storage metrics (no credentials).
+type predastoreTOML struct {
+	RS      predastoreRS       `toml:"rs"`
+	DB      []predastoreDBNode `toml:"db"`
+	Nodes   []predastoreNode   `toml:"nodes"`
+	Buckets []predastoreBucket `toml:"buckets"`
+}
+
+type predastoreRS struct {
+	Data   int `toml:"data"`
+	Parity int `toml:"parity"`
+}
+
+type predastoreDBNode struct {
+	ID   int    `toml:"id"`
+	Host string `toml:"host"`
+	Port int    `toml:"port"`
+}
+
+type predastoreNode struct {
+	ID   int    `toml:"id"`
+	Host string `toml:"host"`
+	Port int    `toml:"port"`
+}
+
+type predastoreBucket struct {
+	Name   string `toml:"name"`
+	Type   string `toml:"type"`
+	Region string `toml:"region"`
+}
+
+// handleStorageConfig responds with parsed predastore config (topology only, no creds).
+// Used by the gateway to build the GetStorageStatus response.
+func (d *Daemon) handleStorageConfig(msg *nats.Msg) {
+	configDir := filepath.Dir(d.configPath)
+	predastorePath := filepath.Join(configDir, "predastore", "predastore.toml")
+
+	data, err := os.ReadFile(predastorePath)
+	if err != nil {
+		slog.Debug("handleStorageConfig: failed to read predastore config", "path", predastorePath, "err", err)
+		respondWithError(msg, "InternalError")
+		return
+	}
+
+	var cfg predastoreTOML
+	if err := toml.Unmarshal(data, &cfg); err != nil {
+		slog.Error("handleStorageConfig: failed to parse predastore config", "err", err)
+		respondWithError(msg, "InternalError")
+		return
+	}
+
+	resp := types.StorageConfigResponse{
+		Encoding: types.StorageEncoding{
+			DataShards:   cfg.RS.Data,
+			ParityShards: cfg.RS.Parity,
+		},
+	}
+
+	for _, db := range cfg.DB {
+		resp.DBNodes = append(resp.DBNodes, types.StorageDBNode{
+			ID:   db.ID,
+			Host: db.Host,
+			Port: db.Port,
+		})
+	}
+	if resp.DBNodes == nil {
+		resp.DBNodes = []types.StorageDBNode{}
+	}
+
+	for _, n := range cfg.Nodes {
+		resp.ShardNodes = append(resp.ShardNodes, types.StorageShardNode{
+			ID:   n.ID,
+			Host: n.Host,
+			Port: n.Port,
+		})
+	}
+	if resp.ShardNodes == nil {
+		resp.ShardNodes = []types.StorageShardNode{}
+	}
+
+	for _, b := range cfg.Buckets {
+		resp.Buckets = append(resp.Buckets, types.StorageBucket{
+			Name:   b.Name,
+			Type:   b.Type,
+			Region: b.Region,
+		})
+	}
+	if resp.Buckets == nil {
+		resp.Buckets = []types.StorageBucket{}
+	}
+
+	respondWithJSON(msg, resp)
+}

spinifex/daemon/daemon_handlers_test.go

@@ -16,10 +16,13 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
 	handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
+	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
 	handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw"
 	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
+	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
@@ -1733,6 +1736,27 @@ func TestHandleEC2ModifyInstanceAttribute_ChangeUserData(t *testing.T) {
 	assert.Equal(t, "IyEvYmluL2Jhc2g=", *updated.RunInstancesInput.UserData)
 }
 
+func TestHandleEC2ModifyInstanceAttribute_SourceDestCheck(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.ModifyInstanceAttribute", "spinifex-workers", daemon.handleEC2ModifyInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	// SourceDestCheck is a no-op that succeeds without requiring a stopped instance
+	// in KV — Terraform sends this on running instances right after creation.
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:      aws.String("i-modify-sdc-001"),
+		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.ModifyInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, `{}`, string(reply.Data))
+}
+
 func TestHandleEC2ModifyInstanceAttribute_InstanceNotFound(t *testing.T) {
 	natsURL := sharedJSNATSURL
 
@@ -1918,6 +1942,373 @@ func TestHandleEC2ModifyInstanceAttribute_InvalidJSON(t *testing.T) {
 	assert.Equal(t, "ServerInternal", errResp["Code"])
 }
 
+// --- DescribeInstanceAttribute daemon tests ---
+
+func TestHandleEC2DescribeInstanceAttribute_RunningInstance_InstanceType(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-run-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String(instanceID),
+			InstanceType: aws.String("t3.micro"),
+		},
+	}
+
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instanceID] = instance
+	daemon.Instances.Mu.Unlock()
+	t.Cleanup(func() {
+		daemon.Instances.Mu.Lock()
+		delete(daemon.Instances.VMS, instanceID)
+		daemon.Instances.Mu.Unlock()
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.InstanceType)
+	assert.Equal(t, "t3.micro", *output.InstanceType.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_StoppedInstance_InstanceType(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-stop-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.medium",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String(instanceID),
+			InstanceType: aws.String("t3.medium"),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.InstanceType)
+	assert.Equal(t, "t3.medium", *output.InstanceType.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_UserData(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-ud-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		UserData:     "#!/bin/bash",
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameUserData),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.UserData)
+	assert.Equal(t, "#!/bin/bash", *output.UserData.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTermination(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-def-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameDisableApiTermination),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.DisableApiTermination)
+	assert.Equal(t, false, *output.DisableApiTermination.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_ShutdownBehavior(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-shut-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceInitiatedShutdownBehavior),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.InstanceInitiatedShutdownBehavior)
+	assert.Equal(t, "stop", *output.InstanceInitiatedShutdownBehavior.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_GroupSet_WithSecurityGroups(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-gs-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+			SecurityGroups: []*ec2.GroupIdentifier{
+				{GroupId: aws.String("sg-111"), GroupName: aws.String("default")},
+				{GroupId: aws.String("sg-222"), GroupName: aws.String("web")},
+			},
+		},
+	}
+
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instanceID] = instance
+	daemon.Instances.Mu.Unlock()
+	t.Cleanup(func() {
+		daemon.Instances.Mu.Lock()
+		delete(daemon.Instances.VMS, instanceID)
+		daemon.Instances.Mu.Unlock()
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameGroupSet),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.Len(t, output.Groups, 2)
+	assert.Equal(t, "sg-111", *output.Groups[0].GroupId)
+	assert.Equal(t, "sg-222", *output.Groups[1].GroupId)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_GroupSet_NilInstance(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-gs-nil-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance:     nil,
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameGroupSet),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.Groups, "Groups should be empty slice, not nil")
+	assert.Empty(t, output.Groups)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_InstanceNotFound(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-nonexistent"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "InvalidInstanceID.NotFound", errResp["Code"])
+}
+
+func TestHandleEC2DescribeInstanceAttribute_UnsupportedAttribute(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-unsup-001"
+	instance := &vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateStopped,
+		AccountID: testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameBlockDeviceMapping),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "InvalidParameterValue", errResp["Code"])
+}
+
+func TestHandleEC2DescribeInstanceAttribute_InvalidJSON(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := daemon.natsConn.Request("ec2.DescribeInstanceAttribute", []byte(`{invalid`), 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "ServerInternal", errResp["Code"])
+}
+
 // --- Delegate handler round-trip tests (table-driven) ---
 // Each of these handlers is a single line delegating to handleNATSRequest.
 // This test verifies the wiring is correct by sending a NATS request and
@@ -3482,3 +3873,505 @@ func TestHandleEC2TerminateStoppedInstance_WritesToTerminatedKV(t *testing.T) {
 
 	_ = daemon.jsManager.DeleteTerminatedInstance(stoppedVM.ID)
 }
+
+// --- Bead 5: EIP daemon handler tests ---
+
+func TestDelegateHandlers_EIP(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+	ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []handlers_ec2_vpc.ExternalPoolConfig{
+		{Name: "test-pool", RangeStart: "192.168.100.2", RangeEnd: "192.168.100.254", Gateway: "192.168.100.1", PrefixLen: 24},
+	})
+	require.NoError(t, err)
+
+	eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(nc, ipam, daemon.vpcService)
+	require.NoError(t, err)
+	daemon.eipService = eipSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"AllocateAddress",
+			"ec2.test.AllocateAddress",
+			daemon.handleEC2AllocateAddress,
+			&ec2.AllocateAddressInput{},
+		},
+		{
+			"ReleaseAddress",
+			"ec2.test.ReleaseAddress",
+			daemon.handleEC2ReleaseAddress,
+			&ec2.ReleaseAddressInput{AllocationId: aws.String("eipalloc-nonexistent")},
+		},
+		{
+			"AssociateAddress",
+			"ec2.test.AssociateAddress",
+			daemon.handleEC2AssociateAddress,
+			&ec2.AssociateAddressInput{AllocationId: aws.String("eipalloc-nonexistent")},
+		},
+		{
+			"DisassociateAddress",
+			"ec2.test.DisassociateAddress",
+			daemon.handleEC2DisassociateAddress,
+			&ec2.DisassociateAddressInput{AssociationId: aws.String("eipassoc-nonexistent")},
+		},
+		{
+			"DescribeAddresses",
+			"ec2.test.DescribeAddresses",
+			daemon.handleEC2DescribeAddresses,
+			&ec2.DescribeAddressesInput{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "EIP response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 6: Security Group daemon handler tests ---
+
+func TestDelegateHandlers_SecurityGroup(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Create a VPC first so SG operations have a valid target
+	createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", daemon.handleEC2CreateVpc)
+	require.NoError(t, err)
+	defer createVpcSub.Unsubscribe()
+
+	vpcInput := &ec2.CreateVpcInput{CidrBlock: aws.String("10.50.0.0/16")}
+	reqData, _ := json.Marshal(vpcInput)
+	reply, err := natsRequest(daemon.natsConn, "ec2.CreateVpc", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var vpcOut ec2.CreateVpcOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &vpcOut))
+	vpcID := *vpcOut.Vpc.VpcId
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreateSecurityGroup",
+			"ec2.test.CreateSecurityGroup",
+			daemon.handleEC2CreateSecurityGroup,
+			&ec2.CreateSecurityGroupInput{
+				GroupName:   aws.String("test-sg"),
+				Description: aws.String("test security group"),
+				VpcId:       aws.String(vpcID),
+			},
+		},
+		{
+			"DescribeSecurityGroups",
+			"ec2.test.DescribeSecurityGroups",
+			daemon.handleEC2DescribeSecurityGroups,
+			&ec2.DescribeSecurityGroupsInput{},
+		},
+		{
+			"AuthorizeSecurityGroupIngress",
+			"ec2.test.AuthorizeSecurityGroupIngress",
+			daemon.handleEC2AuthorizeSecurityGroupIngress,
+			&ec2.AuthorizeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"AuthorizeSecurityGroupEgress",
+			"ec2.test.AuthorizeSecurityGroupEgress",
+			daemon.handleEC2AuthorizeSecurityGroupEgress,
+			&ec2.AuthorizeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"RevokeSecurityGroupIngress",
+			"ec2.test.RevokeSecurityGroupIngress",
+			daemon.handleEC2RevokeSecurityGroupIngress,
+			&ec2.RevokeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"RevokeSecurityGroupEgress",
+			"ec2.test.RevokeSecurityGroupEgress",
+			daemon.handleEC2RevokeSecurityGroupEgress,
+			&ec2.RevokeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"DeleteSecurityGroup",
+			"ec2.test.DeleteSecurityGroup",
+			daemon.handleEC2DeleteSecurityGroup,
+			&ec2.DeleteSecurityGroupInput{GroupId: aws.String("sg-nonexistent")},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "SG response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 7: Route Table daemon handler tests ---
+
+func TestDelegateHandlers_RouteTable(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Route table service needs its own JetStream for KV buckets
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(daemon.config, nc)
+	require.NoError(t, err)
+	daemon.routeTableService = rtbSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreateRouteTable",
+			"ec2.test.CreateRouteTable",
+			daemon.handleEC2CreateRouteTable,
+			&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-nonexistent")},
+		},
+		{
+			"DeleteRouteTable",
+			"ec2.test.DeleteRouteTable",
+			daemon.handleEC2DeleteRouteTable,
+			&ec2.DeleteRouteTableInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DescribeRouteTables",
+			"ec2.test.DescribeRouteTables",
+			daemon.handleEC2DescribeRouteTables,
+			&ec2.DescribeRouteTablesInput{},
+		},
+		{
+			"CreateRoute",
+			"ec2.test.CreateRoute",
+			daemon.handleEC2CreateRoute,
+			&ec2.CreateRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DeleteRoute",
+			"ec2.test.DeleteRoute",
+			daemon.handleEC2DeleteRoute,
+			&ec2.DeleteRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"ReplaceRoute",
+			"ec2.test.ReplaceRoute",
+			daemon.handleEC2ReplaceRoute,
+			&ec2.ReplaceRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"AssociateRouteTable",
+			"ec2.test.AssociateRouteTable",
+			daemon.handleEC2AssociateRouteTable,
+			&ec2.AssociateRouteTableInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DisassociateRouteTable",
+			"ec2.test.DisassociateRouteTable",
+			daemon.handleEC2DisassociateRouteTable,
+			&ec2.DisassociateRouteTableInput{AssociationId: aws.String("rtbassoc-nonexistent")},
+		},
+		{
+			"ReplaceRouteTableAssociation",
+			"ec2.test.ReplaceRouteTableAssociation",
+			daemon.handleEC2ReplaceRouteTableAssociation,
+			&ec2.ReplaceRouteTableAssociationInput{
+				AssociationId: aws.String("rtbassoc-nonexistent"),
+				RouteTableId:  aws.String("rtb-nonexistent"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "RouteTable response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 8: Placement Group daemon handler tests ---
+
+func TestDelegateHandlers_PlacementGroup(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	pgSvc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(daemon.config, nc)
+	require.NoError(t, err)
+	daemon.placementGroupService = pgSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreatePlacementGroup",
+			"ec2.test.CreatePlacementGroup",
+			daemon.handleEC2CreatePlacementGroup,
+			&ec2.CreatePlacementGroupInput{
+				GroupName: aws.String("test-pg"),
+				Strategy:  aws.String("spread"),
+			},
+		},
+		{
+			"DescribePlacementGroups",
+			"ec2.test.DescribePlacementGroups",
+			daemon.handleEC2DescribePlacementGroups,
+			&ec2.DescribePlacementGroupsInput{},
+		},
+		{
+			"DeletePlacementGroup",
+			"ec2.test.DeletePlacementGroup",
+			daemon.handleEC2DeletePlacementGroup,
+			&ec2.DeletePlacementGroupInput{GroupName: aws.String("pg-nonexistent")},
+		},
+		{
+			"ReserveSpreadNodes",
+			"ec2.test.ReserveSpreadNodes",
+			daemon.handleEC2ReserveSpreadNodes,
+			&handlers_ec2_placementgroup.ReserveSpreadNodesInput{
+				GroupName:     "pg-nonexistent",
+				EligibleNodes: []string{"node-1"},
+				MinCount:      1,
+				MaxCount:      1,
+			},
+		},
+		{
+			"FinalizeSpreadInstances",
+			"ec2.test.FinalizeSpreadInstances",
+			daemon.handleEC2FinalizeSpreadInstances,
+			&handlers_ec2_placementgroup.FinalizeSpreadInstancesInput{
+				GroupName:     "pg-nonexistent",
+				NodeInstances: map[string][]string{"node-1": {"i-123"}},
+			},
+		},
+		{
+			"ReleaseSpreadNodes",
+			"ec2.test.ReleaseSpreadNodes",
+			daemon.handleEC2ReleaseSpreadNodes,
+			&handlers_ec2_placementgroup.ReleaseSpreadNodesInput{
+				GroupName: "pg-nonexistent",
+				Nodes:     []string{"node-1"},
+			},
+		},
+		{
+			"RemoveInstanceFromPlacementGroup",
+			"ec2.test.RemoveInstanceFromPlacementGroup",
+			daemon.handleEC2RemoveInstanceFromPlacementGroup,
+			&handlers_ec2_placementgroup.RemoveInstanceInput{
+				GroupName:  "pg-nonexistent",
+				NodeName:   "node-1",
+				InstanceID: "i-123",
+			},
+		},
+		{
+			"ReserveClusterNode",
+			"ec2.test.ReserveClusterNode",
+			daemon.handleEC2ReserveClusterNode,
+			&handlers_ec2_placementgroup.ReserveClusterNodeInput{
+				GroupName:     "pg-nonexistent",
+				EligibleNodes: []string{"node-1"},
+			},
+		},
+		{
+			"FinalizeClusterInstances",
+			"ec2.test.FinalizeClusterInstances",
+			daemon.handleEC2FinalizeClusterInstances,
+			&handlers_ec2_placementgroup.FinalizeClusterInstancesInput{
+				GroupName:     "pg-nonexistent",
+				NodeInstances: map[string][]string{"node-1": {"i-123"}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "PlacementGroup response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 9: VPC attribute daemon handler tests (untested handlers) ---
+
+func TestDelegateHandlers_VPCAttributes(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Create a VPC first
+	createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", daemon.handleEC2CreateVpc)
+	require.NoError(t, err)
+	defer createVpcSub.Unsubscribe()
+
+	vpcInput := &ec2.CreateVpcInput{CidrBlock: aws.String("10.60.0.0/16")}
+	reqData, _ := json.Marshal(vpcInput)
+	reply, err := natsRequest(daemon.natsConn, "ec2.CreateVpc", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var vpcOut ec2.CreateVpcOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &vpcOut))
+	vpcID := *vpcOut.Vpc.VpcId
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"ModifySubnetAttribute",
+			"ec2.test.ModifySubnetAttribute",
+			daemon.handleEC2ModifySubnetAttribute,
+			&ec2.ModifySubnetAttributeInput{SubnetId: aws.String("subnet-nonexistent")},
+		},
+		{
+			"ModifyVpcAttribute",
+			"ec2.test.ModifyVpcAttribute",
+			daemon.handleEC2ModifyVpcAttribute,
+			&ec2.ModifyVpcAttributeInput{VpcId: aws.String(vpcID)},
+		},
+		{
+			"DescribeVpcAttribute",
+			"ec2.test.DescribeVpcAttribute",
+			daemon.handleEC2DescribeVpcAttribute,
+			&ec2.DescribeVpcAttributeInput{
+				VpcId:     aws.String(vpcID),
+				Attribute: aws.String("enableDnsSupport"),
+			},
+		},
+		{
+			"ModifyNetworkInterfaceAttribute",
+			"ec2.test.ModifyNetworkInterfaceAttribute",
+			daemon.handleEC2ModifyNetworkInterfaceAttribute,
+			&ec2.ModifyNetworkInterfaceAttributeInput{NetworkInterfaceId: aws.String("eni-nonexistent")},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "VPC attribute response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}

spinifex/daemon/daemon_handlers_vpc.go

@@ -4,6 +4,9 @@ import (
 	"encoding/json"
 	"log/slog"
 
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 )
 
@@ -31,6 +34,18 @@ func (d *Daemon) handleEC2DescribeSubnets(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DescribeSubnets)
 }
 
+func (d *Daemon) handleEC2ModifySubnetAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.ModifySubnetAttribute)
+}
+
+func (d *Daemon) handleEC2ModifyVpcAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.ModifyVpcAttribute)
+}
+
+func (d *Daemon) handleEC2DescribeVpcAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.DescribeVpcAttribute)
+}
+
 func (d *Daemon) handleEC2CreateNetworkInterface(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.CreateNetworkInterface)
 }
@@ -39,6 +54,10 @@ func (d *Daemon) handleEC2DeleteNetworkInterface(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DeleteNetworkInterface)
 }
 
+func (d *Daemon) handleEC2ModifyNetworkInterfaceAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.ModifyNetworkInterfaceAttribute)
+}
+
 func (d *Daemon) handleEC2DescribeNetworkInterfaces(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DescribeNetworkInterfaces)
 }
@@ -56,8 +75,197 @@ func (d *Daemon) handleAccountCreated(msg *nats.Msg) {
 		slog.Error("Account creation event has empty account ID")
 		return
 	}
-	if err := d.vpcService.EnsureDefaultVPC(evt.AccountID); err != nil {
+	if _, err := d.vpcService.EnsureDefaultVPC(evt.AccountID); err != nil {
 		slog.Error("Failed to create default VPC for new account",
 			"accountID", evt.AccountID, "error", err)
 	}
+	d.ensureDefaultVPCInfrastructure()
+}
+
+// ensureDefaultVPCInfrastructure creates an IGW and default security group for
+// each default VPC that doesn't already have them. Matches AWS behavior where
+// the default VPC comes with an attached IGW and a default security group.
+func (d *Daemon) ensureDefaultVPCInfrastructure() {
+	if d.igwService == nil || d.vpcService == nil {
+		return
+	}
+
+	for _, accountID := range []string{utils.GlobalAccountID, admin.DefaultAccountID()} {
+		// Find the default VPC for this account
+		descOut, err := d.vpcService.DescribeVpcs(&ec2.DescribeVpcsInput{}, accountID)
+		if err != nil {
+			continue
+		}
+		var defaultVpcId string
+		for _, vpc := range descOut.Vpcs {
+			if vpc.IsDefault != nil && *vpc.IsDefault {
+				defaultVpcId = *vpc.VpcId
+				break
+			}
+		}
+		if defaultVpcId == "" {
+			continue
+		}
+
+		// Check if IGW already attached
+		igwOut, err := d.igwService.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{}, accountID)
+		if err != nil {
+			continue
+		}
+		hasIGW := false
+		for _, igw := range igwOut.InternetGateways {
+			for _, att := range igw.Attachments {
+				if att.VpcId != nil && *att.VpcId == defaultVpcId {
+					hasIGW = true
+					break
+				}
+			}
+		}
+		if !hasIGW {
+			// Create and attach an IGW — use bootstrap ID if available
+			var createOut *ec2.CreateInternetGatewayOutput
+			var err error
+			if accountID == admin.DefaultAccountID() && d.clusterConfig != nil && d.clusterConfig.Bootstrap.IgwId != "" {
+				createOut, err = d.igwService.CreateInternetGatewayWithID(&ec2.CreateInternetGatewayInput{}, accountID, d.clusterConfig.Bootstrap.IgwId)
+			} else {
+				createOut, err = d.igwService.CreateInternetGateway(&ec2.CreateInternetGatewayInput{}, accountID)
+			}
+			if err != nil {
+				slog.Error("Failed to create default IGW", "accountID", accountID, "err", err)
+				continue
+			}
+			igwId := *createOut.InternetGateway.InternetGatewayId
+			_, err = d.igwService.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
+				InternetGatewayId: &igwId,
+				VpcId:             &defaultVpcId,
+			}, accountID)
+			if err != nil {
+				slog.Error("Failed to attach default IGW", "igwId", igwId, "vpcId", defaultVpcId, "err", err)
+			} else {
+				slog.Info("Attached default IGW to default VPC", "igwId", igwId, "vpcId", defaultVpcId, "accountID", accountID)
+			}
+		}
+
+		// Add 0.0.0.0/0 → IGW route to the main route table (if not already present)
+		if d.routeTableService != nil {
+			d.ensureDefaultIGWRoute(accountID, defaultVpcId)
+		}
+
+		// Check if default security group exists for this VPC
+		sgOut, err := d.vpcService.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{}, accountID)
+		if err != nil {
+			continue
+		}
+		hasDefaultSG := false
+		for _, sg := range sgOut.SecurityGroups {
+			if sg.VpcId != nil && *sg.VpcId == defaultVpcId && sg.GroupName != nil && *sg.GroupName == "default" {
+				hasDefaultSG = true
+				break
+			}
+		}
+		if !hasDefaultSG {
+			desc := "default VPC security group"
+			groupName := "default"
+			createSGOut, err := d.vpcService.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
+				GroupName:   &groupName,
+				Description: &desc,
+				VpcId:       &defaultVpcId,
+			}, accountID)
+			if err != nil {
+				slog.Error("Failed to create default security group", "vpcId", defaultVpcId, "err", err)
+				continue
+			}
+			sgId := *createSGOut.GroupId
+
+			// Default SG rules (AWS behavior):
+			// - Allow all inbound from same SG (self-referencing)
+			// - Allow all outbound to 0.0.0.0/0
+			allProto := "-1"
+			allCidr := "0.0.0.0/0"
+			_, _ = d.vpcService.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
+				GroupId: &sgId,
+				IpPermissions: []*ec2.IpPermission{
+					{
+						IpProtocol:       &allProto,
+						UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: &sgId}},
+					},
+				},
+			}, accountID)
+			_, _ = d.vpcService.AuthorizeSecurityGroupEgress(&ec2.AuthorizeSecurityGroupEgressInput{
+				GroupId: &sgId,
+				IpPermissions: []*ec2.IpPermission{
+					{
+						IpProtocol: &allProto,
+						IpRanges:   []*ec2.IpRange{{CidrIp: &allCidr}},
+					},
+				},
+			}, accountID)
+			slog.Info("Created default security group for default VPC",
+				"groupId", sgId, "vpcId", defaultVpcId, "accountID", accountID)
+		}
+	}
+}
+
+// ensureDefaultIGWRoute adds 0.0.0.0/0 → igw-xxx to the main route table if not present
+func (d *Daemon) ensureDefaultIGWRoute(accountID, vpcID string) {
+	// Find the main route table for this VPC
+	vpcFilter := "vpc-id"
+	mainFilter := "association.main"
+	trueVal := "true"
+	descOut, err := d.routeTableService.DescribeRouteTables(&ec2.DescribeRouteTablesInput{
+		Filters: []*ec2.Filter{
+			{Name: &vpcFilter, Values: []*string{&vpcID}},
+			{Name: &mainFilter, Values: []*string{&trueVal}},
+		},
+	}, accountID)
+	if err != nil {
+		slog.Warn("Failed to query main route table for default IGW route", "vpcId", vpcID, "err", err)
+		return
+	}
+	if len(descOut.RouteTables) == 0 {
+		slog.Debug("No main route table found for default IGW route", "vpcId", vpcID, "accountID", accountID)
+		return
+	}
+
+	mainRtb := descOut.RouteTables[0]
+
+	// Check if 0.0.0.0/0 route already exists
+	for _, r := range mainRtb.Routes {
+		if r.DestinationCidrBlock != nil && *r.DestinationCidrBlock == "0.0.0.0/0" {
+			return // Already has a default route
+		}
+	}
+
+	// Find the attached IGW for this VPC
+	igwOut, err := d.igwService.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{}, accountID)
+	if err != nil {
+		slog.Warn("Failed to query IGWs for default route", "vpcId", vpcID, "err", err)
+		return
+	}
+	var igwID string
+	for _, igw := range igwOut.InternetGateways {
+		for _, att := range igw.Attachments {
+			if att.VpcId != nil && *att.VpcId == vpcID {
+				igwID = *igw.InternetGatewayId
+				break
+			}
+		}
+	}
+	if igwID == "" {
+		return // No IGW attached
+	}
+
+	// Add the default route
+	dest := "0.0.0.0/0"
+	_, err = d.routeTableService.CreateRoute(&ec2.CreateRouteInput{
+		RouteTableId:         mainRtb.RouteTableId,
+		DestinationCidrBlock: &dest,
+		GatewayId:            &igwID,
+	}, accountID)
+	if err != nil {
+		slog.Warn("Failed to add default IGW route to main route table", "err", err)
+	} else {
+		slog.Info("Added default IGW route to main route table",
+			"routeTableId", *mainRtb.RouteTableId, "igwId", igwID, "vpcId", vpcID)
+	}
 }