systemd tweaks for binary install to correctly function

Access key and secret key values were printed to stdout when using
--access-key and --secret-key flags. These are captured by systemd
journal and log aggregation, exposing secrets. Log that keys were
overridden without printing the actual values.

Addresses H-5 from security audit.

.githooks/pre-commit

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Only enforce preflight on dev and main
+if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
+  exit 0
+fi
+
+# Only run preflight if Go files were staged
+if git diff --cached --name-only | grep -q '\.go$'; then
+  echo "Go files staged on $branch — running make preflight..."
+  make preflight
+else
+  echo "No Go files staged, skipping preflight."
+fi

.github/dependabot.yml

@@ -1,7 +1,7 @@
 version: 2
 updates:
   - package-ecosystem: gomod
-    target-branch: dev
+    target-branch: main
     directory: /
     schedule:
       interval: weekly
@@ -15,11 +15,14 @@ updates:
           - "*"
 
   - package-ecosystem: github-actions
+    target-branch: main
     directory: /
     schedule:
       interval: weekly
       day: monday
       timezone: Australia/Sydney
+    cooldown:
+      default-days: 1
     ignore:
       - dependency-name: "*"
         update-types:
@@ -31,7 +34,7 @@ updates:
           - "*"
 
   - package-ecosystem: npm
-    target-branch: dev
+    target-branch: main
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
@@ -39,6 +42,10 @@ updates:
       timezone: Australia/Sydney
     cooldown:
       default-days: 1
+    ignore:
+      - dependency-name: "@types/node"
+        update-types:
+          - "version-update:semver-major"
     groups:
       npm-dependencies:
         patterns:

.github/workflows/e2e.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
@@ -13,6 +12,13 @@ on:
       - tests/e2e/**
       - .github/workflows/e2e.yml
   workflow_dispatch:
+    inputs:
+      sha:
+        description: "PR head SHA to report status on"
+        required: false
+
+permissions:
+  statuses: write
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -25,7 +31,7 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -72,7 +78,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh'
+            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh && ./tests/e2e/run-cert-e2e.sh'
 
       - name: Teardown
         if: always()
@@ -89,7 +95,7 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -152,7 +158,7 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
@@ -207,7 +213,7 @@ jobs:
           done
 
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
+            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}"
 
       - name: Teardown
         if: always()
@@ -216,3 +222,29 @@ jobs:
           tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
+
+  e2e_status:
+    name: Report E2E Status
+    if: always() && inputs.sha
+    needs: [e2e_single, e2e_pseudo_multi, e2e_real_multi]
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 5
+    steps:
+      - name: Report status to PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ needs.e2e_single.result }}" == "success" && \
+                "${{ needs.e2e_pseudo_multi.result }}" == "success" && \
+                "${{ needs.e2e_real_multi.result }}" == "success" ]]; then
+            STATE="success"
+            DESC="All E2E suites passed"
+          else
+            STATE="failure"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}"
+          fi
+          gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
+            -f state="$STATE" \
+            -f context=e2e \
+            -f description="$DESC" \
+            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/spinifex.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
@@ -14,7 +13,6 @@ on:
   pull_request:
     branches:
       - main
-      - dev
   workflow_dispatch:
 
 jobs:

.github/workflows/ui.yml

@@ -4,14 +4,12 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - spinifex/services/spinifexui/frontend/**
       - .github/workflows/ui.yml
   pull_request:
     branches:
       - main
-      - dev
     paths:
       - spinifex/services/spinifexui/frontend/**
       - .github/workflows/ui.yml

.gitignore

@@ -40,3 +40,4 @@ docs/PLAN.md
 docs/TODO.md
 docs/development/
 docs/internal/
+scripts/iac/aws/spinifex-test.pem

Makefile

@@ -63,6 +63,19 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+build-alb-agent:
+	@echo -e "\n....Building alb-agent (static)"
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/alb-agent cmd/alb-agent/main.go
+
+build-system-image: ## Build a system image from manifest (use IMAGE=alb)
+ifndef IMAGE
+	$(error IMAGE is required. Usage: make build-system-image IMAGE=alb)
+endif
+	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
+
+build-alb-image: ## Build ALB Alpine image
+	$(MAKE) build-system-image IMAGE=alb
+
 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
 	$(GOPATH)/bin/$(GO_PROJECT_NAME)
@@ -194,7 +207,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
+.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean

build/Dockerfile.distro

@@ -42,6 +42,7 @@ RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-s
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
+RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
 RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
 
 FROM scratch AS export

build/scripts/predastore-start.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Helper script for spinifex-predastore.service
+# Detects host, port, backend, and node-id from spinifex.toml
+# then execs spx. Falls back to single-node defaults if no config found.
+
+CONF=/etc/spinifex/spinifex.toml
+BIND="0.0.0.0:8443"
+BACKEND="filesystem"
+NODE_ID="0"
+
+if [ -f "$CONF" ]; then
+    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$H" ]; then
+        BIND="$H"
+        BACKEND="distributed"
+    fi
+
+    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
+    if [ -n "$N" ] && [ "$N" != "0" ]; then
+        NODE_ID="$N"
+    fi
+fi
+
+export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
+export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
+export SPINIFEX_PREDASTORE_BACKEND="$BACKEND"
+export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
+
+exec /usr/local/bin/spx service predastore start

build/scripts/wait-for-nats.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Wait for NATS to be reachable before starting a dependent service.
+# Reads the NATS address from spinifex.toml [nodes.*.nats] section,
+# falls back to nats.conf, then to 127.0.0.1:4222.
+
+CONF=/etc/spinifex/spinifex.toml
+NATS_CONF=/etc/spinifex/nats/nats.conf
+NATS_HOST=""
+NATS_PORT="4222"
+TIMEOUT=30
+
+# Try spinifex.toml first
+if [ -f "$CONF" ]; then
+    NATS_BIND=$(awk -F'"' '/\[nodes\..*\.nats\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$NATS_BIND" ]; then
+        NATS_HOST="${NATS_BIND%%:*}"
+        NATS_PORT="${NATS_BIND##*:}"
+    fi
+fi
+
+# Fall back to nats.conf
+if [ -z "$NATS_HOST" ] && [ -f "$NATS_CONF" ]; then
+    NATS_HOST=$(grep "^listen:" "$NATS_CONF" | sed 's/listen: *//;s/:.*//')
+fi
+
+# Final fallback
+NATS_HOST="${NATS_HOST:-127.0.0.1}"
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "Timed out waiting for NATS at $NATS_HOST:$NATS_PORT" >&2
+exit 1

build/systemd/spinifex-awsgw.service

@@ -2,13 +2,13 @@
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-daemon.service

@@ -2,13 +2,13 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-predastore.service

@@ -2,14 +2,14 @@
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
-ExecStart=/usr/local/bin/spx service predastore start
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/var/lib/spinifex/predastore-start.sh
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500

build/systemd/spinifex-ui.service

@@ -11,6 +11,8 @@ ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
+Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -2,13 +2,13 @@
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-predastore.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5

build/systemd/spinifex-vpcd.service

@@ -2,13 +2,13 @@
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
 User=spinifex
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5

cmd/alb-agent/main.go

@@ -0,0 +1,62 @@
+// alb-agent runs inside ALB VMs and manages HAProxy configuration.
+//
+// It connects to NATS, subscribes to config updates for its load balancer,
+// writes HAProxy configs to disk, and reloads HAProxy on changes. It also
+// responds to health pings so the daemon can verify the agent is alive.
+//
+// Usage:
+//
+//	alb-agent --lb-id=lb-xxxxx --nats=nats://10.0.0.1:4222
+//
+// The LB ID defaults to the hostname (set by cloud-init to the LB ID).
+// The NATS URL defaults to nats://127.0.0.1:4222.
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mulgadc/spinifex/spinifex/albagent"
+)
+
+func main() {
+	var (
+		lbID    string
+		natsURL string
+	)
+
+	hostname, _ := os.Hostname()
+
+	flag.StringVar(&lbID, "lb-id", hostname, "Load balancer ID (defaults to hostname)")
+	flag.StringVar(&natsURL, "nats", "nats://127.0.0.1:4222", "NATS server URL")
+	flag.Parse()
+
+	if lbID == "" {
+		slog.Error("--lb-id is required (or set hostname to the LB ID)")
+		os.Exit(1)
+	}
+
+	slog.Info("Starting ALB agent", "lbId", lbID, "nats", natsURL)
+
+	agent, err := albagent.New(lbID, natsURL)
+	if err != nil {
+		slog.Error("Failed to create agent", "err", err)
+		os.Exit(1)
+	}
+
+	if err := agent.Start(); err != nil {
+		slog.Error("Failed to start agent", "err", err)
+		os.Exit(1)
+	}
+
+	// Wait for shutdown signal
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+	sig := <-sigCh
+
+	slog.Info("Received signal, shutting down", "signal", sig)
+	agent.Stop()
+}

cmd/spinifex/cmd/admin.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
@@ -32,6 +33,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -158,6 +160,22 @@ var accountListCmd = &cobra.Command{
 	Run:   runAccountList,
 }
 
+var certCmd = &cobra.Command{
+	Use:   "cert",
+	Short: "Manage TLS certificates",
+	Long:  `Manage TLS certificates for the Spinifex platform.`,
+}
+
+var certRenewCmd = &cobra.Command{
+	Use:   "renew",
+	Short: "Regenerate the server certificate with current IPs",
+	Long: `Regenerate the server certificate signed by the existing CA.
+The new certificate will include all current network interface IPs
+and the machine hostname in its SANs. Use this after adding a new
+network interface or changing IP addresses.`,
+	Run: runCertRenew,
+}
+
 /*
 CLI ideas
 
@@ -195,6 +213,11 @@ func init() {
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
+
+	adminCmd.AddCommand(certCmd)
+	certCmd.AddCommand(certRenewCmd)
+	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
+	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
 	accountCreateCmd.MarkFlagRequired("name")
 
@@ -215,12 +238,13 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
+	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
 
 	// External networking flags
 	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
-	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'static' (default, uses --external-pool range) or 'dhcp' (from router DHCP)")
+	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
 	adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)")
 	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
@@ -238,6 +262,7 @@ func init() {
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
+	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 
@@ -556,6 +581,31 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	noExternal, _ := cmd.Flags().GetBool("no-external")
 
+	// Fire telemetry in background (completes during init work, waited at end)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID:    admin.ReadMachineID(),
+				Event:        "init",
+				Region:       region,
+				AZ:           az,
+				Node:         node,
+				Nodes:        nodes,
+				BindIP:       bindIP,
+				Version:      Version,
+				ExternalMode: externalMode,
+			})
+		})
+	}
+
 	// Auto-detect network topology
 	var poolStart, poolEnd string
 	var detectedNet *admin.DetectedNetwork
@@ -613,6 +663,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 	if externalMode == "pool" {
+		// Resolve source: if not explicitly set, infer from whether a pool range was given
+		if externalSource == "" {
+			if externalPool != "" {
+				externalSource = "static"
+			} else {
+				externalSource = "dhcp"
+			}
+		}
 		if externalSource == "dhcp" {
 			// DHCP source: no static range needed, just gateway
 			if externalGateway == "" {
@@ -620,7 +678,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 		} else {
-			// Static source (default): need pool range
+			// Static source: need pool range
 			if externalPool == "" {
 				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n")
 				fmt.Fprintf(os.Stderr, "   Or use --external-source=dhcp to get IPs from router DHCP\n")
@@ -1188,6 +1246,16 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	admin.CreateServiceDirectories(spxRoot)
 
+	// In production layout (running as root), fix ownership so the service user
+	// can read config and write data. SUDO_USER identifies the operator account.
+	if os.Getuid() == 0 {
+		sudoUser := os.Getenv("SUDO_USER")
+		if sudoUser != "" {
+			admin.ChownRecursive(configDir, sudoUser)
+			admin.ChownRecursive(spxRoot, sudoUser)
+		}
+	}
+
 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
 	time.Sleep(15 * time.Second)
@@ -1291,9 +1359,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // formation server uses ephemeral self-signed cert
+		},
+	}
 
-	joinURL := fmt.Sprintf("http://%s/formation/join", leaderHost)
+	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
 	resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
@@ -1322,7 +1395,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Registered with formation server (%d/%d nodes joined)\n", joinResp.Joined, joinResp.Expected)
 
 	// Poll status until formation is complete
-	statusURL := fmt.Sprintf("http://%s/formation/status", leaderHost)
+	statusURL := fmt.Sprintf("https://%s/formation/status", leaderHost)
 	var statusResp formation.StatusResponse
 
 	for {
@@ -1354,6 +1427,30 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	fmt.Printf("✅ Cluster formation complete! (%d nodes)\n\n", statusResp.Expected)
 
+	// Fire telemetry after formation (now we know the cluster topology)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID: admin.ReadMachineID(),
+				Event:     "join",
+				Region:    region,
+				AZ:        az,
+				Node:      node,
+				Nodes:     statusResp.Expected,
+				BindIP:    bindIP,
+				Version:   Version,
+			})
+		})
+	}
+
 	// Extract credentials and CA from formation status
 	creds := statusResp.Credentials
 	if creds == nil {
@@ -1551,6 +1648,16 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(dataDir)
 
+	// In production layout (running as root), fix ownership so the service user
+	// can read config and write data. SUDO_USER identifies the operator account.
+	if os.Getuid() == 0 {
+		sudoUser := os.Getenv("SUDO_USER")
+		if sudoUser != "" {
+			admin.ChownRecursive(configDir, sudoUser)
+			admin.ChownRecursive(dataDir, sudoUser)
+		}
+	}
+
 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
 	fmt.Printf("   Cluster: %s (%d nodes)\n", creds.ClusterName, len(statusResp.Nodes))
@@ -1760,6 +1867,33 @@ func runAccountList(cmd *cobra.Command, args []string) {
 	}
 }
 
+func runCertRenew(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+
+	// Verify CA files exist.
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	if !admin.FileExists(caCertPath) || !admin.FileExists(caKeyPath) {
+		fmt.Fprintf(os.Stderr, "Error: CA files not found in %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	extraIPs, _ := cmd.Flags().GetStringSlice("extra-ip")
+	extraDNS, _ := cmd.Flags().GetStringSlice("extra-dns")
+
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("✅ Server certificate regenerated with current IPs and hostname")
+	fmt.Printf("   Certificate: %s\n", serverCertPath)
+	fmt.Println("\n⚠️  Restart awsgw and daemon services to pick up the new certificate.")
+}
+
 // writeBootstrapResult holds the admin credentials so callers can
 // write them to ~/.aws/credentials instead of the system credentials.
 type writeBootstrapResult struct {

cmd/spinifex/cmd/service.go

@@ -25,6 +25,7 @@ import (
 	"github.com/mulgadc/predastore/s3"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/service"
+	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
@@ -244,13 +245,13 @@ var viperblockStartCmd = &cobra.Command{
 
 		accessKey := viper.GetString("access-key")
 		if accessKey != "" {
-			fmt.Println("Overwriting access-key:", accessKey)
+			fmt.Println("Overwriting access-key: ****")
 			nodeConfig.Predastore.AccessKey = accessKey
 		}
 
 		secretKey := viper.GetString("secret-key")
 		if secretKey != "" {
-			fmt.Println("Overwriting secret-key:", secretKey)
+			fmt.Println("Overwriting secret-key: ****")
 			nodeConfig.Predastore.SecretKey = secretKey
 		}
 
@@ -557,6 +558,7 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
 
+		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)
 
 		if err != nil {
@@ -711,7 +713,9 @@ var vpcdStartCmd = &cobra.Command{
 			})
 		}
 
-		// Derive OVN chassis names from cluster node names (format: "chassis-{hostname}")
+		// Fallback chassis names derived from config node names. vpcd prefers
+		// discovering chassis from the OVN Southbound DB at startup; these are
+		// only used if that discovery fails.
 		var chassisNames []string
 		for nodeName := range clusterConfig.Nodes {
 			chassisNames = append(chassisNames, "chassis-"+nodeName)

docs/COMMANDS.md

@@ -54,11 +54,23 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured | **DONE** |
 | `spx admin join` | `--host` (required), `--node` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error) | **DONE** |
 
+### Version
+
+| Command | Flags | Prerequisites | Basic Logic | Status |
+|---------|-------|---------------|-------------|--------|
+| `spx version` | — | None | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) | **DONE** |
+
 ### Cluster Operations
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------|---------------|-------------|------------|--------|
-| `spx admin cluster shutdown` | — | Cluster must be running | Gracefully shuts down the entire cluster | 1. Shutdown running cluster<br>2. All nodes stop cleanly | **DONE** |
+| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Cluster must be running | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. | 1. Shutdown running cluster<br>2. All nodes stop cleanly<br>3. Force shutdown with unresponsive nodes<br>4. Dry-run prints plan | **DONE** |
+
+### Certificate Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
 
 ### Account Management
 
@@ -81,7 +93,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--subnet-id` (auto-creates ENI, assigns private IP), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName only — routes via spread or cluster strategy) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → if Placement.GroupName set, looks up strategy: spread → `distributeInstancesSpread()` (1 instance per node, atomic CAS reservation), cluster → `distributeInstancesCluster()` (pin all to single node); otherwise NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → if SubnetId provided, auto-creates ENI with private IP → cloud-init injects user-data/keys → on termination, removes instance from placement group → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. Launch with subnet-id (auto-creates ENI)<br>6. Invalid instance type returns error<br>7. Launch with spread placement group (1 per node)<br>8. Launch with cluster placement group (all on one node)<br>9. Insufficient capacity for placement group (error) | **DONE** |
-| `describe-instances` | `--instance-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes | **DONE** |
+| `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:\*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list. Filters applied per-node before aggregation (reduces payload). Also applies to stopped/terminated instances via `describeInstancesFromKV()`. | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes<br>6. Filter by tag<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
 | `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
@@ -89,6 +101,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
 | `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data` | `--disable-api-termination`, `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must be stopped (in NATS KV) | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon loads stopped instance from JetStream KV → applies attribute change → writes back to KV → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Modify running instance (error: NotFound — running instances not in KV)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Instance not stopped (error: IncorrectInstanceState)<br>6. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>7. StateReason cleared on type change (recovery from capacity-unavailable)<br>8. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>9. No attribute set (error: InvalidParameterValue)<br>10. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest` (always returns latest), `--dry-run` | Instance must be running on a node | Gateway sends NATS `ec2.{instanceId}.GetConsoleOutput` (per-instance topic, routed to owning node) → daemon reads console log file from disk → returns last 64KB base64-encoded with timestamp. Always available regardless of serial console access setting (matches AWS behavior). | 1. Get output from running instance<br>2. Empty log file returns empty output<br>3. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`) return real values; unstored attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, `disableApiTermination`=false, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get default disableApiTermination<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | None | Gateway-only stub — returns `CpuCredits: "standard"` for each requested instance ID. No daemon round-trip. T-series credit mode is not persisted. | 1. Get credit spec for T-series instance<br>2. Multiple instance IDs | **DONE** |
 | `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |
 
@@ -97,7 +111,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format` (pem/ppk), `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreateKeyPair` → daemon generates SSH keypair → stores public key in Predastore S3 (`/bucket/ec2/{name}.pub`) → returns private key material (one-time) | 1. Create RSA key pair<br>2. Create ED25519 key pair<br>3. Duplicate key name (error: InvalidKeyPair.Duplicate)<br>4. Verify key material returned only on creation | **DONE** |
-| `describe-key-pairs` | `--key-names`, `--key-pair-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty | **DONE** |
+| `describe-key-pairs` | `--key-names`, `--key-pair-ids`, `--filters` (key-pair-id, key-name, fingerprint, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → applies filters → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 | `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | Key must exist | NATS `ec2.DeleteKeyPair` → daemon deletes public key from Predastore S3 → returns success | 1. Delete existing key pair<br>2. Delete non-existent key (idempotent, no error)<br>3. Verify key no longer in describe-key-pairs | **DONE** |
 | `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | None | NATS `ec2.ImportKeyPair` → daemon stores provided public key in Predastore S3 → returns key name and fingerprint | 1. Import valid SSH public key<br>2. Import invalid key material (error)<br>3. Import duplicate name (error)<br>4. Verify imported key usable with run-instances | **DONE** |
 
@@ -105,7 +119,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias) | `--executable-users`, `--filters`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds and Owners → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName) | **DONE** |
+| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias), `--filters` (name, state, architecture, image-id, is-public, owner-id, description, image-type, tag:\*) | `--executable-users`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds, Owners, and Filters → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName)<br>6. Filter by name wildcard<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `create-image` | `--instance-id`, `--name`, `--description`, `--tag-specifications` | `--no-reboot`, `--block-device-mappings`, `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.{instanceId}.CreateImage` (per-instance topic) → daemon extracts root volume → snapshots via `ebs.snapshot` NATS (running) or offline S3 copy (stopped) → creates AMI metadata in S3 (`{amiId}/config.json`) → stores tags → returns ami-ID. Duplicate AMI name validation enforced. | 1. Create image from running instance<br>2. Create image from stopped instance<br>3. Invalid instance ID (error)<br>4. Duplicate AMI name (error)<br>5. Verify new AMI appears in describe-images<br>6. Launch new instance from created AMI | **DONE** |
 | `register-image` | — | `--name`, `--description`, `--architecture`, `--root-device-name`, `--virtualization-type`, `--block-device-mappings`, `--boot-mode`, `--ena-support`, `--sriov-net-support`, `--tpm-support`, `--imds-support` | Image data must exist in S3 | Create AMI metadata entry in Predastore → assign ami-ID → set state to available → return ami-ID | 1. Register with valid metadata<br>2. Missing required name (error)<br>3. Verify registered image in describe-images | **NOT STARTED** |
 | `deregister-image` | — | `--image-id`, `--dry-run` | AMI must exist, no instances using it | Mark AMI metadata as deregistered in S3 → AMI no longer available for new launches → existing instances unaffected | 1. Deregister existing AMI<br>2. Deregister non-existent AMI (error)<br>3. Verify deregistered AMI not in describe-images<br>4. Existing instances from AMI still run | **NOT STARTED** |
@@ -119,13 +133,13 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata) | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value | **DONE** |
+| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata), `--filters` (volume-id, status, size, volume-type, attachment.instance-id, attachment.status, attachment.device, availability-zone, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → applies filters → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value<br>6. Filter by status, size, volume-type<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | Volume must exist | NATS `ec2.ModifyVolume` → daemon sends resize request to viperblock → NBD does not support live resize, instance must be stopped → returns modification state | 1. Increase volume size<br>2. Modify volume type<br>3. Decrease size (error - not supported)<br>4. Modify attached volume (requires stop/start) | **DONE** |
 | `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3 only), `--snapshot-id` (creates volume from snapshot) | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput`, `--tag-specifications` | Valid AZ configured via `spinifex init` | Gateway validates input → NATS `ec2.CreateVolume` → daemon generates vol-ID via viperblock → creates volume (empty or from snapshot) of specified size → persists config.json to Predastore S3 → returns vol-ID with state=available | 1. Create 80GB gp3 volume<br>2. Boundary sizes (1 GiB min, 16384 GiB max)<br>3. Invalid AZ (error)<br>4. Verify volume in describe-volumes<br>5. Unsupported volume type (error - only gp3)<br>6. Size out of range (error)<br>7. Create from snapshot | **DONE** |
 | `delete-volume` | `--volume-id` | `--dry-run` | Volume must exist and be detached (state=available) | Gateway validates vol- prefix → NATS `ec2.DeleteVolume` → daemon confirms state=available and no AttachedInstance → NATS `ebs.delete` to viperblockd (stops nbdkit/WAL) → deletes S3 objects under vol-id/, vol-id-efi/, vol-id-cloudinit/ → returns success | 1. Delete detached volume<br>2. Delete attached volume (error: VolumeInUse)<br>3. Delete non-existent volume (error: InvalidVolume.NotFound)<br>4. Verify volume gone from describe-volumes<br>5. Malformed volume ID (error: InvalidVolumeID.Malformed)<br>6. Double delete (idempotent NotFound) | **DONE** |
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (optional, auto-assigns `/dev/sd[f-p]`) | `--dry-run` | Volume must exist (available), instance must exist (running) | Gateway sends to `ec2.cmd.{instanceId}` → daemon validates volume (Predastore) → `ebs.mount` via NATS (viperblock starts NBD server) → QMP `blockdev-add` (nbd-{volId}) → QMP `device_add` (virtio-blk-pci, vdisk-{volId}) → three-phase rollback on failure → update EBSRequests + BlockDeviceMappings → persist to JetStream + Predastore → respond with VolumeAttachment | 1. Attach volume to running instance<br>2. Auto-assign device name<br>3. Attach already-attached volume (VolumeInUse)<br>4. Attach to non-existent instance (InvalidInstanceID.NotFound)<br>5. Attach to stopped instance (IncorrectInstanceState)<br>6. Volume not found (InvalidVolume.NotFound)<br>7. All device slots full (AttachmentLimitExceeded)<br>8. Volume persists across stop/start | **DONE** |
 | `detach-volume` | `--volume-id`, `--instance-id` (optional, resolved via DescribeVolumes), `--device` (optional cross-check), `--force` | `--dry-run` | Volume must be attached, instance must be running | Gateway resolves InstanceId if omitted (via DescribeVolumes) → sends to `ec2.cmd.{instanceId}` → daemon validates (running, attached, not boot/EFI/CloudInit, device match) → three-phase hot-unplug: QMP `device_del` (force continues on failure) → QMP `blockdev-del` (abort if fails, preserves state to prevent double-attach) → `ebs.unmount` via NATS (best-effort) → remove from EBSRequests + BlockDeviceMappings → update volume metadata to available → persist state → respond with VolumeAttachment (state=detaching) | 1. Detach with explicit InstanceId<br>2. Detach without InstanceId (gateway resolution)<br>3. Detach with correct --device cross-check<br>4. Missing VolumeId (InvalidParameterValue)<br>5. Volume not attached (IncorrectState)<br>6. Nonexistent volume (InvalidVolume.NotFound)<br>7. Nonexistent instance (InvalidInstanceID.NotFound)<br>8. Instance not running (IncorrectInstanceState)<br>9. Device mismatch (InvalidParameterValue)<br>10. Boot volume protection (OperationNotPermitted)<br>11. Force flag (continues past device_del failure)<br>12. Volume reusability (re-attach after detach) | **DONE** |
-| `describe-volume-status` | `--volume-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes | **DONE** |
+| `describe-volume-status` | `--volume-ids`, `--filters` (volume-id, volume-status.status, availability-zone) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → applies filters → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | None | Query pending/completed volume modifications → return modification state, progress, original/target size | 1. Check in-progress modification<br>2. Check completed modification<br>3. No modifications returns empty | **NOT STARTED** |
 
 ### EC2 - Snapshot Management
@@ -135,7 +149,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `create-snapshot` | `--volume-id`, `--description`, `--tag-specifications` | `--dry-run` | Volume must exist in Predastore | Gateway validates vol- prefix → NATS `ec2.CreateSnapshot` → daemon reads VolumeConfig from Predastore → generates snap-ID via viperblock → stores SnapshotConfig (metadata-only, points to source volume) as completed → returns Snapshot with state=completed, progress=100% | 1. Snapshot from valid volume<br>2. Missing volume ID (InvalidParameterValue)<br>3. Volume not found (InvalidVolume.NotFound)<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Snapshot with description<br>6. Snapshot with tag specifications | **DONE** |
 | `create-snapshots` | — | `--instance-specification`, `--description`, `--tag-specifications` | Instance must exist, instance-volume attachment tracking | Create snapshots of all volumes attached to instance → return list of snapshot IDs. Blocked: requires instance-volume attachment tracking to discover which volumes to snapshot. | 1. Snapshot all volumes on instance<br>2. Instance with no volumes | **NOT STARTED** |
 | `delete-snapshot` | `--snapshot-id` | `--dry-run` | Snapshot must exist | Gateway validates snap- prefix → NATS `ec2.DeleteSnapshot` → daemon verifies snapshot exists in Predastore → lists and deletes all objects under snapshot prefix → returns success | 1. Delete existing snapshot<br>2. Delete non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing snapshot ID (InvalidParameterValue)<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
-| `describe-snapshots` | `--snapshot-ids` | `--owner-ids`, `--filters`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → filters by snapshot ID if specified → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
+| `describe-snapshots` | `--snapshot-ids`, `--filters` (snapshot-id, status, volume-id, volume-size, owner-id, tag:\*) | `--owner-ids`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → applies filters → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed)<br>5. Filter by status, volume-id, volume-size<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
 | `copy-snapshot` | `--source-snapshot-id`, `--source-region`, `--description` | `--encrypted`, `--dry-run` | Source snapshot must exist | Gateway validates snap- prefix + source region → NATS `ec2.CopySnapshot` → daemon reads source SnapshotConfig → generates new snap-ID → copies metadata (preserves tags, description override) → stores as completed → returns new snapshot ID | 1. Copy within same region<br>2. Copy non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing source ID (InvalidParameterValue)<br>4. Missing source region (MissingParameter)<br>5. Copy preserves tags<br>6. Copy with description override | **DONE** |
 
 ### EC2 - Tags
@@ -188,7 +202,7 @@ Gateway routing and KV CRUD are wired up but no OVN/OVS integration exists. EIGW
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-egress-only-internet-gateway` | `--vpc-id`, `--tag-specifications` | `--client-token`, `--dry-run` | VPC must exist | Gateway validates VpcId → NATS `ec2.CreateEgressOnlyInternetGateway` → daemon generates eigw-ID → creates EgressOnlyIGWRecord in `spinifex-eigw` KV bucket with state=attached (always attached at creation, unlike IGW) → returns EgressOnlyInternetGateway with attachment | 1. Create with valid VpcId<br>2. Missing VpcId (error: MissingParameter)<br>3. Create with inline tags<br>4. Verify ID format eigw-{16hex} | **STARTED** (KV only, no OVN) |
 | `delete-egress-only-internet-gateway` | `--egress-only-internet-gateway-id` | `--dry-run` | EIGW must exist | NATS `ec2.DeleteEgressOnlyInternetGateway` → daemon verifies EIGW exists in KV → deletes from `spinifex-eigw` KV → returns ReturnCode=true. No detach required (unlike IGW). | 1. Delete existing EIGW<br>2. Delete non-existent EIGW (error)<br>3. Missing ID (error: MissingParameter) | **STARTED** (KV only, no OVN) |
-| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list | **STARTED** (KV only, no OVN) |
+| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids`, `--filters` (egress-only-internet-gateway-id, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → applies filters → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list<br>4. Unknown filter returns InvalidParameterValue | **STARTED** (KV only, no OVN) |
 
 ### EC2 - Placement Groups
 
@@ -236,67 +250,76 @@ Placement groups are stored in `spinifex-placement-groups` JetStream KV bucket,
 
 ### EC2 - Elastic IP
 
+EIP operations are conditionally available — only registered when external IPAM is configured (pool mode). EIPs are stored in `spinifex-eip-allocations` KV bucket. Association publishes `vpc.add-nat`/`vpc.delete-nat` events to vpcd for OVN NAT configuration.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `allocate-address` | — | `--domain` (vpc), `--tag-specifications`, `--dry-run` | None | NATS `ec2.AllocateAddress` → daemon allocates IP from pool → stores in JetStream KV with eipalloc-ID → return AllocationId, PublicIp, Domain | 1. Allocate EIP<br>2. Verify in describe-addresses<br>3. Allocate without --domain uses default | **NOT STARTED** |
-| `release-address` | — | `--allocation-id`, `--dry-run` | EIP must exist and be disassociated | NATS `ec2.ReleaseAddress` → daemon verifies not associated → release IP back to pool → delete from KV → return success | 1. Release disassociated EIP<br>2. Release associated EIP (error)<br>3. Release non-existent (error) | **NOT STARTED** |
-| `associate-address` | — | `--allocation-id`, `--instance-id` or `--network-interface-id`, `--dry-run` | EIP and target must exist | NATS `ec2.AssociateAddress` → daemon links EIP to instance/ENI → configure OVS DNAT/SNAT rules → return eipassoc-ID | 1. Associate with instance<br>2. Re-associate (moves EIP)<br>3. Associate non-existent EIP (error) | **NOT STARTED** |
-| `disassociate-address` | — | `--association-id`, `--dry-run` | Association must exist | NATS `ec2.DisassociateAddress` → daemon unlinks EIP from instance/ENI → remove OVS DNAT/SNAT rules → return success | 1. Disassociate existing<br>2. Disassociate non-existent (error) | **NOT STARTED** |
-| `describe-addresses` | — | `--allocation-ids`, `--filters`, `--public-ips` | None | NATS `ec2.DescribeAddresses` → daemon lists EIPs from KV → return list with association info | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP | **NOT STARTED** |
+| `allocate-address` | `--public-ipv4-pool` (optional, specific IPAM pool), `--tag-specifications` | `--domain` (hardcoded vpc), `--dry-run` | EIP service must be initialized (external IPAM configured) | Gateway validates input → NATS `ec2.AllocateAddress` with `spinifex-workers` queue group → daemon allocates IP from external IPAM pool (named or best-matching) → creates EIPRecord with state=allocated in `spinifex-eip-allocations` KV → returns AllocationId, PublicIp, Domain=vpc, PublicIpv4Pool | 1. Allocate EIP from default pool<br>2. Allocate from specific pool<br>3. Pool exhausted (InsufficientAddressCapacity)<br>4. With tag specifications<br>5. Verify in describe-addresses | **DONE** |
+| `release-address` | `--allocation-id` | `--dry-run` | EIP must exist and be disassociated | Gateway validates AllocationId non-empty → NATS `ec2.ReleaseAddress` → daemon retrieves EIP record from KV → checks state != associated (InvalidAddressLocked if still associated) → releases IP back to IPAM pool → deletes KV record → returns success | 1. Release disassociated EIP<br>2. Release associated EIP (error: InvalidAddressLocked)<br>3. Release non-existent (error: InvalidAllocationID.NotFound)<br>4. Missing AllocationId (error: MissingParameter) | **DONE** |
+| `associate-address` | `--allocation-id`, `--network-interface-id` OR `--instance-id`, `--private-ip-address` (optional override) | `--dry-run`, `--allow-reassociation` | EIP and target ENI/instance must exist | Gateway validates AllocationId + one of NetworkInterfaceId/InstanceId required → NATS `ec2.AssociateAddress` → daemon retrieves EIP → resolves target ENI (by ID or instance lookup) → generates AssociationId → updates record state=associated with ENI/instance/VPC/private IP → publishes `vpc.add-nat` event to vpcd (OVN NAT configuration) → returns AssociationId | 1. Associate with instance<br>2. Associate with specific ENI<br>3. Associate with private IP override<br>4. Missing AllocationId (MissingParameter)<br>5. Missing target (MissingParameter)<br>6. Non-existent EIP (InvalidAllocationID.NotFound)<br>7. Non-existent ENI (InvalidNetworkInterfaceID.NotFound) | **DONE** |
+| `disassociate-address` | `--association-id` | `--dry-run` | Association must exist | Gateway validates AssociationId non-empty → NATS `ec2.DisassociateAddress` → daemon scans KV to find EIP by AssociationId → publishes `vpc.delete-nat` event to vpcd → clears association fields → reverts state to allocated → updates KV → returns success | 1. Disassociate existing<br>2. Non-existent association (InvalidAssociationID.NotFound)<br>3. Missing AssociationId (MissingParameter) | **DONE** |
+| `describe-addresses` | `--allocation-ids`, `--public-ips`, `--filters` (allocation-id, public-ip, instance-id, association-id, domain, tag:\*) | `--dry-run` | None | NATS `ec2.DescribeAddresses` → daemon scans all EIP records in account → filters by AllocationIds and/or PublicIps if specified → applies filters → returns Address list with allocation, association, PublicIp, PrivateIpAddress, NetworkInterfaceId, InstanceId, Domain | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP<br>4. Non-existent AllocationId returns error<br>5. Empty result when no EIPs<br>6. Filter by instance-id, domain<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
+| `describe-addresses-attribute` | `--allocation-ids` (optional filter) | `--attribute`, `--dry-run`, `--max-results`, `--next-token` | None | Gateway validates input → NATS `ec2.DescribeAddressesAttribute` with `spinifex-workers` queue group → daemon looks up EIP records by AllocationIds (or scans all account EIPs if none specified) → returns AddressAttribute list with AllocationId and PublicIp per entry. Unlike DescribeAddresses, returns empty list (not error) for non-existent AllocationIds. PtrRecord always nil (reverse-DNS not supported). | 1. List all EIP attributes<br>2. Filter by allocation IDs<br>3. Non-existent AllocationId returns empty (not error)<br>4. Nil input returns error | **DONE** |
 
 ### EC2 - NAT Gateway
 
+NAT gateways are stored in `spinifex-vpc-nat-gateways` KV bucket. Deleted gateways are moved to `spinifex-vpc-deleted-nat-gateways` with a 1-hour TTL for Terraform polling. SNAT rule coordination uses `vpc.add-nat-gateway` and `vpc.delete-nat-gateway` NATS events (published via route table service on CreateRoute and during DeleteNatGateway respectively).
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-nat-gateway` | — | `--subnet-id`, `--allocation-id`, `--connectivity-type`, `--tag-specifications` | Subnet must exist, EIP must be allocated (for public type) | NATS `ec2.CreateNatGateway` → daemon creates NAT gateway in JetStream KV with nat-ID → configure OVS SNAT rules → state=pending→available → return NatGateway | 1. Create public NAT gateway<br>2. Create private NAT gateway (no allocation-id)<br>3. Invalid subnet (error) | **NOT STARTED** |
-| `delete-nat-gateway` | — | `--nat-gateway-id`, `--dry-run` | NAT gateway must exist | NATS `ec2.DeleteNatGateway` → daemon marks as deleting → remove OVS SNAT rules → delete from KV → return NatGatewayId | 1. Delete existing NAT gateway<br>2. Delete non-existent (error) | **NOT STARTED** |
-| `describe-nat-gateways` | — | `--nat-gateway-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNatGateways` → daemon lists NAT gateways from KV → return list with state, subnet, addresses | 1. List all NAT gateways<br>2. Filter by ID<br>3. Filter by VPC ID | **NOT STARTED** |
+| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type` (public only), `--tag-specifications`, `--dry-run` | Subnet must exist, EIP must be allocated and unassociated | Gateway validates SubnetId + AllocationId (required) → NATS `ec2.CreateNatGateway` with `spinifex-workers` queue group → daemon verifies subnet exists in `spinifex-vpc-subnets` KV → verifies EIP exists in `spinifex-vpc-eips` KV and has no AssociationId (Resource.AlreadyAssociated if in use) → generates nat-ID → stores NatGatewayRecord in `spinifex-vpc-nat-gateways` KV with state=available → returns NatGateway with addresses | 1. Create NAT gateway (happy path)<br>2. Subnet not found (InvalidSubnetID.NotFound)<br>3. EIP not found (InvalidAllocationID.NotFound)<br>4. EIP already associated (Resource.AlreadyAssociated)<br>5. Missing SubnetId or AllocationId (MissingParameter) | **DONE** |
+| `delete-nat-gateway` | `--nat-gateway-id` | `--dry-run` | NAT gateway must exist | Gateway validates NatGatewayId (required) → NATS `ec2.DeleteNatGateway` → daemon retrieves record from KV → scans all route tables in VPC for routes targeting this NAT gateway → publishes `vpc.delete-nat-gateway` event per affected subnet (SNAT rule cleanup) → sets state=deleted → stores in `spinifex-vpc-deleted-nat-gateways` with 1-hour TTL → deletes from active KV bucket | 1. Delete existing NAT gateway<br>2. Delete non-existent (InvalidNatGatewayID.NotFound)<br>3. Deleted gateway visible when queried by ID (state=deleted)<br>4. Deleted gateway excluded from general list queries (TTL auto-expire) | **DONE** |
+| `describe-nat-gateways` | `--nat-gateway-ids`, `--filters` (vpc-id, state) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeNatGateways` → daemon scans `spinifex-vpc-nat-gateways` KV → applies NatGatewayIds filter and/or vpc-id/state filters → for specific IDs not found in active bucket, checks deleted bucket → returns InvalidNatGatewayID.NotFound if ID never existed → returns list with state, addresses, subnet, VPC | 1. List all NAT gateways<br>2. Filter by specific IDs<br>3. Filter by state<br>4. Filter by vpc-id<br>5. Non-existent ID (InvalidNatGatewayID.NotFound)<br>6. Deleted gateway visible by ID query | **DONE** |
 | `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | NAT gateway must exist (private type) | NATS `ec2.AssignPrivateNatGatewayAddress` → daemon assigns private IPs to NAT gateway → return assigned addresses | 1. Assign address<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 | `associate-nat-gateway-address` | — | `--nat-gateway-id`, `--allocation-ids` | NAT gateway and EIPs must exist | NATS `ec2.AssociateNatGatewayAddress` → daemon associates EIPs with NAT gateway → return association info | 1. Associate EIP<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 
 ### EC2 - VPC Networking (Core)
 
-Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, and routers. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket).
+Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI/SG CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, routers, and ACLs. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket). Security groups stored in `spinifex-vpc-security-groups` KV bucket with optimistic locking on rule updates.
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-vpc` | `--cidr-block`, `--tag-specifications` | `--instance-tenancy`, `--dry-run` | None | Gateway validates CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateVpc` → daemon allocates unique VNI from `spinifex-vpc-vni-counter` KV → generates vpc-ID → stores VPCRecord in `spinifex-vpc-vpcs` KV → publishes `vpc.create` event to vpcd (creates OVN logical switch) → returns VPC with state=available | 1. Create VPC with /16 CIDR<br>2. Invalid CIDR format (error: InvalidVpcRange)<br>3. CIDR prefix outside /16–/28 (error)<br>4. Verify in describe-vpcs<br>5. Tags applied at creation | **DONE** |
 | `delete-vpc` | `--vpc-id` | `--dry-run` | VPC must be empty (no subnets) | Gateway validates VpcId → NATS `ec2.DeleteVpc` → daemon checks for dependent subnets in `spinifex-vpc-subnets` KV → returns DependencyViolation if subnets exist → deletes from `spinifex-vpc-vpcs` KV → publishes `vpc.delete` event to vpcd (removes OVN resources) | 1. Delete empty VPC<br>2. Delete VPC with subnets (DependencyViolation)<br>3. Delete non-existent VPC (error) | **DONE** |
-| `describe-vpcs` | `--vpc-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error | **DONE** |
-| `modify-vpc-attribute` | — | `--vpc-id`, `--enable-dns-support`, `--enable-dns-hostnames` | VPC must exist | NATS `ec2.ModifyVpcAttribute` → daemon updates VPC attribute in KV → return success | 1. Enable DNS support<br>2. Enable DNS hostnames<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `describe-vpcs` | `--vpc-ids`, `--filters` (vpc-id, state, cidr-block, is-default, owner-id, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → applies filters → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error<br>4. Filter by state, cidr-block, is-default<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-vpc-attribute` | `--vpc-id`, `--enable-dns-hostnames`, `--enable-dns-support`, `--enable-network-address-usage-metrics` | `--dry-run` | VPC must exist | Gateway validates VpcId non-empty → NATS `ec2.ModifyVpcAttribute` → daemon retrieves VPC record from `spinifex-vpc-vpcs` KV → updates boolean flags if provided → updates KV with optimistic locking → returns empty output | 1. Enable DNS hostnames<br>2. Enable DNS support<br>3. Enable network address usage metrics<br>4. Missing VPC ID (MissingParameter)<br>5. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
+| `describe-vpc-attribute` | `--vpc-id`, `--attribute` (enableDnsHostnames, enableDnsSupport, enableNetworkAddressUsageMetrics) | `--dry-run` | VPC must exist | Gateway validates VpcId + Attribute non-empty → NATS `ec2.DescribeVpcAttribute` → daemon retrieves VPC record → returns single attribute as AttributeBooleanValue (one attribute per call, matches AWS behavior) → validates attribute name | 1. Get enableDnsHostnames<br>2. Get enableDnsSupport<br>3. Get enableNetworkAddressUsageMetrics<br>4. Invalid attribute name (InvalidParameterValue)<br>5. Missing VPC ID (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
 | `associate-vpc-cidr-block` | — | `--vpc-id`, `--cidr-block` | VPC must exist | NATS `ec2.AssociateVpcCidrBlock` → daemon adds secondary CIDR to VPC metadata → return association | 1. Add secondary CIDR<br>2. Overlapping CIDR (error)<br>3. Max CIDR blocks exceeded (error) | **NOT STARTED** |
 | `disassociate-vpc-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateVpcCidrBlock` → daemon removes secondary CIDR from VPC → return success | 1. Remove secondary CIDR<br>2. Remove primary CIDR (error) | **NOT STARTED** |
 | `create-subnet` | `--vpc-id`, `--cidr-block`, `--availability-zone`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates VpcId + CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateSubnet` → daemon verifies parent VPC exists → validates subnet CIDR is within VPC CIDR range → checks for overlapping subnets in same VPC → calculates available IPs (total hosts - 5 AWS-reserved: .0/.1/.2/.3/.255) → generates subnet-ID → stores SubnetRecord in `spinifex-vpc-subnets` KV → publishes `vpc.create-subnet` event to vpcd → defaults AZ from config if omitted | 1. Create subnet within VPC CIDR<br>2. Subnet CIDR outside VPC range (error: InvalidSubnetRange)<br>3. Overlapping subnet CIDRs (error: InvalidSubnetConflict)<br>4. CIDR prefix outside /16–/28 (error)<br>5. Missing VPC or CIDR (error: MissingParameter) | **DONE** |
 | `delete-subnet` | `--subnet-id` | `--dry-run` | Subnet must exist | NATS `ec2.DeleteSubnet` → daemon verifies subnet exists → deletes from `spinifex-vpc-subnets` KV → publishes `vpc.delete-subnet` event to vpcd | 1. Delete existing subnet<br>2. Delete non-existent subnet (error) | **DONE** |
-| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies vpc-id filter if present → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error | **DONE** |
-| `modify-subnet-attribute` | — | `--subnet-id`, `--map-public-ip-on-launch`, `--assign-ipv6-address-on-creation` | Subnet must exist | NATS `ec2.ModifySubnetAttribute` → daemon updates subnet attribute in KV → return success | 1. Enable auto-assign public IP<br>2. Missing subnet ID (error) | **NOT STARTED** |
+| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id, subnet-id, availability-zone, cidr-block, state, default-for-az, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies filters → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error<br>5. Filter by availability-zone, cidr-block, state<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-subnet-attribute` | `--subnet-id`, `--map-public-ip-on-launch` | `--assign-ipv6-address-on-creation`, `--dry-run` | Subnet must exist | Gateway validates SubnetId non-empty → NATS `ec2.ModifySubnetAttribute` → daemon retrieves Subnet record from `spinifex-vpc-subnets` KV → updates MapPublicIpOnLaunch flag if provided → updates KV with optimistic locking → returns empty output | 1. Enable auto-assign public IP<br>2. Disable auto-assign public IP<br>3. Missing SubnetId (MissingParameter)<br>4. Non-existent subnet (InvalidSubnetID.NotFound) | **DONE** |
 | `associate-subnet-cidr-block` | — | `--subnet-id`, `--ipv6-cidr-block` | Subnet must exist | NATS `ec2.AssociateSubnetCidrBlock` → daemon adds IPv6 CIDR to subnet → return association | 1. Add IPv6 CIDR<br>2. Missing subnet ID (error) | **NOT STARTED** |
 | `disassociate-subnet-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateSubnetCidrBlock` → daemon removes IPv6 CIDR from subnet → return success | 1. Remove IPv6 CIDR<br>2. Invalid association (error) | **NOT STARTED** |
-| `create-security-group` | — | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | VPC must exist | Create security group metadata → create default OVS flow rules (deny all inbound, allow all outbound) → assign sg-ID | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error)<br>3. Verify default rules | **NOT STARTED** |
-| `delete-security-group` | — | `--group-id` | SG must not be in use by instances | Verify no instances reference SG → remove OVS flow rules → delete metadata | 1. Delete unused SG<br>2. Delete SG in use (error) | **NOT STARTED** |
-| `describe-security-groups` | — | `--group-ids`, `--group-names`, `--filters`, `--max-results` | None | Query NATS KV for SG metadata → return SG list with rules | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name | **NOT STARTED** |
-| `authorize-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr`, `--source-group` | SG must exist | Add inbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Duplicate rule (idempotent) | **NOT STARTED** |
-| `authorize-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist | Add outbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR | **NOT STARTED** |
-| `revoke-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove inbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
-| `revoke-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove outbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
+| `create-security-group` | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates GroupName + VpcId required → NATS `ec2.CreateSecurityGroup` → daemon verifies VPC exists → checks duplicate name in same VPC → generates sg-ID → creates SecurityGroupRecord in `spinifex-vpc-security-groups` KV with default egress rule (allow all outbound 0.0.0.0/0), no ingress rules → publishes `vpc.create-sg` event to vpcd (OVN ACL configuration) → returns GroupId | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error: InvalidGroup.Duplicate)<br>3. Verify default egress rule (allow all)<br>4. Verify no default ingress rules<br>5. Missing GroupName (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound)<br>7. With tag specifications | **DONE** |
+| `delete-security-group` | `--group-id` | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.DeleteSecurityGroup` → daemon retrieves SG from KV → deletes from `spinifex-vpc-security-groups` KV → publishes `vpc.delete-sg` event to vpcd → returns success | 1. Delete existing SG<br>2. Delete non-existent SG (error: InvalidGroup.NotFound)<br>3. Missing GroupId (MissingParameter) | **DONE** |
+| `describe-security-groups` | `--group-ids`, `--filters` (vpc-id, group-name, group-id, description, ip-permission.cidr, tag:\*) | `--group-names`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSecurityGroups` → daemon scans all SG records in account → applies filters (AND logic across filter names, OR across values) → returns SecurityGroup list with ingress/egress rules → returns error for non-existent requested GroupIds | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name<br>4. Filter by GroupId<br>5. Non-existent GroupId (InvalidGroup.NotFound)<br>6. Empty result when no SGs<br>7. Filter by description, ip-permission.cidr<br>8. Unknown filter returns InvalidParameterValue | **DONE** |
+| `authorize-security-group-ingress` | `--group-id`, `--ip-permissions` (IpProtocol, FromPort, ToPort, IpRanges/CidrIp, UserIdGroupPairs/GroupId) | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.AuthorizeSecurityGroupIngress` → daemon retrieves SG → converts IpPermissions to SGRule objects → appends to IngressRules → updates KV with optimistic locking (revision check) → publishes `vpc.update-sg` event to vpcd → returns true | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Allow from source security group<br>4. Missing GroupId (MissingParameter)<br>5. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `authorize-security-group-egress` | `--group-id`, `--ip-permissions` | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.AuthorizeSecurityGroupEgress` → daemon retrieves SG → appends rules to EgressRules → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR<br>3. Missing GroupId (MissingParameter)<br>4. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-ingress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupIngress` → daemon retrieves SG → removes matching rules from IngressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-egress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupEgress` → daemon retrieves SG → removes matching rules from EgressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `create-internet-gateway` | `--tag-specifications` | `--dry-run` | None | NATS `ec2.CreateInternetGateway` → daemon generates igw-ID → creates IGWRecord in `spinifex-igw` KV bucket with state=available (not attached) → returns InternetGateway with empty Attachments | 1. Create IGW<br>2. Verify in describe-internet-gateways<br>3. Tags applied at creation | **DONE** |
 | `attach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW and VPC must exist | NATS `ec2.AttachInternetGateway` → daemon validates IGW exists and is not already attached (ResourceAlreadyAssociated) → sets VpcId on record → state=attached → publishes `vpc.igw-attach` event to vpcd (creates OVN external switch, gateway port, SNAT rules) | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (ResourceAlreadyAssociated)<br>3. Non-existent IGW (error)<br>4. Missing params (MissingParameter) | **DONE** |
 | `detach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW must be attached to specified VPC | NATS `ec2.DetachInternetGateway` → daemon validates IGW exists and is attached to specified VPC (GatewayNotAttached if wrong VPC) → clears VpcId → state=available → publishes `vpc.igw-detach` event to vpcd (removes OVN resources) | 1. Detach attached IGW<br>2. Detach from wrong VPC (GatewayNotAttached)<br>3. Detach unattached IGW (error) | **DONE** |
 | `delete-internet-gateway` | `--internet-gateway-id` | `--dry-run` | IGW must be detached | NATS `ec2.DeleteInternetGateway` → daemon validates IGW exists and VpcId is empty (DependencyViolation if attached) → deletes from `spinifex-igw` KV | 1. Delete detached IGW<br>2. Delete attached IGW (DependencyViolation)<br>3. Delete non-existent (error) | **DONE** |
-| `describe-internet-gateways` | `--internet-gateway-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info | **DONE** |
+| `describe-internet-gateways` | `--internet-gateway-ids`, `--filters` (internet-gateway-id, attachment.vpc-id, attachment.state, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → applies filters → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info<br>4. Filter by attachment.vpc-id, attachment.state<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 
 ### EC2 - Route Tables
 
+Route tables are stored in `spinifex-vpc-route-tables` JetStream KV bucket, keyed by `{accountID}.{routeTableId}`. Each route table has a Routes array (with local route pre-populated) and an Associations array. All operations validate VPC consistency. When a route targeting a NAT gateway is created, `vpc.add-nat-gateway` events are published per associated subnet for vpcd SNAT rule creation.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-route-table` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateRouteTable` → daemon creates route table in KV with rtb-ID → add default local route for VPC CIDR → return route table | 1. Create route table<br>2. Verify default local route<br>3. Missing VPC ID (error) | **NOT STARTED** |
-| `delete-route-table` | — | `--route-table-id`, `--dry-run` | Route table must exist and not be main route table | NATS `ec2.DeleteRouteTable` → daemon verifies not main table → delete from KV → return success | 1. Delete non-main route table<br>2. Delete main route table (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
-| `describe-route-tables` | — | `--route-table-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeRouteTables` → daemon lists route tables from KV → return tables with routes | 1. List all route tables<br>2. Filter by VPC | **NOT STARTED** |
-| `create-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` or `--egress-only-internet-gateway-id` | Route table and target must exist | NATS `ec2.CreateRoute` → daemon adds route entry → configure OVS flow for destination CIDR → update metadata | 1. Add internet route via IGW<br>2. Add route to NAT gateway<br>3. Add route to EIGW<br>4. Conflicting route (error) | **NOT STARTED** |
-| `delete-route` | — | `--route-table-id`, `--destination-cidr-block` | Route must exist | NATS `ec2.DeleteRoute` → daemon removes route entry → remove OVS flow → update metadata | 1. Delete existing route<br>2. Delete non-existent route (error)<br>3. Delete local route (error) | **NOT STARTED** |
-| `replace-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` | Route must exist | NATS `ec2.ReplaceRoute` → daemon replaces route target → update OVS flow → update metadata | 1. Replace IGW route with NAT<br>2. Replace non-existent route (error) | **NOT STARTED** |
-| `associate-route-table` | — | `--route-table-id`, `--subnet-id` or `--gateway-id` | Route table and target must exist | NATS `ec2.AssociateRouteTable` → daemon links route table to subnet/gateway → return rtbassoc-ID | 1. Associate with subnet<br>2. Missing route table ID (error) | **NOT STARTED** |
-| `disassociate-route-table` | — | `--association-id` | Association must exist, not main table association | NATS `ec2.DisassociateRouteTable` → daemon unlinks route table → return success | 1. Disassociate existing<br>2. Disassociate main table (error) | **NOT STARTED** |
+| `create-route-table` | `--vpc-id` | `--tag-specifications`, `--dry-run` | VPC must exist | Gateway validates VpcId (required) → NATS `ec2.CreateRouteTable` with `spinifex-workers` queue group → daemon verifies VPC exists in `spinifex-vpc-vpcs` KV → retrieves VPC CIDR → generates rtb-ID → creates RouteTableRecord with IsMain=false, local route pre-populated (destination=VPC CIDR, gateway="local"), empty associations → stores in `spinifex-vpc-route-tables` KV → returns RouteTable | 1. Create route table with local route (happy path)<br>2. VPC not found (InvalidVpcID.NotFound)<br>3. Missing VpcId (MissingParameter) | **DONE** |
+| `delete-route-table` | `--route-table-id` | `--dry-run` | Route table must exist, not main, no subnet associations | Gateway validates RouteTableId (required) → NATS `ec2.DeleteRouteTable` → daemon retrieves record → rejects if IsMain=true (DependencyViolation) → rejects if has non-main subnet associations (DependencyViolation) → deletes from KV → returns success | 1. Delete custom route table<br>2. Delete main route table (DependencyViolation)<br>3. Delete table with subnet associations (DependencyViolation)<br>4. Delete non-existent (InvalidRouteTableID.NotFound) | **DONE** |
+| `describe-route-tables` | `--route-table-ids`, `--filters` (vpc-id, route-table-id, association.main, association.route-table-association-id, association.subnet-id, route.destination-cidr-block, route.gateway-id) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeRouteTables` → daemon scans all keys in `spinifex-vpc-route-tables` KV → filters by RouteTableIds if specified → applies all provided filters (AND logic) → returns error for non-existent requested IDs → converts records to EC2 RouteTable objects | 1. List all route tables<br>2. Filter by vpc-id<br>3. Filter by association.main (true/false)<br>4. Non-existent ID (InvalidRouteTableID.NotFound)<br>5. Empty result when no route tables | **DONE** |
+| `create-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id`, `--nat-gateway-id` | `--egress-only-internet-gateway-id`, `--vpc-peering-connection-id`, `--dry-run` | Route table and target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.CreateRoute` → daemon retrieves route table → checks for duplicate destination (RouteAlreadyExists) → validates exactly one of GatewayId/NatGatewayId provided → if GatewayId: verifies IGW exists in `spinifex-igw` KV and is attached to same VPC → if NatGatewayId: verifies NAT GW exists in `spinifex-vpc-nat-gateways` KV and is in same VPC → for NAT GW routes, publishes `vpc.add-nat-gateway` event per associated subnet → appends RouteRecord (state=active, origin=CreateRoute) → updates KV → returns Return=true | 1. Add IGW route (happy path)<br>2. Duplicate destination (RouteAlreadyExists)<br>3. IGW not found (InvalidInternetGatewayID.NotFound)<br>4. NAT GW not found (InvalidNatGatewayID.NotFound)<br>5. Gateway not in same VPC (InvalidParameterValue)<br>6. Missing RouteTableId or DestinationCidrBlock (MissingParameter) | **DONE** |
+| `delete-route` | `--route-table-id`, `--destination-cidr-block` | `--dry-run` | Route must exist | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.DeleteRoute` → daemon retrieves route table → finds matching route → rejects if gateway is "local" (InvalidParameterValue — local routes cannot be deleted) → removes route from Routes array → updates KV → returns success | 1. Delete IGW route (happy path, local route remains)<br>2. Delete local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. Missing parameters (MissingParameter) | **DONE** |
+| `replace-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` | `--nat-gateway-id`, `--dry-run` | Route must exist, new target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock + GatewayId (required) → NATS `ec2.ReplaceRoute` → daemon retrieves route table → finds matching route → rejects if local route → verifies IGW exists and is attached to same VPC → updates route GatewayId, clears NatGatewayId, sets state=active → updates KV → returns success | 1. Replace route target (IGW swap)<br>2. Replace local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. IGW not found (InvalidInternetGatewayID.NotFound) | **DONE** |
+| `associate-route-table` | `--route-table-id`, `--subnet-id` | `--gateway-id`, `--dry-run` | Route table and subnet must exist in same VPC | Gateway validates RouteTableId + SubnetId (required) → NATS `ec2.AssociateRouteTable` → daemon retrieves route table → verifies subnet exists and is in same VPC → scans ALL route tables in VPC to check subnet isn't already explicitly associated (Resource.AlreadyAssociated) → generates rtbassoc-ID → appends AssociationRecord (Main=false) → updates KV → returns AssociationId with state=associated | 1. Associate subnet (happy path)<br>2. Duplicate subnet association (Resource.AlreadyAssociated)<br>3. Subnet not found (InvalidSubnetID.NotFound)<br>4. Subnet in different VPC (InvalidParameterValue)<br>5. Missing parameters (MissingParameter) | **DONE** |
+| `disassociate-route-table` | `--association-id` | `--dry-run` | Association must exist, not main table association | Gateway validates AssociationId (required) → NATS `ec2.DisassociateRouteTable` → daemon scans ALL route tables in account to find matching AssociationId → rejects if Main=true (InvalidParameterValue — main associations are immutable) → removes association from Associations array → updates KV → returns success | 1. Disassociate subnet (happy path)<br>2. Disassociate main table (InvalidParameterValue)<br>3. Association not found (InvalidAssociationID.NotFound) | **DONE** |
+| `replace-route-table-association` | `--association-id`, `--route-table-id` | `--dry-run` | Association and new route table must exist | Gateway validates AssociationId + RouteTableId (required) → NATS `ec2.ReplaceRouteTableAssociation` → daemon verifies new route table exists → scans all route tables to find matching AssociationId → if Main=true: atomically swaps IsMain flag between old and new table → two-phase commit: removes association from old table, creates new AssociationId on new table → compensating write on failure (logs CRITICAL if compensation fails) → returns new AssociationId with state=associated | 1. Move subnet between tables (happy path)<br>2. Replace main table association (swaps IsMain flags)<br>3. Association not found (InvalidAssociationID.NotFound)<br>4. Route table not found (InvalidRouteTableID.NotFound) | **DONE** |
 
 ### EC2 - VPC Peering
 
@@ -376,7 +399,7 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `describe-network-interfaces` | `--network-interface-ids`, `--filters` (subnet-id, vpc-id, attachment.instance-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeNetworkInterfaces` → daemon lists all keys from `spinifex-vpc-enis` KV → applies filters (subnet-id, vpc-id, attachment.instance-id) → filters by ENI IDs if specified → returns error for non-existent requested IDs | 1. List all ENIs<br>2. Filter by subnet-id<br>3. Filter by vpc-id<br>4. Filter by attachment.instance-id<br>5. Non-existent ENI returns error | **DONE** |
 | `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | ENI and instance must exist | Internal operation used by `run-instances --subnet-id`. Marks ENI as "in-use", sets instance attachment, generates attachment ID. No standalone API endpoint yet. | 1. Attach ENI to instance<br>2. Already attached (error)<br>3. Missing device index (error) | **NOT STARTED** (internal only) |
 | `detach-network-interface` | — | `--attachment-id`, `--force` | ENI must be attached | Internal operation used by terminate-instances. Marks ENI as "available", clears attachment info. No standalone API endpoint yet. | 1. Detach ENI<br>2. Force detach<br>3. Not attached (error) | **NOT STARTED** (internal only) |
-| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--source-dest-check`, `--groups` | ENI must exist | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon updates ENI attributes in KV → return success | 1. Modify description<br>2. Toggle source/dest check<br>3. Update security groups | **NOT STARTED** |
+| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--groups` | ENI must exist; at least one attribute required | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon validates ENI exists → updates description and/or security groups in `spinifex-vpc-enis` KV (optimistic locking) → return success | 1. Modify description<br>2. Update security groups<br>3. No attributes (InvalidParameterValue)<br>4. Non-existent ENI (error) | **DONE** |
 | `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` or `--secondary-private-ip-address-count` | ENI must exist | NATS `ec2.AssignPrivateIpAddresses` → daemon allocates IPs from subnet pool → assign to ENI → return assigned IPs | 1. Assign specific IPs<br>2. Assign by count<br>3. Cannot specify both (error) | **NOT STARTED** |
 | `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | IPs must be assigned to ENI | NATS `ec2.UnassignPrivateIpAddresses` → daemon releases IPs → return success | 1. Unassign secondary IP<br>2. Unassign primary IP (error) | **NOT STARTED** |
 
@@ -409,19 +432,6 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `list-snapshot-blocks` | — | `--snapshot-id`, `--max-results`, `--next-token` | Snapshot must exist | List all non-empty blocks in snapshot with tokens and sizes | 1. List blocks of populated snapshot<br>2. List blocks of empty snapshot | **NOT STARTED** |
 | `list-changed-blocks` | — | `--second-snapshot-id`, `--first-snapshot-id`, `--max-results` | Snapshots must exist | Compare two snapshots → return list of differing block indexes | 1. Compare parent and child snapshots<br>2. Compare unrelated snapshots | **NOT STARTED** |
 
-### S3 (via Predastore)
-
-Currently S3 operations are handled directly by Predastore. Consider moving control/data plane to Spinifex format for AWS gateway integration.
-
-| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
-|---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `s3 mb` (CreateBucket) | — | `s3://bucket-name` | None | NATS `s3.createbucket` → Predastore creates bucket → return success | 1. Create valid bucket<br>2. Duplicate bucket name (error)<br>3. Invalid bucket name (error) | **NOT STARTED** (Predastore handles directly) |
-| `s3 rb` (DeleteBucket) | — | `s3://bucket-name`, `--force` | Bucket must exist | NATS `s3.deletebucket` → verify bucket empty (or force) → Predastore deletes bucket | 1. Delete empty bucket<br>2. Delete non-empty bucket (error)<br>3. Force delete non-empty bucket | **NOT STARTED** |
-| `s3 ls` (ListBuckets/Objects) | — | `s3://bucket/prefix`, `--recursive` | None | NATS `s3.listobjects` → Predastore lists objects with prefix → return list | 1. List all buckets<br>2. List objects in bucket<br>3. List with prefix filter | **NOT STARTED** |
-| `s3 cp` (PutObject/GetObject) | — | `source dest`, `--recursive`, `--acl` | Bucket must exist | NATS `s3.putobject`/`s3.getobject` → Predastore stores/retrieves with Reed-Solomon encoding | 1. Upload single file<br>2. Download single file<br>3. Recursive upload/download<br>4. Large file (multipart) | **NOT STARTED** |
-| `s3 rm` (DeleteObject) | — | `s3://bucket/key`, `--recursive` | Object must exist | NATS `s3.deleteobject` → Predastore deletes object shards | 1. Delete single object<br>2. Recursive delete<br>3. Delete non-existent (idempotent) | **NOT STARTED** |
-| `s3 sync` | — | `source dest`, `--delete`, `--exclude`, `--include` | Bucket must exist | Compare source and dest → upload changed/new files → optionally delete removed files | 1. Sync local dir to S3<br>2. Sync with --delete<br>3. Sync with exclude filter | **NOT STARTED** |
-
 ### IAM - User Management
 
 All IAM operations are account-scoped. Users, policies, and access keys are namespaced per account. The gateway extracts the account ID from the authenticated access key via `c.Locals("sigv4.accountId")` and passes it to all IAM service methods. Root user (account `000000000000`) bypasses policy evaluation entirely.
@@ -477,5 +487,140 @@ The gateway evaluates IAM policies on every request (except root user bypass). P
 | Explicit deny | Deny statements override Allow (deny wins) | **DONE** |
 | Wildcard matching | `*` matches all, `ec2:*` matches all EC2, `ec2:Describe*` matches prefix | **DONE** |
 | Account-scoped evaluation | Policies resolved from caller's account context | **DONE** |
-| EC2 action mapping | 54 EC2 actions mapped to IAM policy strings | **DONE** |
+| EC2 action mapping | All registered EC2 actions mapped to IAM policy strings | **DONE** |
 | IAM action mapping | 16 IAM actions mapped to IAM policy strings | **DONE** |
+
+### IAM - Roles
+
+IAM roles are required for instance profiles, cross-account access, and service-linked permissions. Without roles, workloads must use static access keys. Roles should be stored in a `spinifex-iam-roles` KV bucket keyed by `{accountID}.{roleName}`. Trust policies (AssumeRolePolicyDocument) define who can assume the role.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-role` | — | `--role-name`, `--assume-role-policy-document`, `--path`, `--description`, `--max-session-duration`, `--permissions-boundary`, `--tags` | None | IAMService.CreateRole(accountID, input) → validate RoleName + AssumeRolePolicyDocument (required) → validate trust policy JSON → generate AROA-prefixed RoleId → store RoleRecord in `spinifex-iam-roles` KV at `{accountID}.{roleName}` → ARN: `arn:aws:iam::{accountID}:role/{path}{roleName}` → return Role | 1. Create with valid trust policy<br>2. Malformed trust policy (MalformedPolicyDocument)<br>3. Duplicate name (EntityAlreadyExists)<br>4. Missing RoleName (InvalidInput)<br>5. With path and description | **NOT STARTED** |
+| `get-role` | — | `--role-name` | Role must exist | IAMService.GetRole(accountID, input) → reads from KV at `{accountID}.{roleName}` → returns Role with ARN, Path, AssumeRolePolicyDocument, CreateDate | 1. Get existing role<br>2. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-roles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListRoles(accountID, input) → prefix-scans KV for `{accountID}.*` → filters by path prefix → returns role list scoped to caller's account | 1. List all roles<br>2. List with path prefix<br>3. Empty list<br>4. Account isolation | **NOT STARTED** |
+| `delete-role` | — | `--role-name` | Role must exist, no attached policies, no instance profiles | IAMService.DeleteRole(accountID, input) → verify no attached policies (DeleteConflict) → verify not in any instance profile (DeleteConflict) → delete from KV | 1. Delete role with no attachments<br>2. Delete role with policies (DeleteConflict)<br>3. Delete role in instance profile (DeleteConflict)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-role` | — | `--role-name`, `--description`, `--max-session-duration` | Role must exist | IAMService.UpdateRole(accountID, input) → update description and/or session duration in KV → return success | 1. Update description<br>2. Update max session duration<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-assume-role-policy` | — | `--role-name`, `--policy-document` | Role must exist | IAMService.UpdateAssumeRolePolicy(accountID, input) → validate trust policy JSON → update AssumeRolePolicyDocument in KV → return success | 1. Update trust policy<br>2. Malformed policy (MalformedPolicyDocument)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `attach-role-policy` | — | `--role-name`, `--policy-arn` | Role and policy must exist | IAMService.AttachRolePolicy(accountID, input) → verify policy exists → add ARN to role's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent role (NoSuchEntity)<br>4. Non-existent policy (NoSuchEntity) | **NOT STARTED** |
+| `detach-role-policy` | — | `--role-name`, `--policy-arn` | Policy must be attached | IAMService.DetachRolePolicy(accountID, input) → remove ARN from role's AttachedPolicies | 1. Detach attached policy<br>2. Not attached (NoSuchEntity)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-role-policies` | — | `--role-name`, `--path-prefix`, `--max-items`, `--marker` | Role must exist | IAMService.ListAttachedRolePolicies(accountID, input) → reads role's AttachedPolicies list → returns ARNs and names | 1. List attached policies<br>2. No policies (empty list)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Instance Profiles
+
+Instance profiles are containers for IAM roles that allow EC2 instances to assume a role and obtain temporary credentials. Required for `run-instances --iam-instance-profile`. Stored in `spinifex-iam-instance-profiles` KV bucket keyed by `{accountID}.{profileName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-instance-profile` | — | `--instance-profile-name`, `--path`, `--tags` | None | IAMService.CreateInstanceProfile(accountID, input) → validate name (required) → generate AIPA-prefixed InstanceProfileId → store in `spinifex-iam-instance-profiles` KV with empty Roles array → ARN: `arn:aws:iam::{accountID}:instance-profile/{path}{name}` → return InstanceProfile | 1. Create profile<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-instance-profile` | — | `--instance-profile-name` | Profile must exist | IAMService.GetInstanceProfile(accountID, input) → reads from KV → returns InstanceProfile with Roles array | 1. Get existing profile<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-instance-profiles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListInstanceProfiles(accountID, input) → prefix-scans KV → returns list | 1. List all profiles<br>2. Empty list | **NOT STARTED** |
+| `list-instance-profiles-for-role` | — | `--role-name`, `--max-items`, `--marker` | Role must exist | IAMService.ListInstanceProfilesForRole(accountID, input) → scan all profiles → filter where Roles contains role name → return list | 1. List profiles for role<br>2. Role not in any profile (empty) | **NOT STARTED** |
+| `delete-instance-profile` | — | `--instance-profile-name` | Profile must exist, Roles must be empty | IAMService.DeleteInstanceProfile(accountID, input) → verify Roles empty (DeleteConflict if not) → delete from KV | 1. Delete empty profile<br>2. Delete profile with role (DeleteConflict)<br>3. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-role-to-instance-profile` | — | `--instance-profile-name`, `--role-name` | Both must exist, max 1 role per profile | IAMService.AddRoleToInstanceProfile(accountID, input) → verify profile exists → verify role exists → check Roles array is empty (LimitExceeded if role already present) → append role → update KV | 1. Add role to empty profile<br>2. Profile already has role (LimitExceeded)<br>3. Non-existent profile (NoSuchEntity)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `remove-role-from-instance-profile` | — | `--instance-profile-name`, `--role-name` | Role must be in profile | IAMService.RemoveRoleFromInstanceProfile(accountID, input) → remove role from Roles array → update KV | 1. Remove role<br>2. Role not in profile (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Groups
+
+Groups provide a way to attach policies to multiple users. Stored in `spinifex-iam-groups` KV bucket keyed by `{accountID}.{groupName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-group` | — | `--group-name`, `--path` | None | IAMService.CreateGroup(accountID, input) → validate GroupName (required) → generate AGPA-prefixed GroupId → store in KV → return Group | 1. Create group<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-group` | — | `--group-name` | Group must exist | IAMService.GetGroup(accountID, input) → reads group from KV → returns Group with member Users list | 1. Get existing group<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-groups` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListGroups(accountID, input) → prefix-scans KV → returns list | 1. List all groups<br>2. Empty list | **NOT STARTED** |
+| `delete-group` | — | `--group-name` | Group must exist, no members, no attached policies | IAMService.DeleteGroup(accountID, input) → verify no members → verify no attached policies → delete from KV | 1. Delete empty group<br>2. Delete group with members (DeleteConflict)<br>3. Delete group with policies (DeleteConflict)<br>4. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-user-to-group` | — | `--group-name`, `--user-name` | Both must exist | IAMService.AddUserToGroup(accountID, input) → verify group + user exist → add user to group's Members list → idempotent | 1. Add user<br>2. Already a member (idempotent)<br>3. Non-existent group (NoSuchEntity)<br>4. Non-existent user (NoSuchEntity) | **NOT STARTED** |
+| `remove-user-from-group` | — | `--group-name`, `--user-name` | User must be in group | IAMService.RemoveUserFromGroup(accountID, input) → remove user from Members list → update KV | 1. Remove member<br>2. Not a member (NoSuchEntity) | **NOT STARTED** |
+| `list-groups-for-user` | — | `--user-name`, `--max-items`, `--marker` | User must exist | IAMService.ListGroupsForUser(accountID, input) → scan all groups → filter where Members contains user → return list | 1. List groups for user<br>2. User in no groups (empty) | **NOT STARTED** |
+| `attach-group-policy` | — | `--group-name`, `--policy-arn` | Group and policy must exist | IAMService.AttachGroupPolicy(accountID, input) → verify policy exists → add ARN to group's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent group (NoSuchEntity) | **NOT STARTED** |
+| `detach-group-policy` | — | `--group-name`, `--policy-arn` | Policy must be attached | IAMService.DetachGroupPolicy(accountID, input) → remove ARN from AttachedPolicies | 1. Detach policy<br>2. Not attached (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-group-policies` | — | `--group-name`, `--path-prefix`, `--max-items`, `--marker` | Group must exist | IAMService.ListAttachedGroupPolicies(accountID, input) → reads group's AttachedPolicies → returns ARNs and names | 1. List attached policies<br>2. No policies (empty) | **NOT STARTED** |
+
+### STS (Security Token Service)
+
+STS provides temporary credentials and identity verification. `GetCallerIdentity` is called by every AWS SDK/CLI invocation. `AssumeRole` is required once IAM roles are implemented. STS needs its own gateway service endpoint (`sts` service in the `supportedServices` map) with a dedicated action map.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `get-caller-identity` | — | *(no flags)* | Valid SigV4 credentials | Gateway extracts accountID and username from authenticated request → returns Account, Arn (`arn:aws:iam::{accountID}:user/{userName}` or `arn:aws:sts::{accountID}:assumed-role/{roleName}/{sessionName}`), UserId. Note: a Spinifex-custom version exists at `spinifex.GetCallerIdentity` but needs a proper STS endpoint. | 1. Get identity for IAM user<br>2. Get identity for assumed role<br>3. Get identity for root user<br>4. Invalid credentials (AuthFailure) | **NOT STARTED** |
+| `assume-role` | — | `--role-arn`, `--role-session-name`, `--duration-seconds`, `--policy`, `--external-id`, `--serial-number`, `--token-code`, `--tags`, `--transitive-tag-keys` | Role must exist, caller must be allowed by trust policy | Gateway validates RoleArn + RoleSessionName (required) → looks up role → evaluates trust policy (AssumeRolePolicyDocument) against caller → generates temporary credentials (AccessKeyId with ASIA prefix, SecretAccessKey, SessionToken) → credentials expire after DurationSeconds (default 3600, max 43200) → return Credentials + AssumedRoleUser (ARN, AssumedRoleId) | 1. Assume role with valid trust policy<br>2. Trust policy denies caller (AccessDenied)<br>3. Non-existent role (NoSuchEntity)<br>4. Expired session token rejected at SigV4<br>5. Custom duration<br>6. Session tags | **NOT STARTED** |
+| `get-session-token` | — | `--duration-seconds`, `--serial-number`, `--token-code` | Valid IAM user credentials (not role) | Gateway validates caller is IAM user (not assumed role) → generates temporary credentials (ASIA prefix) → session inherits user's permissions → return Credentials | 1. Get token for IAM user<br>2. Duration within range<br>3. Reject if caller is assumed role | **NOT STARTED** |
+
+### ELBv2 (Elastic Load Balancing v2 — Application Load Balancer)
+
+All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key format: `lb.{lbId}`, `tg.{tgId}`, `listener.{listenerId}`. The data plane uses a system-managed ALB VM running HAProxy, launched automatically during CreateLoadBalancer. HAProxy configuration is pushed via NATS (`elbv2.alb.{lbId}.config` topic) whenever listeners or targets change. Agent readiness is polled via `elbv2.alb.{lbId}.ping`.
+
+#### ELBv2 - Load Balancers
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-load-balancer` | `--name`, `--subnets`, `--security-groups`, `--scheme` (internet-facing/internal), `--tags`, `--ip-address-type` (ipv4 only) | `--type` (hardcoded application), `--customer-owned-ipv4-pool`, `--dry-run` | None (subnets optional) | Gateway validates Name (required) → NATS `elbv2.CreateLoadBalancer` with `spinifex-workers` queue group → daemon validates scheme (internet-facing or internal, InvalidScheme if other) → checks duplicate name in account (DuplicateLoadBalancerName) → generates LB ID → builds DNS name `{name}-{lbId}.{region}.elb.spinifex.local` → if subnets provided, creates one ENI per subnet with `spinifex:managed-by=elbv2` tag (rolls back on failure) → if system AMI available, launches t3.nano ALB VM with cloud-init running `/usr/local/bin/alb-agent` → state=provisioning → background goroutine polls agent ping for ~5min → on agent ready: state=active, on timeout: state=failed → if no VM launcher: state=active immediately → persists to KV → returns LoadBalancer with ARN, DNSName, State | 1. Create with name (happy path)<br>2. Internal scheme<br>3. Invalid scheme (InvalidScheme)<br>4. Duplicate name (DuplicateLoadBalancerName)<br>5. Missing name (MissingParameter)<br>6. With tags<br>7. With subnets (ENI creation) | **DONE** |
+| `delete-load-balancer` | `--load-balancer-arn` | `--dry-run` | LB must exist | Gateway validates LoadBalancerArn (required) → NATS `elbv2.DeleteLoadBalancer` → daemon verifies LB exists and belongs to account → terminates ALB VM (async background) → deletes all listeners for this LB → deletes system-managed ENIs → deletes LB record from KV → returns success | 1. Delete existing LB<br>2. Not found (LoadBalancerNotFound)<br>3. Cascades listener cleanup<br>4. ENI cleanup | **DONE** |
+| `describe-load-balancers` | `--load-balancer-arns`, `--names` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeLoadBalancers` → daemon lists all LB records from KV → filters by account ID → applies ARN filter and/or Name filter → returns LoadBalancer list with State, DNSName, Scheme, VpcId, AvailabilityZones | 1. List all LBs<br>2. Filter by name<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Target Groups
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-target-group` | `--name`, `--protocol` (HTTP), `--port` (default 80), `--vpc-id`, `--target-type` (instance only), `--health-check-protocol`, `--health-check-port`, `--health-check-path` (default /), `--health-check-interval-seconds` (default 30), `--health-check-timeout-seconds` (default 5), `--healthy-threshold-count` (default 5), `--unhealthy-threshold-count` (default 2), `--matcher` (HttpCode, default 200), `--tags` | `--health-check-enabled`, `--protocol-version`, `--ip-address-type`, `--dry-run` | None | Gateway validates Name (required) → NATS `elbv2.CreateTargetGroup` → daemon checks duplicate name in same VPC (DuplicateTargetGroupName) → generates TG ID → creates HealthCheckConfig with provided values or defaults → stores record with empty Targets array → returns TargetGroup with ARN | 1. Create with defaults<br>2. Custom health check<br>3. Duplicate name (DuplicateTargetGroupName)<br>4. Missing name (MissingParameter) | **DONE** |
+| `delete-target-group` | `--target-group-arn` | `--dry-run` | TG must exist, not referenced by listener | Gateway validates TargetGroupArn (required) → NATS `elbv2.DeleteTargetGroup` → daemon verifies TG exists → scans all listeners to check if any DefaultActions reference this TG (TargetGroupInUse if referenced) → deletes from KV | 1. Delete TG<br>2. TG in use by listener (TargetGroupInUse)<br>3. Not found (TargetGroupNotFound) | **DONE** |
+| `describe-target-groups` | `--target-group-arns`, `--names`, `--load-balancer-arn` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeTargetGroups` → daemon lists all TG records → filters by account → applies ARN/Name filters → if LoadBalancerArn specified, lists listeners for that LB and includes only TGs referenced by those listeners → returns TargetGroup list | 1. List all TGs<br>2. Filter by LB ARN (via listener lookup)<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Targets
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `register-targets` | `--target-group-arn`, `--targets` (Id=instance-id, Port=optional override) | `--dry-run` | TG must exist, instances should exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.RegisterTargets` → daemon fetches TG → deduplicates against existing targets (key: `{id}:{port}`, idempotent) → resolves instance ID → private IP via VPC ENI lookup → adds target with state=initial → persists to KV → calls `pushConfigForTargetGroup()` to regenerate HAProxy config on all LBs using this TG → returns success | 1. Register targets<br>2. Idempotent re-registration<br>3. TG not found (TargetGroupNotFound)<br>4. Missing ARN or targets (MissingParameter) | **DONE** |
+| `deregister-targets` | `--target-group-arn`, `--targets` (Id, Port) | `--dry-run` | TG must exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.DeregisterTargets` → daemon fetches TG → builds removal set (key: `{id}:{port}`) → filters Targets array to keep only non-matching → persists to KV → calls `pushConfigForTargetGroup()` to update HAProxy config → returns success | 1. Deregister targets<br>2. TG not found (TargetGroupNotFound)<br>3. Missing parameters (MissingParameter) | **DONE** |
+| `describe-target-health` | `--target-group-arn`, `--targets` (optional filter to specific targets) | `--include` | TG must exist | Gateway validates TargetGroupArn (required) → NATS `elbv2.DescribeTargetHealth` → daemon fetches TG → builds filter set from Targets param (if specified) → for each target, returns TargetHealthDescription with Target (Id, Port) and TargetHealth (State, Description). Note: health state is stored in records, not actively polled — HAProxy runs its own independent health checks. | 1. Describe all targets<br>2. Filter to specific targets<br>3. TG not found (TargetGroupNotFound) | **DONE** |
+
+#### ELBv2 - Listeners
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-listener` | `--load-balancer-arn`, `--default-actions` (Type=forward, TargetGroupArn), `--protocol` (HTTP, default), `--port` (default 80) | `--ssl-policy`, `--certificates`, `--alpn-policy`, `--mutual-authentication`, `--dry-run` | LB must exist | Gateway validates LoadBalancerArn + DefaultActions (required) → NATS `elbv2.CreateListener` → daemon verifies LB exists → checks for duplicate port on same LB (DuplicateListener) → generates listener ID → builds ARN → stores ListenerRecord in KV → calls `pushConfig(lb)` to regenerate HAProxy config (creates frontend for listener port, backend for target group with health checks) → returns Listener with ARN, Protocol, Port, DefaultActions | 1. Create listener<br>2. Duplicate port (DuplicateListener)<br>3. LB not found (LoadBalancerNotFound)<br>4. Graceful handling when NATS unavailable | **DONE** |
+| `delete-listener` | `--listener-arn` | `--dry-run` | Listener must exist | Gateway validates ListenerArn (required) → NATS `elbv2.DeleteListener` → daemon verifies listener exists and belongs to account → deletes from KV → fetches parent LB → calls `pushConfig(lb)` to regenerate HAProxy config without this listener → returns success | 1. Delete listener<br>2. Not found (ListenerNotFound) | **DONE** |
+| `describe-listeners` | `--load-balancer-arn`, `--listener-arns` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeListeners` → daemon lists listeners → if LoadBalancerArn specified, filters to that LB → applies ARN filter → account isolation → returns Listener list | 1. Filter by LB ARN<br>2. Account isolation | **DONE** |
+
+#### ELBv2 - Not Implemented
+
+| Feature | Description | Priority | Status |
+|---------|-------------|----------|--------|
+| Listener rules | Path-based and hostname-based routing (CreateRule, DeleteRule, DescribeRules, ModifyRule) | High | **NOT STARTED** |
+| HTTPS/TLS termination | SSL certificates, SSL policies, ALPN | High | **NOT STARTED** |
+| Modify operations | ModifyLoadBalancerAttributes, ModifyTargetGroup, ModifyTargetGroupAttributes, ModifyListener | Medium | **NOT STARTED** |
+| Connection draining | Deregistration delay for graceful target removal | Medium | **NOT STARTED** |
+| Stickiness | Session affinity / sticky sessions on target groups | Medium | **NOT STARTED** |
+| Active health checking | API-driven health state updates (currently HAProxy-only) | Medium | **NOT STARTED** |
+| Target types | IP and Lambda target types (only instance supported) | Low | **NOT STARTED** |
+| Access logs | S3 access log delivery | Low | **NOT STARTED** |
+| WAF integration | AWS WAF association for web application firewall | Low | **NOT STARTED** |
+
+### CloudWatch (Basic Monitoring)
+
+Basic CloudWatch metrics support is required for `monitor-instances`/`unmonitor-instances` to be meaningful. Daemon nodes should publish CPU, disk, and network metrics for running instances. Metrics stored in a time-series-friendly structure in JetStream KV or dedicated NATS subjects.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `put-metric-data` | — | `--namespace`, `--metric-data` (MetricName, Value, Unit, Timestamp, Dimensions) | None | Gateway validates Namespace + MetricData (required) → NATS `cloudwatch.PutMetricData` → daemon stores metric data points in time-series KV (key: `{namespace}.{metricName}.{dimensionHash}`) → return success | 1. Put single metric<br>2. Put with dimensions (InstanceId)<br>3. Missing namespace (MissingParameter) | **NOT STARTED** |
+| `get-metric-statistics` | — | `--namespace`, `--metric-name`, `--start-time`, `--end-time`, `--period`, `--statistics` (Average, Sum, Min, Max, SampleCount), `--dimensions` | None | Gateway validates required fields → NATS `cloudwatch.GetMetricStatistics` → daemon queries time-series data → aggregates by period → returns Datapoints with requested statistics | 1. Get CPU utilization over 1 hour<br>2. Get with dimensions (InstanceId)<br>3. No data returns empty | **NOT STARTED** |
+| `list-metrics` | — | `--namespace`, `--metric-name`, `--dimensions`, `--recently-active` | None | NATS `cloudwatch.ListMetrics` → daemon scans metric namespaces → returns Metrics list with Namespace, MetricName, Dimensions | 1. List all metrics<br>2. Filter by namespace<br>3. Filter by metric name | **NOT STARTED** |
+| `describe-alarms` | — | `--alarm-names`, `--alarm-name-prefix`, `--state-value`, `--action-prefix` | None | NATS `cloudwatch.DescribeAlarms` → daemon lists alarms from KV → return MetricAlarms with State, Threshold, ComparisonOperator | 1. List all alarms<br>2. Filter by state (OK, ALARM, INSUFFICIENT_DATA) | **NOT STARTED** |
+| `put-metric-alarm` | — | `--alarm-name`, `--namespace`, `--metric-name`, `--statistic`, `--period`, `--evaluation-periods`, `--threshold`, `--comparison-operator`, `--alarm-actions`, `--dimensions` | None | NATS `cloudwatch.PutMetricAlarm` → daemon stores alarm config in KV → alarm evaluation runs on metric publish → state transitions trigger alarm actions | 1. Create alarm<br>2. Update existing alarm<br>3. Missing required fields | **NOT STARTED** |
+| `delete-alarms` | — | `--alarm-names` | Alarms must exist | NATS `cloudwatch.DeleteAlarms` → daemon deletes alarm records from KV → return success | 1. Delete existing alarm<br>2. Non-existent alarm (ResourceNotFound) | **NOT STARTED** |
+
+### Auto Scaling
+
+Auto Scaling manages instance fleets based on scaling policies. Depends on Launch Templates (EC2) and optionally ELBv2 target groups for health-based scaling. Stored in `spinifex-autoscaling` KV bucket.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-auto-scaling-group` | — | `--auto-scaling-group-name`, `--launch-template` (Id/Name, Version), `--min-size`, `--max-size`, `--desired-capacity`, `--vpc-zone-identifier` (subnet IDs), `--target-group-arns`, `--health-check-type` (EC2/ELB), `--health-check-grace-period`, `--tags` | Launch template must exist | NATS `autoscaling.CreateAutoScalingGroup` → daemon validates launch template → stores ASG record in KV → launches DesiredCapacity instances using template → registers with target groups if specified → starts health check loop → return success | 1. Create ASG with launch template<br>2. Scale to desired capacity<br>3. Missing launch template (ValidationError)<br>4. Min > Max (ValidationError) | **NOT STARTED** |
+| `update-auto-scaling-group` | — | `--auto-scaling-group-name`, `--min-size`, `--max-size`, `--desired-capacity`, `--launch-template`, `--health-check-type`, `--health-check-grace-period` | ASG must exist | NATS `autoscaling.UpdateAutoScalingGroup` → daemon updates ASG config in KV → if DesiredCapacity changed, scale up/down accordingly → return success | 1. Update desired capacity<br>2. Update launch template<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-auto-scaling-group` | — | `--auto-scaling-group-name`, `--force-delete` | ASG must exist | NATS `autoscaling.DeleteAutoScalingGroup` → if force: terminate all instances → deregister from target groups → delete from KV → return success. Without force: reject if instances > 0 | 1. Delete empty ASG<br>2. Force delete with instances<br>3. Delete with instances (ResourceInUse) | **NOT STARTED** |
+| `describe-auto-scaling-groups` | — | `--auto-scaling-group-names`, `--filters`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingGroups` → daemon lists ASG records → return list with Instances, DesiredCapacity, MinSize, MaxSize | 1. List all ASGs<br>2. Filter by name | **NOT STARTED** |
+| `set-desired-capacity` | — | `--auto-scaling-group-name`, `--desired-capacity`, `--honor-cooldown` | ASG must exist | NATS `autoscaling.SetDesiredCapacity` → daemon updates desired count → scale up (launch) or scale down (terminate oldest) → return success | 1. Scale up<br>2. Scale down<br>3. Exceeds max (ValidationError) | **NOT STARTED** |
+| `describe-auto-scaling-instances` | — | `--instance-ids`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingInstances` → daemon lists all ASG-managed instances → return list with ASG name, lifecycle state, health status | 1. List all ASG instances<br>2. Filter by instance ID | **NOT STARTED** |
+| `put-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name`, `--policy-type` (TargetTrackingScaling, StepScaling, SimpleScaling), `--target-tracking-configuration`, `--scaling-adjustment`, `--cooldown` | ASG must exist | NATS `autoscaling.PutScalingPolicy` → daemon stores policy in KV → links to CloudWatch alarms for metric-based triggers → return PolicyARN | 1. Create target tracking policy<br>2. Create step scaling policy<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name` | Policy must exist | NATS `autoscaling.DeleteScalingPolicy` → daemon deletes policy → remove CloudWatch alarm links → return success | 1. Delete existing policy<br>2. Non-existent policy | **NOT STARTED** |

docs/creating-accounts/README.md

@@ -32,16 +32,10 @@ Spinifex supports multi-tenant account isolation. Each account gets its own IAM
 ## Create Account
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
-## Verify
-
-```bash
-aws sts get-caller-identity
-```
-
 ## Troubleshooting
 
 ## Credentials Not Working

docs/creating-an-environment/README.md

@@ -1,110 +0,0 @@
----
-title: "Creating an Environment"
-description: "End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances."
-category: "Environments"
-tags:
-  - environment
-  - vpc
-  - ec2
-  - ssh
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Debian Cloud Images"
-    url: "https://cloud.debian.org/images/cloud/bookworm/latest/"
----
-
-# Creating an Environment
-
-> End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-This guide walks through the complete workflow for creating a production-ready Spinifex environment.
-
-## Instructions
-
-## 1. Import SSH Key
-
-```bash
-aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub
-```
-
-## 2. Create VPC
-
-```bash
-aws ec2 create-vpc --cidr-block 10.200.0.0/16
-export SPINIFEX_VPC="vpc-XXX"
-
-aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24
-export SPINIFEX_SUBNET="subnet-XXX"
-```
-
-## 3. Import AMI
-
-```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
-export SPINIFEX_AMI="ami-XXX"
-```
-
-## 4. Launch Instance
-
-```bash
-aws ec2 run-instances \
-  --image-id $SPINIFEX_AMI \
-  --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
-```
-
-## 5. Connect via SSH
-
-```bash
-ps auxw | grep $INSTANCE_ID
-# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
-
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```
-
-## Troubleshooting
-
-## Instance Stuck in Pending
-
-Check the Spinifex daemon and QEMU logs for boot errors:
-
-```bash
-ls ~/spinifex/logs/
-cat ~/spinifex/logs/spinifex.log
-```
-
-Verify the AMI exists and architecture matches your host:
-
-```bash
-aws ec2 describe-images --image-ids $SPINIFEX_AMI
-```
-
-## SSH Connection Refused
-
-cloud-init takes 30-60 seconds to configure networking and SSH after boot. Wait and retry.
-
-Verify the SSH port forwarding is active:
-
-```bash
-ps auxw | grep hostfwd
-```
-
-Ensure you're using the correct key and port:
-
-```bash
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```

docs/hybrid-sync/README.md

@@ -29,6 +29,14 @@ Spinifex's hybrid mode enables bidirectional data synchronization between local
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Push Local to Cloud
 
 ```bash
@@ -61,14 +69,3 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \
 ```
 
 Only changed or missing files will be transferred on subsequent runs.
-
-## Permission Errors During Sync
-
-Verify your AWS credentials are configured for both the source and destination:
-
-```bash
-aws sts get-caller-identity --profile spinifex
-aws sts get-caller-identity --profile default
-```
-
-Ensure the IAM user has `s3:GetObject`, `s3:PutObject`, and `s3:ListBucket` permissions on both buckets.

docs/iam-users-and-policies/README.md

@@ -53,7 +53,6 @@ When you create an account with `spx admin account create`, Spinifex bootstraps
 
 ```bash
 export AWS_PROFILE=spinifex-myteam
-aws sts get-caller-identity
 ```
 
 ## Users
@@ -68,13 +67,13 @@ The response includes the user's ARN, unique ID, and creation date:
 
 ```json
 {
-    "User": {
-        "UserName": "alice",
-        "UserId": "AIDA1A2B3C4D5E6F7890",
-        "Arn": "arn:aws:iam::000000000001:user/alice",
-        "Path": "/",
-        "CreateDate": "2026-03-24T10:00:00Z"
-    }
+  "User": {
+    "UserName": "alice",
+    "UserId": "AIDA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:user/alice",
+    "Path": "/",
+    "CreateDate": "2026-03-24T10:00:00Z"
+  }
 }
 ```
 
@@ -120,7 +119,6 @@ aws iam detach-user-policy --user-name alice \
 aws iam delete-user --user-name alice
 ```
 
-
 ## Access Keys
 
 Access keys are how users authenticate with the AWS CLI. Each user can have up to **2 access keys** at a time, allowing key rotation without downtime.
@@ -133,13 +131,13 @@ aws iam create-access-key --user-name alice
 
 ```json
 {
-    "AccessKey": {
-        "UserName": "alice",
-        "AccessKeyId": "AKIA1A2B3C4D5E6F7890ABCD",
-        "Status": "Active",
-        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-        "CreateDate": "2026-03-24T10:05:00Z"
-    }
+  "AccessKey": {
+    "UserName": "alice",
+    "AccessKeyId": "AKIA1A2B3C4D5E6F7890ABCD",
+    "Status": "Active",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "CreateDate": "2026-03-24T10:05:00Z"
+  }
 }
 ```
 
@@ -153,7 +151,6 @@ Add the credentials to `~/.aws/credentials` under a new profile and set `AWS_PRO
 aws iam list-access-keys --user-name alice
 ```
 
-
 ### Rotate Access Keys
 
 Create a second key, update your configuration, then delete the old one:
@@ -189,7 +186,6 @@ aws iam update-access-key --user-name alice \
   --status Active
 ```
 
-
 ## Policies
 
 Policies are JSON documents that define what actions a user can perform on which resources.
@@ -198,14 +194,14 @@ Policies are JSON documents that define what actions a user can perform on which
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": ["ec2:RunInstances", "ec2:DescribeInstances"],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:RunInstances", "ec2:DescribeInstances"],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -249,32 +245,31 @@ aws iam create-policy \
 
 ```json
 {
-    "Policy": {
-        "PolicyName": "EC2ReadOnly",
-        "PolicyId": "ANPA1A2B3C4D5E6F7890",
-        "Arn": "arn:aws:iam::000000000001:policy/EC2ReadOnly",
-        "Path": "/",
-        "DefaultVersionId": "v1",
-        "CreateDate": "2026-03-24T10:10:00Z"
-    }
+  "Policy": {
+    "PolicyName": "EC2ReadOnly",
+    "PolicyId": "ANPA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:policy/EC2ReadOnly",
+    "Path": "/",
+    "DefaultVersionId": "v1",
+    "CreateDate": "2026-03-24T10:10:00Z"
+  }
 }
 ```
 
-
 ### Common Policy Examples
 
 **Full administrator access:**
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": "*",
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "*",
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -282,19 +277,19 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:GetObject",
-                "s3:PutObject",
-                "s3:ListBucket",
-                "s3:DeleteObject"
-            ],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:ListBucket",
+        "s3:DeleteObject"
+      ],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -302,21 +297,21 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-                "ec2:RunInstances",
-                "ec2:StartInstances",
-                "ec2:StopInstances",
-                "ec2:TerminateInstances",
-                "ec2:DescribeInstances",
-                "ec2:DescribeImages"
-            ],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:RunInstances",
+        "ec2:StartInstances",
+        "ec2:StopInstances",
+        "ec2:TerminateInstances",
+        "ec2:DescribeInstances",
+        "ec2:DescribeImages"
+      ],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -324,14 +319,14 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Deny",
-            "Action": "ec2:TerminateInstances",
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": "ec2:TerminateInstances",
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -352,7 +347,6 @@ aws iam get-policy-version \
   --version-id v1
 ```
 
-
 ### List Policies
 
 ```bash
@@ -368,7 +362,6 @@ aws iam delete-policy \
   --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
 ```
 
-
 ## Attaching Policies
 
 Policies have no effect until attached to a user.
@@ -380,7 +373,6 @@ aws iam attach-user-policy --user-name alice \
   --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
 ```
 
-
 ### List a User's Policies
 
 ```bash
@@ -453,24 +445,24 @@ AWS_PROFILE=spinifex-carol aws ec2 terminate-instances --instance-ids i-123
 
 ## Command Reference
 
-| Command | Key Parameters | Notes |
-|---------|---------------|-------|
-| `create-user` | `--user-name`, `--path` | Path default: `/` |
-| `get-user` | `--user-name` | |
-| `list-users` | `--path-prefix` | |
-| `delete-user` | `--user-name` | Must have no keys or policies |
-| `create-access-key` | `--user-name` | Max 2 per user; secret shown once |
-| `list-access-keys` | `--user-name` | |
-| `delete-access-key` | `--user-name`, `--access-key-id` | |
-| `update-access-key` | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive` |
-| `create-policy` | `--policy-name`, `--policy-document` | Max 6144 bytes |
-| `get-policy` | `--policy-arn` | Metadata only |
-| `get-policy-version` | `--policy-arn`, `--version-id` | Use `v1`; includes document |
-| `list-policies` | | |
-| `delete-policy` | `--policy-arn` | Must not be attached |
-| `attach-user-policy` | `--user-name`, `--policy-arn` | Idempotent |
-| `detach-user-policy` | `--user-name`, `--policy-arn` | |
-| `list-attached-user-policies` | `--user-name` | |
+| Command                       | Key Parameters                               | Notes                             |
+| ----------------------------- | -------------------------------------------- | --------------------------------- |
+| `create-user`                 | `--user-name`, `--path`                      | Path default: `/`                 |
+| `get-user`                    | `--user-name`                                |                                   |
+| `list-users`                  | `--path-prefix`                              |                                   |
+| `delete-user`                 | `--user-name`                                | Must have no keys or policies     |
+| `create-access-key`           | `--user-name`                                | Max 2 per user; secret shown once |
+| `list-access-keys`            | `--user-name`                                |                                   |
+| `delete-access-key`           | `--user-name`, `--access-key-id`             |                                   |
+| `update-access-key`           | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive`            |
+| `create-policy`               | `--policy-name`, `--policy-document`         | Max 6144 bytes                    |
+| `get-policy`                  | `--policy-arn`                               | Metadata only                     |
+| `get-policy-version`          | `--policy-arn`, `--version-id`               | Use `v1`; includes document       |
+| `list-policies`               |                                              |                                   |
+| `delete-policy`               | `--policy-arn`                               | Must not be attached              |
+| `attach-user-policy`          | `--user-name`, `--policy-arn`                | Idempotent                        |
+| `detach-user-policy`          | `--user-name`, `--policy-arn`                |                                   |
+| `list-attached-user-policies` | `--user-name`                                |                                   |
 
 ## Troubleshooting
 

docs/install-airgapped/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Air-Gapped Installation"
+title: "Air-Gapped Install"
 description: "Deploy Spinifex in environments without internet connectivity. Covers offline package caching, USB deployment, and package verification."
 category: "Getting Started"
 tags:
@@ -85,7 +85,7 @@ cp -r /mnt/usb/spinifex-source ~/Development/mulga/spinifex
 cd ~/Development/mulga/spinifex && make build
 ```
 
-Then follow the [Source Install](/docs/source-install) guide from Step 3 (Setup OVN) onwards.
+Then follow the [Source Install](/docs/install-source) guide from Step 3 (Setup OVN) onwards.
 
 ## Troubleshooting
 

docs/install-multi-node/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Multi-Node Installation"
+title: "Multi-Node Install"
 description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
 category: "Getting Started"
 tags:
@@ -42,7 +42,7 @@ A Spinifex cluster distributes services across multiple servers for high availab
 ## Step 1. Install Spinifex on Each Server
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 ```
 
 ## Step 2. Set Node IP Variables

docs/install-source/README.md

@@ -13,7 +13,7 @@ resources:
     url: "https://go.dev/dl/"
 ---
 
-# Source Install
+# Source Installation
 
 > Build Spinifex from source for development, custom builds, or contributing.
 
@@ -27,7 +27,7 @@ resources:
 
 ## Overview
 
-This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/installing-spinifex) is recommended.
+This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/install) is recommended.
 
 **Requirements:**
 
@@ -57,15 +57,7 @@ cd spinifex
 
 Confirm `./bin/spx` exists.
 
-## Step 3. Initialize
-
-```bash
-sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
-```
-
-Save the admin credentials printed during init.
-
-## Step 4. Setup OVN
+## Step 3. Setup OVN
 
 If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:
 
@@ -83,6 +75,14 @@ If your WAN is a physical NIC, choose one:
 ./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
+## Step 4. Initialize
+
+```bash
+sudo ./bin/spx admin init --node node1 --nodes 1
+```
+
+Save the admin credentials printed during init.
+
 ## Step 5. Start Services
 
 ```bash

docs/install/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Installing Spinifex"
+title: "Single-Node Install"
 description: "Install Spinifex on a single server using the binary installer."
 category: "Getting Started"
 tags:
@@ -31,14 +31,14 @@ resources:
 
 Spinifex is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments.
 
-This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Installation](/docs/installing-spinifex-multi-node). To build from source, see [Source Install](/docs/source-install).
+This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Install](/docs/install-multi-node). To build from source, see [Source Install](/docs/install-source).
 
 **Supported Operating Systems:**
 
 - Ubuntu 22.04 / 24.04 / 25.10
 - Debian 12 / 13
 
-**What gets installed:**
+**What Gets Installed:**
 
 - Spinifex daemon and CLI
 - QEMU/KVM for virtual machine management
@@ -52,7 +52,7 @@ This guide installs Spinifex on a single server using the binary installer. For
 ## Step 1. Install Spinifex
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 ```
 
 The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
@@ -70,15 +70,20 @@ If your WAN is a physical NIC, choose one:
 ```bash
 # Dedicated WAN NIC (not your SSH connection):
 sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
-
-# Single-NIC host (SSH-safe macvlan):
-sudo /usr/local/share/spinifex/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
+**Separating VPC tunnel traffic from WAN:** If you want Geneve tunnel traffic (inter-VM east-west for VPC traffic) to use a dedicated interface instead of the WAN IP, add `--encap-ip=<IP>` to specify the tunnel endpoint address:
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --encap-ip=10.0.0.1
+```
+
+This is recommended for production and required in multi-node deployments. See [Multi-Node Install](/docs/install-multi-node) for details.
+
 ## Step 3. Initialize
 
 ```bash
-sudo spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+sudo spx admin init --node node1 --nodes 1
 ```
 
 This auto-detects your network topology, generates configuration and TLS certificates, installs the CA into the system trust store, and configures AWS CLI credentials. Save the admin credentials printed during init — they will not be shown again.

docs/launching-instances/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Launching Instances"
 description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex."
-category: "Environments"
+category: "Compute"
 tags:
   - ec2
   - instances
@@ -20,8 +20,16 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
-- [Instructions](#instructions)
+- [Launch](#launch)
+- [Manage](#manage)
+- [Reboot](#reboot)
+- [Modify Instance Attributes](#modify-instance-attributes)
+- [Console Output](#console-output)
+- [Instance Types](#instance-types)
+- [SSH (Development)](#ssh-development)
 - [Troubleshooting](#troubleshooting)
+  - [Instance Fails to Boot](#instance-fails-to-boot)
+  - [Cannot SSH Into Instance](#cannot-ssh-into-instance)
 
 ---
 
@@ -34,19 +42,29 @@ Spinifex provides EC2-compatible VM management built on QEMU/KVM. Instances supp
 - `run-instances` — Launch new VMs
 - `describe-instances` — Query state
 - `stop-instances` / `start-instances` — Lifecycle
+- `reboot-instances` — In-place restart (QMP reset)
 - `terminate-instances` — Permanent removal
+- `modify-instance-attribute` — Change type, user data, or source/dest check
+- `get-console-output` — Retrieve serial console log
+- `describe-instance-types` — List available instance types
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Launch
 
 ```bash
 aws ec2 run-instances \
   --image-id $SPINIFEX_AMI \
   --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
+  --key-name spinifex-key
 
 export INSTANCE_ID="i-XXX"
 ```
@@ -58,15 +76,72 @@ aws ec2 describe-instances --instance-ids $INSTANCE_ID
 aws ec2 stop-instances --instance-ids $INSTANCE_ID
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 reboot-instances --instance-ids $INSTANCE_ID
+```
+
+## Modify Instance Attributes
+
+Change instance type, user data, or source/dest check. Instance type and user data require the instance to be **stopped** first.
+
+### Change Instance Type
+
+```bash
+aws ec2 stop-instances --instance-ids $INSTANCE_ID
+
+aws ec2 modify-instance-attribute \
+  --instance-id $INSTANCE_ID \
+  --instance-type t3.medium
+
+aws ec2 start-instances --instance-ids $INSTANCE_ID
+```
+
+## Console Output
+
+Retrieve the serial console log for a running instance. Output is base64-encoded.
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID
+```
+
+Decode the output:
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID \
+  --query 'Output' --output text | base64 -d
+```
+
+## Instance Types
+
+List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.
+
+```bash
+aws ec2 describe-instance-types
+```
+
+Filter to a specific type:
+
+```bash
+aws ec2 describe-instance-types --instance-types t3.micro t3.small
+```
+
+Show capacity (how many of each type can still be launched):
+
+```bash
+aws ec2 describe-instance-types \
+  --filters Name=capacity,Values=true
 ```
 
 ## SSH (Development)
 
+In development mode, find the QEMU port forward and connect via localhost:
+
 ```bash
 ps auxw | grep $INSTANCE_ID
 ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
 ```
 
+For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
+
 ## Troubleshooting
 
 ### Instance Fails to Boot
@@ -81,8 +156,8 @@ aws ec2 describe-images --image-ids $SPINIFEX_AMI
 If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), import the correct image:
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
+spx admin images list
+spx admin images import --name debian-12-x86_64
 ```
 
 ### Cannot SSH Into Instance

docs/moving-aws-workload/README.md

@@ -37,10 +37,17 @@ Spinifex provides drop-in compatibility with AWS APIs, making it possible to mig
 
 ## Instructions
 
-## Configure AWS CLI
+## Prerequisites
+
+Ensure the AWS profile is set:
 
 ```bash
 export AWS_PROFILE=spinifex
+```
+
+## Configure AWS CLI
+
+```bash
 aws ec2 describe-instances
 aws s3 ls
 ```

docs/placement-groups/README.md

@@ -0,0 +1,121 @@
+---
+title: "Placement Groups"
+description: "Create and manage spread and cluster placement groups for hardware-level instance placement control."
+category: "Compute"
+tags:
+  - ec2
+  - placement
+  - spread
+  - cluster
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS Placement Groups"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html"
+---
+
+# Placement Groups
+
+> Create and manage spread and cluster placement groups for hardware-level instance placement control.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Create](#create)
+- [Launch Into a Placement Group](#launch-into-a-placement-group)
+- [Describe](#describe)
+- [Delete](#delete)
+- [Troubleshooting](#troubleshooting)
+  - [InsufficientInstanceCapacity Error](#insufficientinstancecapacity-error)
+  - [Cannot Delete Placement Group](#cannot-delete-placement-group)
+
+---
+
+## Overview
+
+Placement groups control how Spinifex distributes instances across physical hosts. Two strategies are supported:
+
+- **Spread** — One instance per physical host, maximising fault isolation
+- **Cluster** — All instances on the same host, minimising latency
+
+The `partition` strategy is not supported.
+
+**Supported operations:**
+
+- `create-placement-group` — Create a new group
+- `describe-placement-groups` — Query groups with optional filters
+- `delete-placement-group` — Remove an empty group
+
+## Instructions
+
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Create
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-spread-group \
+  --strategy spread
+```
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-cluster-group \
+  --strategy cluster
+```
+
+## Launch Into a Placement Group
+
+```bash
+aws ec2 run-instances \
+  --image-id $SPINIFEX_AMI \
+  --instance-type t3.small \
+  --key-name spinifex-key \
+  --placement GroupName=my-spread-group \
+  --count 3
+```
+
+## Describe
+
+```bash
+aws ec2 describe-placement-groups
+aws ec2 describe-placement-groups --group-names my-spread-group
+aws ec2 describe-placement-groups --group-ids pg-abc123
+aws ec2 describe-placement-groups \
+  --filters Name=strategy,Values=spread
+```
+
+## Delete
+
+The group must have no running instances before it can be deleted:
+
+```bash
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 delete-placement-group --group-name my-spread-group
+```
+
+## Troubleshooting
+
+### InsufficientInstanceCapacity Error
+
+Not enough distinct physical hosts for a spread launch. Reduce `--count` or terminate existing instances to free host slots:
+
+```bash
+spx admin nodes list
+```
+
+### Cannot Delete Placement Group
+
+The group still has running instances. Terminate them first:
+
+```bash
+aws ec2 describe-instances \
+  --filters Name=placement-group-name,Values=my-spread-group
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+```

docs/setting-up-your-cluster/README.md

@@ -33,9 +33,9 @@ resources:
 ## Overview
 
 This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
-- [Binary Install (Single Node)](/docs/installing-spinifex)
-- [Multi-Node Installation](/docs/installing-spinifex-multi-node)
-- [Source Install](/docs/source-install)
+- [Binary Install (Single Node)](/docs/install)
+- [Multi-Node Install](/docs/install-multi-node)
+- [Source Install](/docs/install-source)
 
 ## Instructions
 

docs/spinifex-admin-cli/README.md

@@ -27,7 +27,7 @@ resources:
 
 The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
 
-**Binary location:** `~/Development/mulga/spinifex/bin/spx`
+**Binary location:** `/usr/local/bin/spx` (binary install) or `~/Development/mulga/spinifex/bin/spx` (source install)
 
 All services in the Spinifex platform are managed through this single binary.
 
@@ -36,14 +36,14 @@ All services in the Spinifex platform are managed through this single binary.
 ## Account Management
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
 ## Node Management
 
 ```bash
-./bin/spx get nodes
+spx get nodes
 ```
 
 ```
@@ -56,20 +56,20 @@ node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
 ## Monitor Resources
 
 ```bash
-./bin/spx top nodes
+spx top nodes
 ```
 
 ## Image Management
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-arm64
+spx admin images list
+spx admin images import --name debian-12-arm64
 ```
 
 ## Cluster Shutdown
 
 ```bash
-./bin/spx admin cluster shutdown
+spx admin cluster shutdown
 ```
 
 ## Troubleshooting
@@ -79,7 +79,7 @@ node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
 The binary may not be executable. Fix permissions:
 
 ```bash
-chmod +x ./bin/spx
+chmod +x /usr/local/bin/spx
 ```
 
 If you get permission errors during operations, ensure you're running with appropriate privileges. Some OVN and networking commands require `sudo`.

docs/terraform/bastion-private-subnet/README.md

@@ -0,0 +1,158 @@
+---
+title: "Bastion with Private Subnet"
+description: "Deploy a bastion host to access an isolated private compute instance with no internet connectivity."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - bastion
+  - vpc
+  - security
+  - private subnet
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Bastion with Private Subnet
+
+> Deploy a bastion host to access an isolated private compute instance with no internet connectivity.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a VPC with public and private subnets where the private subnet has no internet access. A bastion host in the public subnet is the only way to reach instances in the private subnet. This pattern is used for sensitive workloads that must remain isolated from the internet — the private instance cannot make or receive any connections outside the VPC.
+
+**Architecture:**
+
+```
+WAN ──SSH──▶ Bastion (public subnet)
+                 │
+                 ▼ SSH (private IP)
+              App Server (private subnet, no internet)
+```
+
+**What you'll learn:**
+
+- Creating public and private subnets in a VPC
+- Isolating compute instances from the internet using route tables and security groups
+- Using a bastion host as the sole access point to private resources
+- SSH hopping through a bastion to reach private instances
+
+**Use cases:**
+
+- Sensitive data processing that must not have internet egress
+- Internal services that should only be reachable within the VPC
+- Compliance workloads requiring network isolation
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/bastion-private-subnet
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Connect
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry.
+
+SSH into the bastion:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_public_ip>
+```
+
+From the bastion, SSH to the private instance. The key is pre-installed at `~/.ssh/bastion-demo.pem` via cloud-init:
+
+```bash
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+### Step 4. Verify Isolation
+
+From the private instance, confirm there is no internet connectivity:
+
+```bash
+# This should time out — the private instance has no route to the internet
+curl --connect-timeout 5 https://deb.debian.org || echo "No internet access (expected)"
+```
+
+The private instance can only communicate within the VPC (`10.20.0.0/16`). Its security group restricts egress to VPC-internal traffic only.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Cannot SSH to Private Instance
+
+The private instance has no public IP — it is only reachable from the bastion. SSH to the bastion first, then use the pre-installed key to hop:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+If the key is missing on the bastion, check that cloud-init completed successfully:
+
+```bash
+sudo journalctl -u cloud-init --no-pager
+ls -la ~/.ssh/bastion-demo.pem
+```
+
+### Private Instance Can Reach the Internet
+
+If the private instance unexpectedly has internet access, check that its route table has no default route:
+
+```bash
+aws ec2 describe-route-tables --profile spinifex
+```
+
+The private route table should have no `0.0.0.0/0` route. Also verify the private security group egress is restricted to the VPC CIDR (`10.20.0.0/16`).
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/bastion-private-subnet/main.tf

@@ -0,0 +1,346 @@
+# Example 2: Bastion Host with Private Subnet
+#
+# Deploys a VPC with both public and private subnets. A bastion host in the
+# public subnet provides SSH access to an isolated instance in the private
+# subnet. The private instance has no internet connectivity — ideal for
+# sensitive workloads that must remain air-gapped from the internet.
+#
+# Architecture:
+#
+#   WAN ──SSH──▶ Bastion (public subnet)
+#                    │
+#                    ▼ SSH (private IP)
+#                 App Server (private subnet, no internet)
+#
+# Usage:
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   # SSH to the bastion
+#   ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+#
+#   # From the bastion, SSH to the private instance
+#   # (the key is pre-installed at ~/.ssh/bastion-demo.pem via cloud-init)
+#   ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair (shared by bastion and private instances)
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "bastion" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "bastion" {
+  key_name   = "bastion-demo"
+  public_key = tls_private_key.bastion.public_key_openssh
+}
+
+resource "local_file" "bastion_pem" {
+  filename        = "${path.module}/bastion-demo.pem"
+  content         = tls_private_key.bastion.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "bastion-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — only the public subnet routes through this
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet — bastion host lives here
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "bastion-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "bastion-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Private Subnet — isolated instances live here (no public IPs, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "private" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.2.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = false
+
+  tags = {
+    Name = "bastion-demo-private"
+  }
+}
+
+# Private route table — no default route, no internet access
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private" {
+  subnet_id      = aws_subnet.private.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Groups
+# ---------------------------------------------------------------------------
+
+# Bastion: SSH from anywhere
+resource "aws_security_group" "bastion" {
+  name        = "bastion-demo-bastion-sg"
+  description = "Bastion: SSH from WAN"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "bastion-demo-bastion-sg"
+  }
+}
+
+# Private instances: SSH only from the bastion security group
+resource "aws_security_group" "private" {
+  name        = "bastion-demo-private-sg"
+  description = "Private: SSH from bastion only"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description     = "SSH from bastion"
+    from_port       = 22
+    to_port         = 22
+    protocol        = "tcp"
+    security_groups = [aws_security_group.bastion.id]
+  }
+
+  egress {
+    description = "VPC internal only"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["10.20.0.0/16"]
+  }
+
+  tags = {
+    Name = "bastion-demo-private-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Bastion Host (public subnet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "bastion" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.bastion.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  associate_public_ip_address = true
+
+  # Copy the SSH private key onto the bastion so you can hop to private instances
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+    mkdir -p /home/ec2-user/.ssh
+    cat > /home/ec2-user/.ssh/bastion-demo.pem <<'KEY'
+    ${tls_private_key.bastion.private_key_openssh}
+    KEY
+    chmod 600 /home/ec2-user/.ssh/bastion-demo.pem
+    chown -R ec2-user:ec2-user /home/ec2-user/.ssh
+  USERDATA
+  )
+
+  tags = {
+    Name = "bastion-demo-bastion"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Private Instance (private subnet — no public IP, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "private" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.private.id
+  vpc_security_group_ids = [aws_security_group.private.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  tags = {
+    Name = "bastion-demo-private-app"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry."
+}
+
+output "bastion_public_ip" {
+  value = aws_instance.bastion.public_ip
+}
+
+output "private_instance_ip" {
+  value = aws_instance.private.private_ip
+}
+
+output "ssh_to_bastion" {
+  description = "SSH to the bastion host"
+  value       = "ssh -i bastion-demo.pem ec2-user@${aws_instance.bastion.public_ip}"
+}
+
+output "ssh_to_private_from_bastion" {
+  description = "From the bastion, SSH to the private instance (key is pre-installed via cloud-init)"
+  value       = "ssh -i ~/.ssh/bastion-demo.pem ec2-user@${aws_instance.private.private_ip}"
+}

docs/terraform/nginx-webserver/README.md

@@ -0,0 +1,129 @@
+---
+title: "Nginx Web Server"
+description: "Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - nginx
+  - ec2
+  - vpc
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Nginx Web Server
+
+> Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC, public subnet, internet gateway, route table, security group, SSH key pair, and an EC2 instance with cloud-init user-data that installs and starts Nginx.
+
+**What you'll learn:**
+
+- Configuring the AWS Terraform provider to target Spinifex
+- Creating a VPC with public internet access
+- Provisioning an EC2 instance with cloud-init user-data
+- Generating SSH key pairs with the TLS provider
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/nginx-webserver
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Verify
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+After apply completes, use the outputs to test:
+
+```bash
+curl http://<public_ip>
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+```
+
+Open the `web_url` output in your browser to see the Nginx welcome page.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image. Check available AMIs:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```
+
+### Provider Connection Refused
+
+Verify Spinifex services are running:
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
+
+### SSH Connection Timeout
+
+Check that the security group allows inbound SSH (port 22) and that the instance has a public IP assigned. Verify the instance is running:
+
+```bash
+aws ec2 describe-instances --profile spinifex
+```
+
+### Nginx Not Responding
+
+SSH into the instance and check cloud-init logs:
+
+```bash
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+sudo journalctl -u cloud-init --no-pager
+sudo systemctl status nginx
+```

docs/terraform/nginx-webserver/main.tf

@@ -0,0 +1,286 @@
+# Example 1: Nginx Web Server on Spinifex
+#
+# Deploys a VPC with a public subnet and an EC2 instance running Nginx.
+# Demonstrates: VPC, subnet, internet gateway, route table, security group,
+# key pair, cloud-init user-data, and EC2 instance provisioning.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/01-nginx-webserver
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>        # Nginx welcome page
+#   ssh -i nginx-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — point the AWS provider at Spinifex
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"] # Spinifex system images
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "nginx" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "nginx" {
+  key_name   = "nginx-demo"
+  public_key = tls_private_key.nginx.public_key_openssh
+}
+
+resource "local_file" "nginx_pem" {
+  filename        = "${path.module}/nginx-demo.pem"
+  content         = tls_private_key.nginx.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.10.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "nginx-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — gives the public subnet a route to the internet
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "nginx-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.10.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "nginx-demo-public"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Route Table — send 0.0.0.0/0 through the internet gateway
+# ---------------------------------------------------------------------------
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "nginx-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "web" {
+  name        = "nginx-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nginx-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Debian 12 with Nginx installed via cloud-init
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install Nginx
+    apt-get update -y
+    apt-get install -y nginx
+
+    # Write a custom landing page
+    cat > /var/www/html/index.html <<'HTML'
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p>This Nginx server was deployed with Terraform on Spinifex infrastructure.</p>
+      <hr>
+      <p><small>Instance provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    # Ensure Nginx is running
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.nginx.id
+}
+
+output "public_ip" {
+  value = aws_instance.nginx.public_ip
+}
+
+output "ssh_command" {
+  value = "ssh -i nginx-demo.pem ec2-user@${aws_instance.nginx.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.nginx.public_ip}"
+}

docs/terraform/s3-webapp/README.md

@@ -0,0 +1,162 @@
+---
+title: "S3-Backed Web App"
+description: "Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - s3
+  - predastore
+  - flask
+  - webapp
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: S3-Backed Web App
+
+> Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy an EC2 instance running a Flask file-sharing web application backed by S3 (Predastore). Users can upload files through a web form and browse uploaded content — demonstrating Terraform managing both compute and object storage together.
+
+**Architecture:**
+
+```
+Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
+                      │
+                      ▼ S3 API (boto3)
+                  Predastore (port 8443)
+```
+
+**What you'll learn:**
+
+- Configuring the AWS provider with both Spinifex and Predastore endpoints
+- Creating S3 buckets on Predastore via Terraform
+- Deploying a Python webapp with cloud-init that talks to S3
+- Passing credentials and configuration to instances via user-data
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- Predastore running (S3 API on port 8443)
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+- The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/s3-webapp
+```
+
+Or create the files manually and paste the full configuration below.
+
+### Step 2. Create terraform.tfvars
+
+Before deploying, create a `terraform.tfvars` with your Predastore credentials. The `predastore_host` must be reachable from inside the VPC — use the host's br-wan or LAN IP, not localhost.
+
+<!-- INCLUDE: terraform.tfvars.example lang:hcl -->
+
+### Step 3. Create main.tf
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 4. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 5. Test the Application
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+Open the `web_url` output in your browser. You should see the file browser UI. Upload a file and verify it appears in the list.
+
+```bash
+# Verify via CLI
+curl http://<public_ip>
+
+# Check the S3 bucket directly
+aws s3 ls s3://webapp-uploads/ --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Predastore Connection Refused from Instance
+
+The EC2 instance cannot reach `localhost` on the host. Set `predastore_host` to the host's br-wan or LAN IP address:
+
+```hcl
+predastore_host = "192.168.1.10:8443"
+```
+
+### S3 Bucket Creation Fails
+
+Verify Predastore is running and accessible:
+
+```bash
+curl -k https://localhost:8443/
+aws s3 ls --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Flask App Not Starting
+
+SSH into the instance and check the service:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+sudo systemctl status s3-webapp
+sudo journalctl -u s3-webapp --no-pager -n 50
+```
+
+### Upload Fails or Files Not Appearing
+
+Check that the S3 credentials in `/opt/webapp/.env` are correct and that the bucket exists:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+cat /opt/webapp/.env
+/opt/webapp/venv/bin/python -c "import boto3; print('boto3 OK')"
+```
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/s3-webapp/main.tf

@@ -0,0 +1,445 @@
+# Example 3: S3-Backed Web Application
+#
+# Deploys an EC2 instance running a simple file-sharing webapp backed by S3
+# (Predastore). Users can upload files through a web form and browse uploaded
+# content — demonstrating Terraform managing both compute (Spinifex) and
+# object storage (Predastore) resources together.
+#
+# Architecture:
+#
+#   Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
+#                         │
+#                         ▼ S3 API (boto3)
+#                     Predastore (port 8443)
+#
+# Prerequisites:
+#   - Spinifex services running (gateway on port 9999)
+#   - Predastore running (S3 API on port 8443)
+#   - The EC2 instance must be able to reach the Predastore endpoint.
+#     Set `predastore_host` to the IP reachable from inside the VPC
+#     (e.g. the host's br-wan IP, NOT localhost).
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/03-s3-webapp
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>          # File browser UI
+#   ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint (EC2/IAM)"
+}
+
+variable "predastore_endpoint" {
+  type        = string
+  default     = "https://localhost:8443"
+  description = "Predastore S3 endpoint (for Terraform to create buckets)"
+}
+
+variable "predastore_host" {
+  type        = string
+  description = "Predastore host:port reachable from inside the VPC (e.g. 192.168.1.10:8443)"
+}
+
+variable "s3_access_key" {
+  type        = string
+  description = "S3 access key for Predastore"
+}
+
+variable "s3_secret_key" {
+  type        = string
+  sensitive   = true
+  description = "S3 secret key for Predastore"
+}
+
+variable "bucket_name" {
+  type    = string
+  default = "webapp-uploads"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — EC2 via Spinifex gateway, S3 via Predastore
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region     = var.region
+  access_key = var.s3_access_key
+  secret_key = var.s3_secret_key
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    s3  = var.predastore_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  s3_use_path_style = true
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# S3 Bucket (created on Predastore)
+# ---------------------------------------------------------------------------
+
+resource "aws_s3_bucket" "uploads" {
+  bucket = var.bucket_name
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "webapp" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "webapp" {
+  key_name   = "s3-webapp-demo"
+  public_key = tls_private_key.webapp.public_key_openssh
+}
+
+resource "local_file" "webapp_pem" {
+  filename        = "${path.module}/s3-webapp-demo.pem"
+  content         = tls_private_key.webapp.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC + Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.30.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "s3-webapp-demo-vpc"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "s3-webapp-demo-igw"
+  }
+}
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.30.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "s3-webapp-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "webapp" {
+  name        = "s3-webapp-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Flask webapp that talks to Predastore S3
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "webapp" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.webapp.id]
+  key_name               = aws_key_pair.webapp.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install dependencies
+    apt-get update -y
+    apt-get install -y python3-pip python3-venv
+
+    # Create app directory and virtualenv
+    mkdir -p /opt/webapp
+    python3 -m venv /opt/webapp/venv
+    /opt/webapp/venv/bin/pip install flask boto3
+
+    # Write S3 credentials config
+    cat > /opt/webapp/.env <<'ENVFILE'
+    S3_ENDPOINT=https://${var.predastore_host}
+    S3_BUCKET=${var.bucket_name}
+    S3_ACCESS_KEY=${var.s3_access_key}
+    S3_SECRET_KEY=${var.s3_secret_key}
+    S3_REGION=${var.region}
+    ENVFILE
+
+    # Write the Flask application
+    cat > /opt/webapp/app.py <<'PYEOF'
+    import os, io, urllib3
+    from flask import Flask, request, redirect, url_for, Response
+
+    # Suppress TLS warnings for self-signed certs
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    # Load env
+    env = {}
+    with open("/opt/webapp/.env") as f:
+        for line in f:
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, v = line.split("=", 1)
+                env[k] = v
+
+    import boto3
+    from botocore.config import Config
+
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=env["S3_ENDPOINT"],
+        aws_access_key_id=env["S3_ACCESS_KEY"],
+        aws_secret_access_key=env["S3_SECRET_KEY"],
+        region_name=env["S3_REGION"],
+        verify=False,
+        config=Config(s3={"addressing_style": "path"}),
+    )
+
+    BUCKET = env["S3_BUCKET"]
+    app = Flask(__name__)
+
+    @app.route("/")
+    def index():
+        # List objects in the bucket
+        try:
+            resp = s3.list_objects_v2(Bucket=BUCKET)
+            objects = resp.get("Contents", [])
+        except Exception as e:
+            objects = []
+
+        rows = ""
+        for obj in objects:
+            key = obj["Key"]
+            size = obj["Size"]
+            rows += f'<tr><td><a href="/files/{key}">{key}</a></td><td>{size} bytes</td></tr>\n'
+
+        return f"""<!DOCTYPE html>
+    <html>
+    <head><title>Spinifex S3 File Browser</title></head>
+    <body style="font-family: sans-serif; max-width: 700px; margin: 40px auto;">
+      <h1>Spinifex S3 File Browser</h1>
+      <p>Bucket: <code>{BUCKET}</code></p>
+
+      <h2>Upload a File</h2>
+      <form method="POST" action="/upload" enctype="multipart/form-data">
+        <input type="file" name="file" required>
+        <button type="submit">Upload</button>
+      </form>
+
+      <h2>Files</h2>
+      <table border="1" cellpadding="6" cellspacing="0" style="border-collapse: collapse;">
+        <tr><th>Key</th><th>Size</th></tr>
+        {rows if rows else '<tr><td colspan="2">No files yet</td></tr>'}
+      </table>
+
+      <hr>
+      <p><small>Powered by Spinifex + Predastore</small></p>
+    </body>
+    </html>"""
+
+    @app.route("/upload", methods=["POST"])
+    def upload():
+        f = request.files.get("file")
+        if not f or not f.filename:
+            return redirect("/")
+        s3.put_object(Bucket=BUCKET, Key=f.filename, Body=f.read())
+        return redirect("/")
+
+    @app.route("/files/<path:key>")
+    def download(key):
+        try:
+            obj = s3.get_object(Bucket=BUCKET, Key=key)
+            return Response(
+                obj["Body"].read(),
+                headers={"Content-Disposition": f'inline; filename="{key}"'},
+            )
+        except Exception:
+            return "Not found", 404
+
+    if __name__ == "__main__":
+        app.run(host="0.0.0.0", port=80)
+    PYEOF
+
+    # Create a systemd service so the webapp starts on boot
+    cat > /etc/systemd/system/s3-webapp.service <<'SVCEOF'
+    [Unit]
+    Description=S3 File Browser Webapp
+    After=network.target
+
+    [Service]
+    Type=simple
+    ExecStart=/opt/webapp/venv/bin/python /opt/webapp/app.py
+    WorkingDirectory=/opt/webapp
+    Restart=always
+    RestartSec=3
+
+    [Install]
+    WantedBy=multi-user.target
+    SVCEOF
+
+    systemctl daemon-reload
+    systemctl enable s3-webapp
+    systemctl start s3-webapp
+  USERDATA
+  )
+
+  tags = {
+    Name = "s3-webapp-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.webapp.id
+}
+
+output "public_ip" {
+  value = aws_instance.webapp.public_ip
+}
+
+output "bucket_name" {
+  value = aws_s3_bucket.uploads.id
+}
+
+output "ssh_command" {
+  value = "ssh -i s3-webapp-demo.pem ec2-user@${aws_instance.webapp.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.webapp.public_ip}"
+}

docs/terraform/s3-webapp/terraform.tfvars.example

@@ -0,0 +1,14 @@
+# Copy this to terraform.tfvars and fill in your values.
+#
+# The predastore_host must be reachable from INSIDE the VPC guest — not
+# localhost. Use the host's br-wan or LAN IP, e.g. "192.168.1.10:8443".
+
+predastore_host = "192.168.1.10:8443"
+s3_access_key   = "AKIAIOSFODNN7EXAMPLE"
+s3_secret_key   = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+
+# Optional overrides:
+# spinifex_endpoint    = "https://localhost:9999"
+# predastore_endpoint  = "https://localhost:8443"
+# bucket_name          = "webapp-uploads"
+# region               = "ap-southeast-2"

docs/vpc-networking/README.md

@@ -1,7 +1,7 @@
 ---
 title: "VPC Networking"
 description: "How Spinifex implements AWS-compatible VPC networking with public and private subnets, security groups, and Elastic IPs using OVN."
-category: "Environments"
+category: "Compute"
 tags:
   - vpc
   - networking
@@ -27,14 +27,14 @@ Instances can operate in two modes: **private** (overlay-only, no WAN access) or
 
 Spinifex maps AWS VPC concepts directly to OVN constructs:
 
-| AWS Concept | OVN Construct | What It Does |
-|-------------|---------------|--------------|
-| VPC | Logical Router | Isolates tenant networks, routes between subnets |
-| Subnet | Logical Switch + DHCP | L2 broadcast domain with automatic IP assignment |
-| ENI | Logical Switch Port | Per-instance network interface with MAC/IP binding |
-| Internet Gateway | External Switch + NAT | Connects VPC router to physical WAN |
-| Security Group | Port Group + ACLs | Stateful firewall rules enforced in OVS datapath |
-| Elastic IP | `dnat_and_snat` NAT rule | Static 1:1 NAT between public and private IP |
+| AWS Concept      | OVN Construct            | What It Does                                       |
+| ---------------- | ------------------------ | -------------------------------------------------- |
+| VPC              | Logical Router           | Isolates tenant networks, routes between subnets   |
+| Subnet           | Logical Switch + DHCP    | L2 broadcast domain with automatic IP assignment   |
+| ENI              | Logical Switch Port      | Per-instance network interface with MAC/IP binding |
+| Internet Gateway | External Switch + NAT    | Connects VPC router to physical WAN                |
+| Security Group   | Port Group + ACLs        | Stateful firewall rules enforced in OVS datapath   |
+| Elastic IP       | `dnat_and_snat` NAT rule | Static 1:1 NAT between public and private IP       |
 
 ## Network Path
 
@@ -112,14 +112,14 @@ Inbound:  203.0.113.10 ──→ DNAT ──→ arrives at 10.0.1.5
 
 ## Comparison
 
-| | Private Subnet | Public Subnet |
-|---|---|---|
-| Private IP | Yes | Yes |
-| Public IP | No | Auto-assigned from pool |
-| Outbound internet | Only if VPC has IGW (shared SNAT) | Yes (own public IP via SNAT) |
-| Inbound from WAN | No | Yes (via 1:1 NAT to public IP) |
-| Instance sees public IP? | N/A | No — only sees private IP |
-| Elastic IP support | Only if explicitly associated | Yes |
+|                          | Private Subnet                    | Public Subnet                  |
+| ------------------------ | --------------------------------- | ------------------------------ |
+| Private IP               | Yes                               | Yes                            |
+| Public IP                | No                                | Auto-assigned from pool        |
+| Outbound internet        | Only if VPC has IGW (shared SNAT) | Yes (own public IP via SNAT)   |
+| Inbound from WAN         | No                                | Yes (via 1:1 NAT to public IP) |
+| Instance sees public IP? | N/A                               | No — only sees private IP      |
+| Elastic IP support       | Only if explicitly associated     | Yes                            |
 
 ## External Connectivity Modes
 
@@ -213,14 +213,14 @@ is identical:
 
 ### Choosing Static vs DHCP
 
-| | Static Range | DHCP Source |
-|---|---|---|
-| **Public IPs from** | Admin-defined `range_start`..`range_end` | Router's DHCP server |
-| **IP predictability** | You know the exact range | Router assigns whatever is available |
-| **Setup effort** | Must reserve range, shrink router DHCP scope | Just set `source = "dhcp"` |
-| **Dependency** | None | Requires `dhclient` on host, working router DHCP |
-| **Best for** | Datacenters, ISP blocks, production | Homelabs, dev environments, shared networks |
-| **Capacity** | Exact: `range_end - range_start` IPs | Limited by router's DHCP pool size |
+|                       | Static Range                                 | DHCP Source                                      |
+| --------------------- | -------------------------------------------- | ------------------------------------------------ |
+| **Public IPs from**   | Admin-defined `range_start`..`range_end`     | Router's DHCP server                             |
+| **IP predictability** | You know the exact range                     | Router assigns whatever is available             |
+| **Setup effort**      | Must reserve range, shrink router DHCP scope | Just set `source = "dhcp"`                       |
+| **Dependency**        | None                                         | Requires `dhclient` on host, working router DHCP |
+| **Best for**          | Datacenters, ISP blocks, production          | Homelabs, dev environments, shared networks      |
+| **Capacity**          | Exact: `range_end - range_start` IPs         | Limited by router's DHCP pool size               |
 
 Both support the same AWS features: public subnets, Elastic IPs, security groups,
 DescribeInstances showing public IPs.
@@ -257,16 +257,16 @@ communicate within their VPC.
 
 ## Mode Comparison
 
-| Capability | `pool` (static) | `pool` (dhcp) | `nat` | Disabled |
-|------------|-----------------|---------------|-------|----------|
-| Outbound internet | Yes | Yes | Yes | No |
-| Inbound from WAN | Yes (1:1 NAT) | Yes (1:1 NAT) | No | No |
-| Public subnets | Yes | Yes | No | No |
-| Auto-assign public IPs | Yes | Yes | No | No |
-| Elastic IPs | Yes | Yes | No | No |
-| DescribeInstances shows public IP | Yes | Yes | No | No |
-| Admin must reserve IP range | Yes | No | No | No |
-| Needs router DHCP | No | Yes | Optional | No |
+| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`    | Disabled |
+| --------------------------------- | --------------- | ------------- | -------- | -------- |
+| Outbound internet                 | Yes             | Yes           | Yes      | No       |
+| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | No       | No       |
+| Public subnets                    | Yes             | Yes           | No       | No       |
+| Auto-assign public IPs            | Yes             | Yes           | No       | No       |
+| Elastic IPs                       | Yes             | Yes           | No       | No       |
+| DescribeInstances shows public IP | Yes             | Yes           | No       | No       |
+| Admin must reserve IP range       | Yes             | No            | No       | No       |
+| Needs router DHCP                 | No              | Yes           | Optional | No       |
 
 If you start with `nat` and later need public subnets, switch to `pool` and
 define a range (or use `source = "dhcp"`) — no data migration needed.
@@ -286,10 +286,10 @@ the best approach:
 
 Every Spinifex node has two OVS bridges:
 
-| Bridge | Purpose | Ports |
-|--------|---------|-------|
-| `br-int` | VM overlay traffic (Geneve tunnels) | VM TAP devices, tunnel ports |
-| `br-wan` | WAN uplink for public subnet traffic | macvlan on WAN NIC |
+| Bridge   | Purpose                              | Ports                        |
+| -------- | ------------------------------------ | ---------------------------- |
+| `br-int` | VM overlay traffic (Geneve tunnels)  | VM TAP devices, tunnel ports |
+| `br-wan` | WAN uplink for public subnet traffic | macvlan on WAN NIC           |
 
 `br-int` is always created by `setup-ovn.sh`. `br-wan` is created only when
 a WAN bridge is configured (auto-detected or via `--wan-bridge`, required for public subnets / external connectivity).
@@ -322,6 +322,7 @@ used for SSH or management. Datacenter servers with separate management +
 public NICs. Homelab hosts with 2+ NICs.
 
 **Benefits over macvlan:**
+
 - **Distributed NAT**: DNAT processed on the VM's own chassis (no hairpin)
 - **No MAC workarounds**: OVS sees all frames, any MAC
 - **Host can reach VMs**: No macvlan parent↔child isolation
@@ -354,6 +355,7 @@ WAN NIC (host IP — unchanged)
 edge deployments. Adding the NIC directly to OVS would break host connectivity.
 
 **Limitations:**
+
 - Centralized NAT only (macvlan filters unicast not matching its MAC)
 - Host cannot reach VMs at their public IPs (macvlan isolation)
 - vpcd must align macvlan MAC with OVN router MAC on startup
@@ -365,23 +367,23 @@ sudo setup-ovn.sh --macvlan --wan-iface=eth0
 
 ### Mode Comparison
 
-| Capability | Direct bridge | Macvlan |
-|------------|--------------|---------|
-| NAT mode | Distributed (local DNAT) | Centralized (gateway hairpin) |
-| Multi-node external perf | Optimal | Hairpin through gateway chassis |
-| Host → VM public IP | Works | Blocked (kernel isolation) |
-| MAC alignment needed | No | Yes (fragile startup step) |
-| Gratuitous ARPs needed | No | Yes (`nat-addresses=router`) |
-| Works on single-NIC | No (breaks SSH) | Yes |
-| DHCP lease stability | Clean | Fragile (MAC changes post-lease) |
+| Capability               | Direct bridge            | Macvlan                          |
+| ------------------------ | ------------------------ | -------------------------------- |
+| NAT mode                 | Distributed (local DNAT) | Centralized (gateway hairpin)    |
+| Multi-node external perf | Optimal                  | Hairpin through gateway chassis  |
+| Host → VM public IP      | Works                    | Blocked (kernel isolation)       |
+| MAC alignment needed     | No                       | Yes (fragile startup step)       |
+| Gratuitous ARPs needed   | No                       | Yes (`nat-addresses=router`)     |
+| Works on single-NIC      | No (breaks SSH)          | Yes                              |
+| DHCP lease stability     | Clean                    | Fragile (MAC changes post-lease) |
 
 ### How setup-ovn.sh Decides
 
-| Flags | Result |
-|-------|--------|
-| `--wan-bridge=br-wan --wan-iface=eth1` | Direct bridge: NIC added to br-wan |
-| `--macvlan --wan-iface=eth0` | Macvlan: sub-interface created, added to br-wan |
-| (no WAN bridge) | Only br-int created, no WAN connectivity |
+| Flags                                  | Result                                          |
+| -------------------------------------- | ----------------------------------------------- |
+| `--wan-bridge=br-wan --wan-iface=eth1` | Direct bridge: NIC added to br-wan              |
+| `--macvlan --wan-iface=eth0`           | Macvlan: sub-interface created, added to br-wan |
+| (no WAN bridge)                        | Only br-int created, no WAN connectivity        |
 
 ### Setup (Macvlan Mode)
 
@@ -402,12 +404,12 @@ macvlan in `bridge` mode creates a virtual interface that shares the parent NIC'
 physical wire but has its own MAC address. The Linux kernel blocks direct L2
 frames between a parent interface and its macvlan children.
 
-| Path | Works? | Why |
-|------|--------|-----|
-| VM → internet | Yes | SNAT through OVN router → macvlan → WAN NIC → WAN |
-| LAN device → VM public IP | Yes | LAN → WAN NIC → macvlan → br-wan → OVN |
-| Host → VM public IP | No | macvlan isolation (kernel blocks parent↔child) |
-| Host → VM private IP | Yes | Overlay via br-int (unrelated to br-wan) |
+| Path                      | Works? | Why                                               |
+| ------------------------- | ------ | ------------------------------------------------- |
+| VM → internet             | Yes    | SNAT through OVN router → macvlan → WAN NIC → WAN |
+| LAN device → VM public IP | Yes    | LAN → WAN NIC → macvlan → br-wan → OVN            |
+| Host → VM public IP       | No     | macvlan isolation (kernel blocks parent↔child)    |
+| Host → VM private IP      | Yes    | Overlay via br-int (unrelated to br-wan)          |
 
 ## Per-Node Configuration
 
@@ -496,11 +498,11 @@ spinifex.toml
 external_mode = "pool"    # "pool", "nat", or "" (disabled)
 ```
 
-| Value | Behavior |
-|-------|----------|
-| `"pool"` | Full public networking — public subnets, auto-assign, Elastic IPs |
-| `"nat"` | Outbound-only SNAT — all VMs share one external IP |
-| `""` / omitted | Overlay-only — no external connectivity |
+| Value          | Behavior                                                          |
+| -------------- | ----------------------------------------------------------------- |
+| `"pool"`       | Full public networking — public subnets, auto-assign, Elastic IPs |
+| `"nat"`        | Outbound-only SNAT — all VMs share one external IP                |
+| `""` / omitted | Overlay-only — no external connectivity                           |
 
 ## IP Pools: network.external_pools
 
@@ -523,18 +525,18 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 
 ### Field Details
 
-| Field | Required | Description |
-|-------|----------|-------------|
-| `name` | Yes | Unique pool name. Used as NATS KV key and in `AllocateAddress`. |
-| `source` | No | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch. |
-| `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides). |
-| `range_end` | Static only | Last IP in the range. |
-| `gateway` | Yes | Physical router/switch — the WAN default gateway. OVN sets `0.0.0.0/0 → gateway`. |
-| `gateway_ip` | NAT mode | Static IP for OVN router SNAT. In pool mode, defaults to `range_start` (static) or first DHCP lease (dhcp). In NAT mode, this is the single external IP all VMs share. |
-| `prefix_len` | Yes | Subnet mask for the external network (e.g., 24 = /24). |
-| `region` | No | Scopes pool to a region. Instances in this region prefer this pool. |
-| `az` | No | Scopes pool to an AZ. More specific than region. |
-| `dns_servers` | No | DNS servers propagated to VMs via OVN DHCP. |
+| Field         | Required    | Description                                                                                                                                                            |
+| ------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `name`        | Yes         | Unique pool name. Used as NATS KV key and in `AllocateAddress`.                                                                                                        |
+| `source`      | No          | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch.                                  |
+| `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides).                                                                      |
+| `range_end`   | Static only | Last IP in the range.                                                                                                                                                  |
+| `gateway`     | Yes         | Physical router/switch — the WAN default gateway. OVN sets `0.0.0.0/0 → gateway`.                                                                                      |
+| `gateway_ip`  | NAT mode    | Static IP for OVN router SNAT. In pool mode, defaults to `range_start` (static) or first DHCP lease (dhcp). In NAT mode, this is the single external IP all VMs share. |
+| `prefix_len`  | Yes         | Subnet mask for the external network (e.g., 24 = /24).                                                                                                                 |
+| `region`      | No          | Scopes pool to a region. Instances in this region prefer this pool.                                                                                                    |
+| `az`          | No          | Scopes pool to an AZ. More specific than region.                                                                                                                       |
+| `dns_servers` | No          | DNS servers propagated to VMs via OVN DHCP.                                                                                                                            |
 
 ### Why range_start/range_end Instead of CIDR?
 
@@ -561,8 +563,8 @@ ovn_sb_addr        = "tcp:10.1.3.181:6642"   # OVN Southbound DB
 external_interface = "eth1"                   # WAN NIC name
 ```
 
-| Field | Description |
-|-------|-------------|
+| Field                | Description                                                                                                                                                      |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `external_interface` | Physical NIC for WAN traffic. A macvlan sub-interface is created on this NIC for br-wan. Different servers may have different names (eth1, eno2, enp3s0, bond0). |
 
 ## Pool Selection Logic
@@ -588,8 +590,12 @@ name. Allocation uses CAS (Compare-And-Set) for lock-free concurrent access:
   "range_start": "192.168.1.150",
   "range_end": "192.168.1.250",
   "allocated": {
-    "192.168.1.150": {"type": "gateway"},
-    "192.168.1.151": {"type": "auto_assign", "eni_id": "eni-abc", "instance_id": "i-123"}
+    "192.168.1.150": { "type": "gateway" },
+    "192.168.1.151": {
+      "type": "auto_assign",
+      "eni_id": "eni-abc",
+      "instance_id": "i-123"
+    }
   }
 }
 ```
@@ -774,20 +780,21 @@ on the port group control traffic:
 ## Default Security Group
 
 Every VPC gets a default security group that:
+
 - Allows all inbound from instances in the same security group
 - Allows all outbound
 - Denies all other inbound
 
 ## AWS Rule → OVN ACL Translation
 
-| AWS Security Group Rule | OVN ACL Match |
-|------------------------|---------------|
-| Ingress TCP/22 from 0.0.0.0/0 | `outport == @sg && ip4 && tcp.dst == 22` |
+| AWS Security Group Rule         | OVN ACL Match                                                      |
+| ------------------------------- | ------------------------------------------------------------------ |
+| Ingress TCP/22 from 0.0.0.0/0   | `outport == @sg && ip4 && tcp.dst == 22`                           |
 | Ingress TCP/443 from 10.0.0.0/8 | `outport == @sg && ip4 && tcp.dst == 443 && ip4.src == 10.0.0.0/8` |
-| Ingress ALL from sg-other | `outport == @sg && ip4 && ip4.src == $sg_other_ip4` |
-| Ingress ICMP from anywhere | `outport == @sg && ip4 && icmp4` |
-| Egress ALL to 0.0.0.0/0 | `inport == @sg && ip4` |
-| Default deny inbound | `outport == @sg && ip4` (priority 900, action=drop) |
+| Ingress ALL from sg-other       | `outport == @sg && ip4 && ip4.src == $sg_other_ip4`                |
+| Ingress ICMP from anywhere      | `outport == @sg && ip4 && icmp4`                                   |
+| Egress ALL to 0.0.0.0/0         | `inport == @sg && ip4`                                             |
+| Default deny inbound            | `outport == @sg && ip4` (priority 900, action=drop)                |
 
 ## Example: Allow SSH + HTTP
 
@@ -1124,18 +1131,21 @@ ip route show
 ### Instance Has No Public IP
 
 1. Check subnet has `MapPublicIpOnLaunch`:
+
    ```bash
    aws ec2 describe-subnets --subnet-ids $SUBNET \
      --query 'Subnets[0].MapPublicIpOnLaunch'
    ```
 
 2. Check IGW is attached:
+
    ```bash
    aws ec2 describe-internet-gateways \
      --filters Name=attachment.vpc-id,Values=$VPC
    ```
 
 3. Check external IP pool:
+
    ```bash
    nats kv get spinifex-external-ipam wan
    ```
@@ -1272,6 +1282,7 @@ sudo systemctl start ovn-central ovn-controller
 `ovn-sbctl show`. Gateway traffic (SNAT/DNAT) doesn't work.
 
 **Cause:** Three names must align:
+
 1. OVS `external_ids:system-id` → what ovn-controller registers as
 2. OVN SB `Chassis.name` → what actually exists in the SB DB
 3. vpcd's chassis name → `chassis-{nodeName}` from config

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.1.3
+	github.com/mulgadc/predastore v1.1.4
 	github.com/mulgadc/viperblock v1.1.5
 	github.com/nats-io/nats-server/v2 v2.12.6
-	github.com/nats-io/nats.go v1.49.0
+	github.com/nats-io/nats.go v1.50.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
 	github.com/pelletier/go-toml/v2 v2.3.0
 	github.com/pterm/pterm v0.12.83

go.sum

@@ -255,8 +255,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.1.3 h1:WddAOXmGJFQ7tRG+5B3yz/tCX46yW1CmbVL07y+tyBU=
-github.com/mulgadc/predastore v1.1.3/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
+github.com/mulgadc/predastore v1.1.4 h1:czeGQWWqmmZuGnUuvyrORavySN/SGPTwjUxBd/9swuU=
+github.com/mulgadc/predastore v1.1.4/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
 github.com/mulgadc/viperblock v1.1.5 h1:ojmoJQrY7VkDGzdQj07w2F7ICRJX9RlGAKyn87Wa55o=
 github.com/mulgadc/viperblock v1.1.5/go.mod h1:utoIQLwCmPujOBaT0mL5NhW3m2fwjQPQiQRJH6+Wtk8=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -267,8 +267,8 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
 github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
 github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8=
 github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo=
-github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
-github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=
+github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

scripts/build-alb-image.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# build-alb-image.sh — Thin wrapper for backward compatibility
+# Delegates to the generic build-system-image.sh with the ALB manifest.
+exec "$(dirname "$0")/build-system-image.sh" "$(dirname "$0")/images/alb.conf" "$@"

scripts/build-system-image.sh

@@ -0,0 +1,275 @@
+#!/bin/bash
+set -euo pipefail
+
+# build-system-image.sh — Build a minimal Alpine AMI from a manifest
+#
+# Creates a pre-baked Alpine Linux image with custom packages, binaries,
+# and setup scripts installed, ready for import as a Spinifex AMI.
+#
+# Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
+# Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
+#   --import  Also import the image as an AMI via spx admin images import
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+usage() {
+    echo "Usage: $0 <manifest.conf> [--import]"
+    echo ""
+    echo "  manifest.conf  Path to image manifest (see scripts/images/ for examples)"
+    echo "  --import       Import the built image as an AMI"
+    exit 1
+}
+
+if [[ $# -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
+    usage
+fi
+
+MANIFEST="$1"
+shift
+
+if [[ ! -f "$MANIFEST" ]]; then
+    echo "ERROR: Manifest not found: $MANIFEST"
+    exit 1
+fi
+
+DO_IMPORT=false
+if [[ "${1:-}" == "--import" ]]; then
+    DO_IMPORT=true
+fi
+
+# Source the manifest
+# shellcheck source=/dev/null
+source "$MANIFEST"
+
+# Validate required manifest fields
+for field in IMAGE_NAME ALPINE_VERSION IMAGE_SIZE; do
+    if [[ -z "${!field:-}" ]]; then
+        echo "ERROR: Manifest missing required field: $field"
+        exit 1
+    fi
+done
+
+# Derived paths
+BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
+ALPINE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
+ALPINE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${ALPINE_IMAGE}"
+OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
+OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
+NBD_DEV="/dev/nbd0"
+MOUNT_DIR="${BUILD_DIR}/mnt"
+
+cleanup() {
+    echo "Cleaning up..."
+    sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+    sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+    echo "Done."
+}
+trap cleanup EXIT
+
+echo "=== System Image Builder ==="
+echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
+echo "Alpine:  ${ALPINE_VERSION}"
+echo "Size:    ${IMAGE_SIZE}"
+echo "Build:   ${BUILD_DIR}"
+echo ""
+
+# Step 0: Check prerequisites
+if ! command -v qemu-nbd &>/dev/null; then
+    echo "ERROR: qemu-nbd not found. Install qemu-utils."
+    exit 1
+fi
+
+if ! command -v qemu-img &>/dev/null; then
+    echo "ERROR: qemu-img not found. Install qemu-utils."
+    exit 1
+fi
+
+# Build binaries if BUILD_COMMANDS is set
+if [[ -n "${BUILD_COMMANDS:-}" ]]; then
+    echo "Building binaries: ${BUILD_COMMANDS}"
+    if ! (cd "$PROJECT_DIR" && eval "$BUILD_COMMANDS"); then
+        echo "ERROR: BUILD_COMMANDS failed: ${BUILD_COMMANDS}"
+        exit 1
+    fi
+fi
+
+# Verify binaries exist and are statically linked
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        src_path="${PROJECT_DIR}/${src}"
+        if [[ ! -f "$src_path" ]]; then
+            echo "ERROR: Binary not found: $src_path"
+            exit 1
+        fi
+        if ! file "$src_path" | grep -q "statically linked"; then
+            echo "ERROR: $src_path is not statically linked (Alpine uses musl — dynamic glibc binaries will fail)"
+            echo "  Rebuild with: CGO_ENABLED=0 go build ..."
+            exit 1
+        fi
+    done
+fi
+
+mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
+
+# Step 1: Download Alpine cloud image
+if [[ -f "${BUILD_DIR}/${ALPINE_IMAGE}" ]]; then
+    echo "Alpine image already downloaded."
+else
+    echo "Downloading Alpine ${ALPINE_VERSION} cloud image..."
+    if ! curl --fail -L -o "${BUILD_DIR}/${ALPINE_IMAGE}" "$ALPINE_URL"; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Failed to download Alpine image from $ALPINE_URL"
+        exit 1
+    fi
+    # Verify the download is a valid qcow2 image
+    if ! qemu-img info "${BUILD_DIR}/${ALPINE_IMAGE}" &>/dev/null; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Downloaded file is not a valid qcow2 image"
+        exit 1
+    fi
+fi
+
+# Step 2: Copy image for customization
+echo "Copying image for customization..."
+cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
+
+# Resize the image (Alpine cloud images are ~200MB, need room for packages)
+qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
+
+# Step 3: Connect via qemu-nbd
+echo "Connecting image via qemu-nbd..."
+sudo modprobe nbd max_part=4 2>/dev/null || true
+if [[ ! -e "${NBD_DEV}" ]]; then
+    echo "ERROR: ${NBD_DEV} does not exist. Is the nbd kernel module loaded? Try: sudo modprobe nbd"
+    exit 1
+fi
+sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
+sleep 1
+
+# Alpine cloud images have ext4 directly on the block device (no partition table).
+# Resize the filesystem to fill the resized image.
+echo "Checking filesystem..."
+sudo e2fsck -f -y "${NBD_DEV}" || {
+    ec=$?
+    if [[ $ec -gt 1 ]]; then
+        echo "ERROR: e2fsck failed with exit code $ec on ${NBD_DEV}"
+        exit 1
+    fi
+}
+
+echo "Resizing filesystem..."
+if ! sudo resize2fs "${NBD_DEV}"; then
+    echo "ERROR: resize2fs failed on ${NBD_DEV}"
+    exit 1
+fi
+
+# Step 4: Mount and customize
+echo "Mounting root filesystem..."
+sudo mount "${NBD_DEV}" "$MOUNT_DIR"
+
+# Set up resolv.conf for DNS inside chroot
+sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
+
+# Install packages
+if [[ -n "${APK_PACKAGES:-}" ]]; then
+    echo "Installing packages: ${APK_PACKAGES}..."
+    sudo chroot "$MOUNT_DIR" /bin/sh -c "
+set -e
+# Enable community repo
+sed -i 's|^#\(.*community\)|\1|' /etc/apk/repositories 2>/dev/null || true
+
+# Ensure community repo is present
+if ! grep -q community /etc/apk/repositories; then
+    MIRROR=\$(grep main /etc/apk/repositories | head -1 | sed 's|/main|/community|')
+    echo \"\$MIRROR\" >> /etc/apk/repositories
+fi
+
+apk update
+apk add ${APK_PACKAGES}
+"
+fi
+
+# Enable OpenRC services
+if [[ -n "${ENABLE_SERVICES:-}" ]]; then
+    echo "Enabling services: ${ENABLE_SERVICES}..."
+    IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
+    for svc in "${SERVICES[@]}"; do
+        if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
+            echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
+            exit 1
+        fi
+    done
+fi
+
+# Step 5: Copy binaries into the image (before setup script, which may reference them)
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    echo "Installing binaries..."
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        dst="${pair#*:}"
+        src_path="${PROJECT_DIR}/${src}"
+        echo "  ${src} -> ${dst}"
+        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
+        sudo chmod 755 "${MOUNT_DIR}${dst}"
+    done
+fi
+
+# Run custom setup script inside chroot (after binaries are installed)
+if [[ -n "${SETUP_SCRIPT:-}" ]]; then
+    setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
+    if [[ ! -f "$setup_path" ]]; then
+        echo "ERROR: Setup script not found: $setup_path"
+        exit 1
+    fi
+    echo "Running setup script: ${SETUP_SCRIPT}..."
+    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
+fi
+
+# Step 6: Clean up and unmount
+echo "Cleaning up image..."
+sudo chroot "$MOUNT_DIR" /bin/sh -c '
+apk cache clean 2>/dev/null || true
+rm -rf /var/cache/apk/* /tmp/*
+'
+
+# Restore original resolv.conf (cloud-init will set it on boot)
+sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
+
+echo "Unmounting..."
+sudo umount "$MOUNT_DIR"
+sudo qemu-nbd --disconnect "${NBD_DEV}"
+
+# Step 7: Convert to raw for import
+echo "Converting to raw format..."
+qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
+
+echo ""
+echo "=== Build complete ==="
+echo "  Image: ${IMAGE_NAME}"
+echo "  qcow2: $OUTPUT_IMAGE ($(du -h "$OUTPUT_IMAGE" | cut -f1))"
+echo "  raw:   $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
+echo ""
+
+if [[ "$DO_IMPORT" == true ]]; then
+    echo "Importing as AMI..."
+    (cd "$PROJECT_DIR" && ./bin/spx admin images import \
+        --file "$OUTPUT_RAW" \
+        --distro alpine \
+        --version "${ALPINE_VERSION}-${IMAGE_NAME}" \
+        --arch x86_64 \
+        --config "$HOME/spinifex/config/spinifex.toml")
+else
+    echo "To import as AMI, run:"
+    echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
+    echo "    --file $OUTPUT_RAW \\"
+    echo "    --distro alpine --version ${ALPINE_VERSION}-${IMAGE_NAME} --arch x86_64 \\"
+    echo "    --config \$HOME/spinifex/config/spinifex.toml"
+fi

scripts/check-coverage.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]
 
 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=60.0
+MIN=65.0
 QUIET="${2:-}"
 
 if [[ ! -s "$PROFILE" ]]; then

scripts/clone-deps.sh

@@ -3,7 +3,6 @@
 # Spinifex Development Dependencies Setup Script
 # This script clones Viperblock and Predastore repositories for cross-repo development
 
-# TODO: Improve, remove CLAUDE readme reference, simplify next-steps
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

scripts/coverage.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Run Go tests with coverage and report total code coverage.
-
-COVERPROFILE=$(mktemp /tmp/spinifex-coverage-XXXXXX.out)
-trap 'rm -f "$COVERPROFILE"' EXIT
-
-echo "Running tests with coverage..."
-echo ""
-LOG_IGNORE=1 go test -timeout 120s -coverprofile="$COVERPROFILE" -covermode=atomic ./spinifex/... 2>&1
-
-if [[ ! -s "$COVERPROFILE" ]]; then
-    echo "No coverage data collected."
-    exit 1
-fi
-
-echo ""
-echo "=== Total ==="
-go tool cover -func="$COVERPROFILE" | awk '/^total:/ { print $NF }'

scripts/diff-coverage.sh

@@ -7,13 +7,12 @@ set -euo pipefail
 # Usage: scripts/diff-coverage.sh <coverprofile> [--base <ref>] [--threshold <pct>]
 #
 # Base ref auto-detection:
-#   dev branch    → origin/main
 #   main branch   → HEAD~1
-#   other branch  → origin/dev
+#   other branch  → origin/main
 
 PROFILE=""
 BASE_REF=""
-THRESHOLD=60
+THRESHOLD=65
 QUIET="${QUIET:-}"
 
 while [[ $# -gt 0 ]]; do
@@ -45,8 +44,7 @@ if [[ -z "$BASE_REF" ]]; then
     BRANCH="${GITHUB_REF_NAME:-$(git rev-parse --abbrev-ref HEAD)}"
     case "$BRANCH" in
         main) BASE_REF="HEAD~1" ;;
-        dev)  BASE_REF="origin/main" ;;
-        *)    BASE_REF="origin/dev" ;;
+        *)    BASE_REF="origin/main" ;;
     esac
     [[ -z "$QUIET" ]] && echo "Base ref: $BASE_REF (branch: $BRANCH)"
 fi

scripts/iac/aws/basic/main.tf

@@ -0,0 +1,307 @@
+# Spinifex IaC Compatibility Test — "basic" suite
+#
+# Exercises every AWS resource Spinifex currently supports via tofu apply,
+# WITHOUT route tables, NACLs, or other unimplemented networking.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/basic
+#   tofu init
+#   tofu plan
+#   tofu apply        # full round-trip against a running Spinifex cluster
+#   tofu destroy      # clean teardown
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "mulgaos_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.mulgaos_endpoint
+    s3  = var.mulgaos_endpoint
+    iam = var.mulgaos_endpoint
+    sts = var.mulgaos_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 1. VPC + Subnet (no route table needed)
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "test" {
+  cidr_block           = "10.99.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "spx-basic-test-vpc"
+  }
+}
+
+resource "aws_subnet" "test" {
+  vpc_id                  = aws_vpc.test.id
+  cidr_block              = "10.99.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "spx-basic-test-subnet"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 2. Internet Gateway (attach to VPC, but no route table referencing it)
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "test" {
+  vpc_id = aws_vpc.test.id
+
+  tags = {
+    Name = "spx-basic-test-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 3. Security Group — inbound SSH + HTTP, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "test" {
+  name        = "spx-basic-test-sg"
+  description = "Basic test: SSH + HTTP inbound, all outbound"
+  vpc_id      = aws_vpc.test.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "spx-basic-test-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 4. Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "test" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "test" {
+  key_name   = "spx-basic-test-key"
+  public_key = tls_private_key.test.public_key_openssh
+}
+
+resource "local_file" "test_pem" {
+  filename        = "${path.module}/spx-basic-test.pem"
+  content         = tls_private_key.test.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# 5. Placement Group
+# ---------------------------------------------------------------------------
+
+resource "aws_placement_group" "test" {
+  name     = "spx-basic-test-spread"
+  strategy = "spread"
+}
+
+# ---------------------------------------------------------------------------
+# 6. EC2 Instance
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "test" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.test.id
+  vpc_security_group_ids = [aws_security_group.test.id]
+  key_name               = aws_key_pair.test.key_name
+  placement_group        = aws_placement_group.test.name
+
+  associate_public_ip_address = true
+
+  tags = {
+    Name        = "spx-basic-test-vm"
+    Environment = "test"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 7. EBS Volume + Attachment
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_volume" "data" {
+  availability_zone = data.aws_availability_zones.available.names[0]
+  size              = 10
+  type              = "gp3"
+
+  tags = {
+    Name = "spx-basic-test-data-vol"
+  }
+}
+
+
+# ---------------------------------------------------------------------------
+# 8. EBS Snapshot (of the standalone volume)
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_snapshot" "data" {
+  volume_id = aws_ebs_volume.data.id
+
+  tags = {
+    Name = "spx-basic-test-snapshot"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 9. Elastic IP + Association
+#    Requires external IPAM (pool mode) — uncomment when available.
+# ---------------------------------------------------------------------------
+
+# resource "aws_eip" "test" {
+#   domain = "vpc"
+#
+#   tags = {
+#     Name = "spx-basic-test-eip"
+#   }
+# }
+#
+# resource "aws_eip_association" "test" {
+#   allocation_id = aws_eip.test.id
+#   instance_id   = aws_instance.test.id
+# }
+
+# ---------------------------------------------------------------------------
+# 10. Standalone Network Interface (not attached — just tests CRUD)
+# ---------------------------------------------------------------------------
+
+resource "aws_network_interface" "extra" {
+  subnet_id       = aws_subnet.test.id
+  security_groups = [aws_security_group.test.id]
+
+  tags = {
+    Name = "spx-basic-test-eni"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "vpc_id" {
+  value = aws_vpc.test.id
+}
+
+output "instance_id" {
+  value = aws_instance.test.id
+}
+
+output "instance_public_ip" {
+  value = aws_instance.test.public_ip
+}
+
+# output "eip_public_ip" {
+#   value = aws_eip.test.public_ip
+# }
+
+output "volume_id" {
+  value = aws_ebs_volume.data.id
+}
+
+output "snapshot_id" {
+  value = aws_ebs_snapshot.data.id
+}
+
+output "ssh_command" {
+  value = "ssh -i spx-basic-test.pem admin@${aws_instance.test.public_ip}"
+}

scripts/iac/aws/main.tf

@@ -189,12 +189,6 @@ resource "aws_security_group" "web_ssh" {
   }
 }
 
-# 4: Ensure 3 servers on different underlying hardware (spread placement group)
-resource "aws_placement_group" "spread" {
-  name     = "spinifex-test-spread"
-  strategy = "spread"
-}
-
 resource "aws_instance" "vm" {
   count         = 3
   ami           = data.aws_ami.debian12.id
@@ -205,7 +199,6 @@ resource "aws_instance" "vm" {
   key_name               = aws_key_pair.hive_test.key_name
 
   associate_public_ip_address = true
-  placement_group             = aws_placement_group.spread.name
 
   tags = {
     Name = "spinifex-test-vm-${count.index + 1}"

scripts/iac/configure-cluster.sh

@@ -110,7 +110,7 @@ INIT_PID=$!
 # Wait for formation server to be ready
 echo "    Waiting for formation server on $NODE1_IP..."
 attempts=0
-while ! curl -s --connect-timeout 3 "http://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
+while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
     attempts=$((attempts + 1))
     if [ $attempts -ge 60 ]; then
         echo "Error: Formation server did not start on $NODE1_IP after 5 minutes"

scripts/images/alb-setup.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+# alb-setup.sh — ALB-specific chroot setup
+#
+# Runs inside the Alpine image chroot after packages are installed.
+# Sets up HAProxy placeholder config and alb-agent OpenRC init script.
+
+set -e
+
+# Create haproxy config directory and placeholder config
+mkdir -p /etc/haproxy
+cat > /etc/haproxy/haproxy.cfg <<'EOF'
+# Placeholder config — replaced by alb-agent on first config push
+global
+    daemon
+    maxconn 256
+
+defaults
+    mode http
+    timeout connect 5s
+    timeout client 30s
+    timeout server 30s
+
+frontend health
+    bind *:8405
+    http-request return status 200 content-type text/plain string "ok"
+EOF
+
+# Create alb-agent OpenRC init script
+mkdir -p /etc/init.d
+cat > /etc/init.d/alb-agent <<'INITSCRIPT'
+#!/sbin/openrc-run
+
+description="ALB NATS Config Agent"
+command="/usr/local/bin/alb-agent"
+command_args="--lb-id=${ALB_LB_ID:-unknown} --nats=${ALB_NATS_URL:-nats://10.0.2.2:4222}"
+command_background=true
+pidfile="/run/alb-agent.pid"
+output_log="/var/log/alb-agent.log"
+error_log="/var/log/alb-agent.log"
+
+depend() {
+    need net
+    after firewall
+}
+INITSCRIPT
+chmod 755 /etc/init.d/alb-agent

scripts/images/alb.conf

@@ -0,0 +1,22 @@
+# ALB system image manifest
+# Used by build-system-image.sh to create an Alpine AMI with HAProxy + alb-agent
+
+IMAGE_NAME="alb"
+IMAGE_DESCRIPTION="ALB VM with HAProxy and alb-agent"
+ALPINE_VERSION="3.21.6"
+IMAGE_SIZE="512M"
+
+# Packages to install via apk
+APK_PACKAGES="haproxy curl"
+
+# Binaries to copy into the image (source:destination)
+INSTALL_BINARIES="bin/alb-agent:/usr/local/bin/alb-agent"
+
+# Build command for binaries (run before image build)
+BUILD_COMMANDS="make build-alb-agent"
+
+# OpenRC services to enable at boot
+ENABLE_SERVICES="haproxy"
+
+# Custom setup script (runs inside chroot after packages are installed)
+SETUP_SCRIPT="scripts/images/alb-setup.sh"

scripts/reset-dev-env.sh

@@ -160,15 +160,12 @@ elif [ -n "$WAN_IFACE" ]; then
     fi
 fi
 
-# Chassis ID must match what vpcd derives from the node name in spinifex.toml.
-# vpcd generates "chassis-{nodeName}" (e.g., chassis-node1) and uses it to schedule
-# gateway chassis in the NB DB. ovn-controller registers in the SB DB using the OVS
-# system-id. If they don't match, gateway traffic has no host to schedule to.
-NODE_NAME="node1"
-CHASSIS_ID="chassis-${NODE_NAME}"
+# Chassis ID — let setup-ovn.sh auto-detect from hostname. This ensures the
+# system-id matches what ovn-controller registers in the SBDB, which is what
+# vpcd discovers at startup for gateway scheduling.
 
-echo "Re-initializing OVN (chassis-id: $CHASSIS_ID)"
-./scripts/setup-ovn.sh --management $SETUP_OVN_FLAGS --chassis-id="$CHASSIS_ID"
+echo "Re-initializing OVN"
+./scripts/setup-ovn.sh --management $SETUP_OVN_FLAGS
 
 echo "Initializing platform"
 ADMIN_INIT_ARGS="--region $REGION --az ${REGION}a --node node1 --nodes 1"
@@ -198,7 +195,6 @@ if [ -n "$WAN_IFACE" ] && [ -n "$WAN_GW" ]; then
         ADMIN_INIT_ARGS="$ADMIN_INIT_ARGS --external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=${WAN_GW} --external-prefix-len=${WAN_PREFIX}"
     fi
 fi
-./bin/spx admin init $ADMIN_INIT_ARGS
 
 # Re-initialize with --force to regenerate credentials, certs, config, and
 # update ~/.aws/credentials. Must carry the same args so external networking
@@ -212,8 +208,16 @@ if [ ! -f ~/.ssh/spinifex-key.pub ]; then
     ssh-keygen -t ed25519 -f ~/.ssh/spinifex-key -N ""
 fi
 
+# Trust the Spinifex CA certificate so TLS connections to predastore/awsgw work
+if [ -f "$HOME/spinifex/config/ca.pem" ]; then
+    echo "Adding Spinifex CA certificate to system trust store..."
+    sudo cp "$HOME/spinifex/config/ca.pem" /usr/local/share/ca-certificates/spinifex-ca.crt
+    sudo update-ca-certificates
+fi
+
 # Enable pprof for development
-PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
+#PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
+./scripts/start-dev.sh --build
 
 export AWS_PROFILE=spinifex
 
@@ -252,4 +256,46 @@ fi
 
 aws ec2 describe-images
 
-echo "Reset complete, fresh AMI imported, proceed to creating instances"
+# --- Launch a smoke-test instance ---
+echo "Launching smoke-test instance..."
+
+# Pick instance type based on CPU vendor
+if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+    INSTANCE_TYPE="t3a.small"
+else
+    INSTANCE_TYPE="t3.small"
+fi
+
+# Get the AMI we just imported
+AMI_ID=$(aws ec2 describe-images --query "Images[0].ImageId" --output text)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "❌ No AMI found, skipping instance launch"
+    exit 0
+fi
+
+# Find a public subnet (first available)
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --filters "Name=map-public-ip-on-launch,Values=true" \
+    --query "Subnets[0].SubnetId" --output text 2>/dev/null)
+# Fallback: just grab the first subnet if no public-tagged one exists
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    SUBNET_ID=$(aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text)
+fi
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "❌ No subnet found, skipping instance launch"
+    exit 0
+fi
+
+echo "  AMI: $AMI_ID"
+echo "  Instance type: $INSTANCE_TYPE"
+echo "  Subnet: $SUBNET_ID"
+
+INSTANCE_ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+
+echo "✅ Reset complete — instance $INSTANCE_ID launched ($INSTANCE_TYPE)"

scripts/setup-ovn.sh

@@ -614,15 +614,16 @@ sudo tee "$LOGROTATE_FILE" >/dev/null <<'LOGROTATE'
 LOGROTATE
 echo "  logrotate config: $LOGROTATE_FILE (daily, 100M max, 3 rotations, explicit filenames)"
 
-# --- Step 10: Disable auto-start on boot ---
-# start-dev.sh / stop-dev.sh manage the OVN lifecycle. Without spinifex services
-# running, ovn-controller spins in a tight reconnect loop burning CPU.
+# --- Step 10: Enable auto-start on boot ---
+# OVN services should start with the system in production. ovn-controller uses
+# exponential backoff if the SB DB isn't ready yet, so CPU impact is negligible.
+# In dev, stop-dev.sh explicitly stops OVN when not needed.
 echo ""
-echo "Step 10: Disabling OVN auto-start on boot (start-dev.sh will manage lifecycle)..."
-sudo systemctl disable openvswitch-switch 2>/dev/null || true
-sudo systemctl disable ovn-controller 2>/dev/null || true
-echo "  openvswitch-switch: disabled on boot"
-echo "  ovn-controller: disabled on boot"
+echo "Step 10: Enabling OVN auto-start on boot..."
+sudo systemctl enable openvswitch-switch 2>/dev/null || true
+sudo systemctl enable ovn-controller 2>/dev/null || true
+echo "  openvswitch-switch: enabled on boot"
+echo "  ovn-controller: enabled on boot"
 
 # --- Step 11: Health check ---
 echo ""

scripts/setup.sh

@@ -196,7 +196,8 @@ download_spinifex() {
         fi
         info "Checksum verified"
     else
-        warn "Checksum not available, skipping verification"
+        rm -rf "$SPINIFEX_TMPDIR"
+        fatal "Checksum not available at $CHECKSUM_URL. Cannot verify download integrity."
     fi
 
     # Extract
@@ -233,6 +234,15 @@ install_files() {
         $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
         info "  /usr/local/share/spinifex/setup-ovn.sh"
     fi
+
+    # Service helper scripts
+    if [ -d "$EXTRACT_DIR/scripts" ]; then
+        for script in "$EXTRACT_DIR"/scripts/*.sh; do
+            $SUDO install -o "$SPINIFEX_USER" -g "$SPINIFEX_GROUP" -m 0755 \
+                "$script" "/var/lib/spinifex/$(basename "$script")"
+            info "  /var/lib/spinifex/$(basename "$script")"
+        done
+    fi
 }
 
 # --- Create directories ---
@@ -341,16 +351,17 @@ print_summary() {
     echo "  Next steps:"
     echo ""
     echo "  1. Setup OVN networking:"
-    echo "     sudo /usr/local/share/spinifex/setup-ovn.sh --management"
-    echo "     # If WAN is a physical NIC (not a bridge), add:"
-    echo "     #   --wan-bridge=br-wan --wan-iface=<WAN_NIC>   (dedicated WAN NIC)"
-    echo "     #   --macvlan --wan-iface=<WAN_NIC>              (single-NIC, SSH-safe)"
+    echo "     If your WAN interface is already a bridge (e.g. br-wan):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management"
     echo ""
-    echo "  2. Initialize a region:"
-    echo "     sudo spx admin init --region <region> --az <az> --node node1 --nodes 1"
+    echo "     If your WAN is a physical NIC:"
+    echo "       # Dedicated WAN NIC (not your SSH connection):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
+    echo "       # Single-NIC host (SSH-safe macvlan):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --macvlan --wan-iface=enp0s3"
     echo ""
-    echo "  2. Setup OVN networking:"
-    echo "     sudo /usr/local/share/spinifex/setup-ovn.sh --management"
+    echo "  2. Initialize:"
+    echo "     sudo spx admin init --node node1 --nodes 1"
     echo ""
     echo "  3. Start services:"
     echo "     sudo systemctl start spinifex.target"

scripts/start-multi-nats.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-SPINIFEX_CONFIG_PATH=~/node1/config/nats/nats.conf ./bin/spx service nats start

spinifex/admin/admin.go

@@ -427,53 +427,109 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
 	return nil
 }
 
+// DiscoverLocalIPs enumerates all non-loopback network interface addresses on
+// this machine and returns them as strings. Link-local addresses are excluded.
+func DiscoverLocalIPs() []string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		slog.Warn("failed to enumerate network interfaces", "error", err)
+		return nil
+	}
+
+	seen := make(map[string]struct{})
+	var ips []string
+
+	for _, iface := range ifaces {
+		// Skip loopback and down interfaces.
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			slog.Debug("failed to get addresses for interface", "iface", iface.Name, "error", err)
+			continue
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+				continue
+			}
+			s := ip.String()
+			if _, ok := seen[s]; !ok {
+				seen[s] = struct{}{}
+				ips = append(ips, s)
+			}
+		}
+	}
+	return ips
+}
+
+// DiscoverHostname returns the system hostname suitable for use as a DNS SAN.
+// Returns an empty string if the hostname cannot be determined or is "localhost".
+func DiscoverHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		slog.Warn("failed to determine system hostname", "error", err)
+		return ""
+	}
+	if hostname == "" || hostname == "localhost" {
+		return ""
+	}
+	return hostname
+}
+
 // GenerateSignedCert generates a server certificate signed by the CA.
 // extraIPs are additional IP addresses to include in the certificate's SANs.
+// All non-loopback interface IPs on the local machine are automatically included.
 func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
-	// Load CA certificate
+	return GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, nil)
+}
+
+// GenerateSignedCertWithDNS generates a server certificate signed by the CA.
+// All non-loopback interface IPs and the machine hostname are automatically
+// included. extraIPs and extraDNS allow adding additional SANs.
+func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, extraIPs, extraDNS []string) error {
 	caCertPEM, err := os.ReadFile(caCertPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA cert: %w", err)
 	}
-
 	caCertBlock, _ := pem.Decode(caCertPEM)
 	if caCertBlock == nil {
 		return fmt.Errorf("failed to decode CA cert PEM")
 	}
-
 	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA cert: %w", err)
 	}
 
-	// Load CA private key
 	caKeyPEM, err := os.ReadFile(caKeyPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA key: %w", err)
 	}
-
 	caKeyBlock, _ := pem.Decode(caKeyPEM)
 	if caKeyBlock == nil {
 		return fmt.Errorf("failed to decode CA key PEM")
 	}
-
 	caPrivateKey, err := x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA private key: %w", err)
 	}
-
 	caRSAKey, ok := caPrivateKey.(*rsa.PrivateKey)
 	if !ok {
 		return fmt.Errorf("CA key is not RSA")
 	}
 
-	// Generate server private key
 	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
 		return fmt.Errorf("failed to generate server private key: %w", err)
 	}
 
-	// Create server certificate template
 	notBefore := time.Now()
 	notAfter := notBefore.Add(365 * 24 * time.Hour) // 1 year for server certs
 
@@ -482,20 +538,51 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}
 
-	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	// Build IP list: localhost IPs + auto-discovered interface IPs + explicit extras.
+	seen := map[string]struct{}{"127.0.0.1": {}, "::1": {}}
 	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	addIP := func(s string) {
+		if s == "" || s == "0.0.0.0" {
+			return
+		}
+		if _, ok := seen[s]; ok {
+			return
+		}
+		if parsed := net.ParseIP(s); parsed != nil {
+			seen[s] = struct{}{}
+			ipAddresses = append(ipAddresses, parsed)
+		}
+	}
+	for _, ip := range DiscoverLocalIPs() {
+		addIP(ip)
+	}
 	for _, ip := range extraIPs {
-		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
-			if parsed := net.ParseIP(ip); parsed != nil {
-				ipAddresses = append(ipAddresses, parsed)
-			}
+		addIP(ip)
+	}
+
+	// Build DNS names: always include localhost, plus hostname if available.
+	dnsSeen := map[string]struct{}{"localhost": {}}
+	dnsNames := []string{"localhost"}
+	if hostname := DiscoverHostname(); hostname != "" {
+		if _, ok := dnsSeen[hostname]; !ok {
+			dnsSeen[hostname] = struct{}{}
+			dnsNames = append(dnsNames, hostname)
+		}
+	}
+	for _, dns := range extraDNS {
+		if dns == "" {
+			continue
+		}
+		if _, ok := dnsSeen[dns]; !ok {
+			dnsSeen[dns] = struct{}{}
+			dnsNames = append(dnsNames, dns)
 		}
 	}
 
 	serverTemplate := x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
-			CommonName:   "localhost",
+			CommonName:   "Spinifex Server",
 			Organization: []string{"Spinifex Platform"},
 		},
 		NotBefore:             notBefore,
@@ -503,39 +590,33 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
-		DNSNames:              []string{"localhost"},
+		DNSNames:              dnsNames,
 		IPAddresses:           ipAddresses,
 	}
 
-	// Sign the server certificate with the CA
 	serverDerBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, caCert, &serverPrivateKey.PublicKey, caRSAKey)
 	if err != nil {
 		return fmt.Errorf("failed to create server certificate: %w", err)
 	}
 
-	// Write server certificate to file
 	certOut, err := os.Create(certPath)
 	if err != nil {
 		return fmt.Errorf("failed to create cert file: %w", err)
 	}
 	defer certOut.Close()
-
 	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverDerBytes}); err != nil {
 		return fmt.Errorf("failed to write cert: %w", err)
 	}
 
-	// Write server private key to file
 	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to create key file: %w", err)
 	}
 	defer keyOut.Close()
-
 	privBytes, err := x509.MarshalPKCS8PrivateKey(serverPrivateKey)
 	if err != nil {
 		return fmt.Errorf("failed to marshal private key: %w", err)
 	}
-
 	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
 		return fmt.Errorf("failed to write key: %w", err)
 	}

spinifex/admin/admin_test.go

@@ -335,7 +335,7 @@ func TestGenerateSignedCert(t *testing.T) {
 		cert, err := x509.ParseCertificate(block.Bytes)
 		require.NoError(t, err)
 		assert.False(t, cert.IsCA)
-		assert.Equal(t, "localhost", cert.Subject.CommonName)
+		assert.Equal(t, "Spinifex Server", cert.Subject.CommonName)
 		assert.Contains(t, cert.DNSNames, "localhost")
 
 		hasLoopback := false
@@ -346,6 +346,9 @@ func TestGenerateSignedCert(t *testing.T) {
 		}
 		assert.True(t, hasLoopback)
 
+		// Auto-discovered IPs should be present (at least loopback + any interface IPs).
+		assert.GreaterOrEqual(t, len(cert.IPAddresses), 2)
+
 		// Verify against CA
 		caCertPEM, _ := os.ReadFile(caCertPath)
 		caBlock, _ := pem.Decode(caCertPEM)
@@ -392,7 +395,19 @@ func TestGenerateSignedCert(t *testing.T) {
 		block, _ := pem.Decode(certPEM)
 		cert, _ := x509.ParseCertificate(block.Bytes)
 
-		assert.Len(t, cert.IPAddresses, 2)
+		// Should have loopback (127.0.0.1, ::1) + auto-discovered interface IPs.
+		// Duplicates (127.0.0.1, ::1) and specials (0.0.0.0, "") must not add extras.
+		// Count unique IPs from a clean run without extras.
+		baseDir := t.TempDir()
+		baseCert := filepath.Join(baseDir, "server.pem")
+		baseKey := filepath.Join(baseDir, "server.key")
+		require.NoError(t, GenerateSignedCert(baseCert, baseKey, caCertPath, caKeyPath))
+		basePEM, _ := os.ReadFile(baseCert)
+		baseBlock, _ := pem.Decode(basePEM)
+		baseParsed, _ := x509.ParseCertificate(baseBlock.Bytes)
+
+		assert.Equal(t, len(baseParsed.IPAddresses), len(cert.IPAddresses),
+			"passing duplicate/special IPs should not add extra entries")
 	})
 
 	t.Run("InvalidCACert", func(t *testing.T) {
@@ -406,6 +421,178 @@ func TestGenerateSignedCert(t *testing.T) {
 	})
 }
 
+func TestDiscoverLocalIPs(t *testing.T) {
+	t.Parallel()
+	ips := DiscoverLocalIPs()
+	// Should find at least one non-loopback IP on any machine with a network interface.
+	assert.NotEmpty(t, ips, "expected at least one non-loopback IP")
+
+	for _, ip := range ips {
+		parsed := net.ParseIP(ip)
+		require.NotNil(t, parsed, "returned IP should be parseable: %s", ip)
+		assert.False(t, parsed.IsLoopback(), "should not include loopback: %s", ip)
+		assert.False(t, parsed.IsLinkLocalUnicast(), "should not include link-local: %s", ip)
+	}
+
+	// No duplicates.
+	seen := make(map[string]struct{})
+	for _, ip := range ips {
+		_, dup := seen[ip]
+		assert.False(t, dup, "duplicate IP: %s", ip)
+		seen[ip] = struct{}{}
+	}
+}
+
+func TestDiscoverHostname(t *testing.T) {
+	t.Parallel()
+	hostname := DiscoverHostname()
+	// We can't assert a specific value, but if non-empty it shouldn't be "localhost".
+	if hostname != "" {
+		assert.NotEqual(t, "localhost", hostname)
+	}
+}
+
+func TestGenerateSignedCertWithDNS_DedupDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Pass "localhost" as extra DNS — should not duplicate the built-in entry.
+	// Also pass empty strings and duplicates — should be ignored.
+	extraDNS := []string{"localhost", "", "example.local", "example.local", ""}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	// Count occurrences of "localhost" — must be exactly 1.
+	localhostCount := 0
+	exampleCount := 0
+	for _, dns := range cert.DNSNames {
+		if dns == "localhost" {
+			localhostCount++
+		}
+		if dns == "example.local" {
+			exampleCount++
+		}
+	}
+	assert.Equal(t, 1, localhostCount, "localhost should appear exactly once")
+	assert.Equal(t, 1, exampleCount, "example.local should appear exactly once")
+	assert.Contains(t, cert.DNSNames, "example.local")
+}
+
+func TestGenerateSignedCertWithDNS_NilSlices(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Both slices nil — should still produce a valid cert with defaults.
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, nil))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.GreaterOrEqual(t, len(cert.IPAddresses), 2, "should have at least loopback IPs")
+}
+
+func TestGenerateSignedCertWithDNS_InvalidCA(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// Missing CA cert file.
+	err := GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "missing-ca.pem"), filepath.Join(dir, "missing-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to read CA cert")
+
+	// Corrupt CA cert.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.pem"), []byte("not-pem"), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.key"), []byte("not-pem"), 0600))
+	err = GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "bad-ca.pem"), filepath.Join(dir, "bad-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+}
+
+func TestGenerateSignedCertWithDNS_ExtraDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	extraIPs := []string{"10.0.0.42"}
+	extraDNS := []string{"spinifex.local", "node1.spinifex.local"}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.Contains(t, cert.DNSNames, "spinifex.local")
+	assert.Contains(t, cert.DNSNames, "node1.spinifex.local")
+
+	hasExtraIP := false
+	for _, ip := range cert.IPAddresses {
+		if ip.Equal(net.ParseIP("10.0.0.42")) {
+			hasExtraIP = true
+		}
+	}
+	assert.True(t, hasExtraIP, "cert should contain extra IP 10.0.0.42")
+
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+func TestGenerateSignedCert_IncludesHostname(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+	require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	hostname := DiscoverHostname()
+	if hostname != "" {
+		assert.Contains(t, cert.DNSNames, hostname,
+			"cert DNS SANs should include the machine hostname")
+	}
+}
+
 func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
 	t.Parallel()
 	dir := t.TempDir()

spinifex/admin/telemetry.go

@@ -0,0 +1,114 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+)
+
+const defaultTelemetryURL = "https://install.mulgadc.com/install"
+
+// TelemetryPayload is the JSON body sent to the telemetry endpoint.
+type TelemetryPayload struct {
+	MachineID    string `json:"machine_id"`
+	Event        string `json:"event"` // "init" or "join"
+	Region       string `json:"region"`
+	AZ           string `json:"az"`
+	Node         string `json:"node"`
+	Nodes        int    `json:"nodes"`
+	BindIP       string `json:"bind_ip"`
+	Version      string `json:"version"`
+	Arch         string `json:"arch"`
+	OS           string `json:"os"`
+	ExternalMode string `json:"external_mode,omitempty"`
+	Timestamp    string `json:"timestamp"`
+
+	// URL overrides the telemetry endpoint (for testing only). Not serialized.
+	URL string `json:"-"`
+}
+
+// SendTelemetry posts the payload to the telemetry endpoint.
+// It respects the context deadline and logs at Debug level only.
+func SendTelemetry(ctx context.Context, payload TelemetryPayload) {
+	if payload.Arch == "" {
+		payload.Arch = runtime.GOARCH
+	}
+	if payload.OS == "" {
+		payload.OS = runtime.GOOS
+	}
+	if payload.Timestamp == "" {
+		payload.Timestamp = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	url := payload.URL
+	if url == "" {
+		url = defaultTelemetryURL
+	}
+
+	body, err := json.Marshal(payload)
+	if err != nil {
+		slog.Debug("telemetry: failed to marshal payload", "error", err)
+		return
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		slog.Debug("telemetry: failed to create request", "error", err)
+		return
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		slog.Debug("telemetry: request failed", "error", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	slog.Debug("telemetry: sent", "status", resp.StatusCode, "event", payload.Event)
+}
+
+// ReadMachineID returns a stable anonymous identifier for this machine.
+// Reads /etc/machine-id (standard Linux UUID). Falls back to a SHA-256
+// hash of hostname + first non-loopback MAC address.
+func ReadMachineID() string {
+	data, err := os.ReadFile("/etc/machine-id")
+	if err == nil {
+		id := strings.TrimSpace(string(data))
+		if id != "" {
+			return id
+		}
+	}
+
+	// Fallback: hash hostname + MAC
+	hostname, _ := os.Hostname()
+	mac := firstMAC()
+	hash := sha256.Sum256([]byte(hostname + mac))
+	return fmt.Sprintf("%x", hash[:16])
+}
+
+// firstMAC returns the MAC address of the first non-loopback, up interface.
+func firstMAC() string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return ""
+	}
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		if mac := iface.HardwareAddr.String(); mac != "" {
+			return mac
+		}
+	}
+	return ""
+}

spinifex/admin/telemetry_test.go

@@ -0,0 +1,107 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadMachineID_ReturnsNonEmpty(t *testing.T) {
+	t.Parallel()
+	id := ReadMachineID()
+	assert.NotEmpty(t, id, "machine ID should never be empty")
+	assert.Greater(t, len(id), 8, "machine ID should be a reasonable length")
+}
+
+func TestSendTelemetry_PostsCorrectPayload(t *testing.T) {
+	t.Parallel()
+	var received TelemetryPayload
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodPost, r.Method)
+		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+		require.NoError(t, json.Unmarshal(body, &received))
+
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	SendTelemetry(ctx, TelemetryPayload{
+		MachineID: "test-machine-123",
+		Event:     "init",
+		Region:    "ap-southeast-2",
+		AZ:        "ap-southeast-2a",
+		Node:      "node1",
+		Nodes:     3,
+		BindIP:    "10.11.12.1",
+		Version:   "v0.5.0",
+		URL:       server.URL,
+	})
+
+	assert.Equal(t, "test-machine-123", received.MachineID)
+	assert.Equal(t, "init", received.Event)
+	assert.Equal(t, "ap-southeast-2", received.Region)
+	assert.Equal(t, 3, received.Nodes)
+	assert.Equal(t, "v0.5.0", received.Version)
+	assert.NotEmpty(t, received.Arch, "arch should be auto-filled")
+	assert.NotEmpty(t, received.OS, "os should be auto-filled")
+	assert.NotEmpty(t, received.Timestamp, "timestamp should be auto-filled")
+}
+
+func TestSendTelemetry_RespectsTimeout(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(10 * time.Second)
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "timeout-test", URL: server.URL})
+	elapsed := time.Since(start)
+
+	assert.Less(t, elapsed, 2*time.Second, "should respect context timeout")
+}
+
+func TestSendTelemetry_HandlesServerError(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "error-test", URL: server.URL})
+}
+
+func TestSendTelemetry_HandlesUnreachableServer(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	// URL pointing to a port that's not listening
+	SendTelemetry(ctx, TelemetryPayload{MachineID: "unreachable-test", URL: "http://127.0.0.1:1"})
+}
+
+func TestFirstMAC_ReturnsNonEmpty(t *testing.T) {
+	t.Parallel()
+	mac := firstMAC()
+	assert.NotEmpty(t, mac, "expected at least one MAC address")
+}

spinifex/albagent/agent.go

@@ -0,0 +1,305 @@
+// Package albagent implements the NATS config agent that runs inside ALB VMs.
+// It subscribes to config updates, writes HAProxy configuration, and reloads
+// the HAProxy process.
+package albagent
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+const (
+	// Default paths for HAProxy config and PID file.
+	DefaultConfigPath = "/etc/haproxy/haproxy.cfg"
+	DefaultPIDPath    = "/run/haproxy.pid"
+
+	// NATS topic patterns.
+	configTopicPrefix = "elbv2.alb."
+	configTopicSuffix = ".config"
+	pingTopicSuffix   = ".ping"
+)
+
+// Agent manages HAProxy configuration inside an ALB VM.
+type Agent struct {
+	lbID       string
+	natsURL    string
+	configPath string
+	pidPath    string
+
+	nc   *nats.Conn
+	subs []*nats.Subscription
+	mu   sync.Mutex
+
+	// For testing: override the reload function.
+	reloadFn func(configPath, pidPath string) error
+}
+
+// New creates a new ALB agent for the given load balancer.
+func New(lbID, natsURL string) (*Agent, error) {
+	if lbID == "" {
+		return nil, fmt.Errorf("lbID is required")
+	}
+	if natsURL == "" {
+		natsURL = "nats://127.0.0.1:4222"
+	}
+
+	return &Agent{
+		lbID:       lbID,
+		natsURL:    natsURL,
+		configPath: DefaultConfigPath,
+		pidPath:    DefaultPIDPath,
+		reloadFn:   reloadHAProxy,
+	}, nil
+}
+
+// ConfigTopic returns the NATS topic for config updates.
+func ConfigTopic(lbID string) string {
+	return configTopicPrefix + lbID + configTopicSuffix
+}
+
+// PingTopic returns the NATS topic for health pings.
+func PingTopic(lbID string) string {
+	return configTopicPrefix + lbID + pingTopicSuffix
+}
+
+// Start connects to NATS and subscribes to config and ping topics.
+func (a *Agent) Start() error {
+	nc, err := nats.Connect(a.natsURL,
+		nats.Name("alb-agent-"+a.lbID),
+		nats.MaxReconnects(-1),
+		nats.ReconnectWait(2*time.Second),
+		nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
+			slog.Warn("NATS disconnected", "err", err)
+		}),
+		nats.ReconnectHandler(func(_ *nats.Conn) {
+			slog.Info("NATS reconnected")
+		}),
+	)
+	if err != nil {
+		return fmt.Errorf("connect to NATS at %s: %w", a.natsURL, err)
+	}
+	a.nc = nc
+
+	// Subscribe to config updates
+	configSub, err := nc.Subscribe(ConfigTopic(a.lbID), a.handleConfig)
+	if err != nil {
+		nc.Close()
+		return fmt.Errorf("subscribe to config topic: %w", err)
+	}
+
+	// Subscribe to health pings (request/reply)
+	pingSub, err := nc.Subscribe(PingTopic(a.lbID), a.handlePing)
+	if err != nil {
+		nc.Close()
+		return fmt.Errorf("subscribe to ping topic: %w", err)
+	}
+
+	a.mu.Lock()
+	a.subs = append(a.subs, configSub, pingSub)
+	a.mu.Unlock()
+
+	// Flush to ensure the server has processed our subscriptions before
+	// we report that the agent is started and ready.
+	if err := nc.Flush(); err != nil {
+		nc.Close()
+		return fmt.Errorf("flush subscriptions: %w", err)
+	}
+
+	slog.Info("Agent started",
+		"lbId", a.lbID,
+		"configTopic", ConfigTopic(a.lbID),
+		"pingTopic", PingTopic(a.lbID),
+	)
+	return nil
+}
+
+// Stop gracefully shuts down the agent.
+func (a *Agent) Stop() {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	for _, sub := range a.subs {
+		if err := sub.Unsubscribe(); err != nil {
+			slog.Warn("Failed to unsubscribe", "err", err)
+		}
+	}
+	a.subs = nil
+
+	if a.nc != nil {
+		a.nc.Close()
+		a.nc = nil
+	}
+
+	slog.Info("Agent stopped", "lbId", a.lbID)
+}
+
+// handleConfig processes a config update message. The message payload is the
+// raw HAProxy config text. It writes the config to disk and reloads HAProxy.
+func (a *Agent) handleConfig(msg *nats.Msg) {
+	config := string(msg.Data)
+	if config == "" {
+		slog.Warn("Received empty config, ignoring")
+		a.respond(msg, "error", "empty config")
+		return
+	}
+
+	slog.Info("Received config update", "lbId", a.lbID, "size", len(config))
+
+	// Write config to disk
+	if err := WriteConfig(a.configPath, config); err != nil {
+		slog.Error("Failed to write config", "path", a.configPath, "err", err)
+		a.respond(msg, "error", err.Error())
+		return
+	}
+
+	// Reload HAProxy
+	if err := a.reloadFn(a.configPath, a.pidPath); err != nil {
+		slog.Error("Failed to reload HAProxy", "err", err)
+		a.respond(msg, "error", err.Error())
+		return
+	}
+
+	slog.Info("Config applied and HAProxy reloaded", "lbId", a.lbID)
+	a.respond(msg, "ok", "")
+}
+
+// PingResponse is the JSON response to a health ping.
+type PingResponse struct {
+	Status    string `json:"status"`
+	LBID      string `json:"lb_id"`
+	HAProxy   bool   `json:"haproxy_running"`
+	ConfigAge int64  `json:"config_age_seconds"`
+}
+
+// handlePing responds to health check requests.
+func (a *Agent) handlePing(msg *nats.Msg) {
+	haproxyRunning := isHAProxyRunning(a.pidPath)
+	configAge := configAgeSeconds(a.configPath)
+
+	resp := PingResponse{
+		Status:    "ok",
+		LBID:      a.lbID,
+		HAProxy:   haproxyRunning,
+		ConfigAge: configAge,
+	}
+
+	data, err := json.Marshal(resp)
+	if err != nil {
+		slog.Error("Failed to marshal ping response", "err", err)
+		return
+	}
+	if err := msg.Respond(data); err != nil {
+		slog.Warn("Failed to respond to ping", "err", err)
+	}
+}
+
+// respond sends a JSON reply to a NATS message if it has a reply subject.
+func (a *Agent) respond(msg *nats.Msg, status, errMsg string) {
+	if msg.Reply == "" {
+		return
+	}
+	resp := map[string]string{"status": status}
+	if errMsg != "" {
+		resp["error"] = errMsg
+	}
+	data, err := json.Marshal(resp)
+	if err != nil {
+		slog.Error("Failed to marshal response", "err", err)
+		return
+	}
+	if err := msg.Respond(data); err != nil {
+		slog.Warn("Failed to respond", "err", err)
+	}
+}
+
+// WriteConfig atomically writes an HAProxy config file.
+// It writes to a temp file first, then renames for atomicity.
+func WriteConfig(path, content string) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("create config dir: %w", err)
+	}
+
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
+		return fmt.Errorf("write temp config: %w", err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("rename config: %w", err)
+	}
+	return nil
+}
+
+// reloadHAProxy starts or reloads the HAProxy process.
+// If HAProxy is running (PID file exists and process alive), it does a
+// graceful reload with -sf. Otherwise it starts a fresh instance.
+func reloadHAProxy(configPath, pidPath string) error {
+	// Ensure the stats socket directory exists (the config may reference
+	// /tmp/spinifex-haproxy/ which doesn't exist on fresh Alpine VMs).
+	_ = os.MkdirAll("/tmp/spinifex-haproxy", 0o750)
+
+	oldPID := readPID(pidPath)
+
+	var cmd *exec.Cmd
+	if oldPID > 0 {
+		// Graceful reload: new worker starts, old workers finish in-flight requests
+		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D", "-sf", strconv.Itoa(oldPID))
+	} else {
+		// Fresh start
+		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D")
+	}
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("haproxy: %s: %w", strings.TrimSpace(string(out)), err)
+	}
+	return nil
+}
+
+// readPID reads the HAProxy PID from the PID file. Returns 0 if unavailable.
+func readPID(pidPath string) int {
+	data, err := os.ReadFile(pidPath)
+	if err != nil {
+		return 0
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(data)))
+	if err != nil {
+		return 0
+	}
+	// Check if process is still alive
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return 0
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return 0
+	}
+	return pid
+}
+
+// isHAProxyRunning checks if HAProxy is running via its PID file.
+func isHAProxyRunning(pidPath string) bool {
+	return readPID(pidPath) > 0
+}
+
+// configAgeSeconds returns how many seconds since the config file was last modified.
+// Returns -1 if the file doesn't exist.
+func configAgeSeconds(configPath string) int64 {
+	info, err := os.Stat(configPath)
+	if err != nil {
+		return -1
+	}
+	return int64(time.Since(info.ModTime()).Seconds())
+}

spinifex/albagent/agent_test.go

@@ -0,0 +1,342 @@
+package albagent
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	natstest "github.com/nats-io/nats-server/v2/test"
+	"github.com/nats-io/nats.go"
+)
+
+func startTestNATS(t *testing.T) *natsserver.Server {
+	t.Helper()
+	opts := natstest.DefaultTestOptions
+	opts.Port = -1 // random port
+	ns := natstest.RunServer(&opts)
+	t.Cleanup(ns.Shutdown)
+	return ns
+}
+
+func TestNew(t *testing.T) {
+	agent, err := New("lb-test123", "nats://localhost:4222")
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if agent.lbID != "lb-test123" {
+		t.Errorf("lbID = %q, want %q", agent.lbID, "lb-test123")
+	}
+}
+
+func TestNew_EmptyLBID(t *testing.T) {
+	_, err := New("", "nats://localhost:4222")
+	if err == nil {
+		t.Fatal("expected error for empty lbID")
+	}
+}
+
+func TestNew_DefaultNATSURL(t *testing.T) {
+	agent, err := New("lb-test", "")
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if agent.natsURL != "nats://127.0.0.1:4222" {
+		t.Errorf("natsURL = %q, want default", agent.natsURL)
+	}
+}
+
+func TestConfigTopic(t *testing.T) {
+	topic := ConfigTopic("lb-abc123")
+	if topic != "elbv2.alb.lb-abc123.config" {
+		t.Errorf("ConfigTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.config")
+	}
+}
+
+func TestPingTopic(t *testing.T) {
+	topic := PingTopic("lb-abc123")
+	if topic != "elbv2.alb.lb-abc123.ping" {
+		t.Errorf("PingTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.ping")
+	}
+}
+
+func TestWriteConfig(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "haproxy.cfg")
+
+	content := "global\n  log stdout\n"
+	if err := WriteConfig(path, content); err != nil {
+		t.Fatalf("WriteConfig: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(data) != content {
+		t.Errorf("config = %q, want %q", string(data), content)
+	}
+}
+
+func TestWriteConfig_CreatesDir(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "subdir", "haproxy.cfg")
+
+	if err := WriteConfig(path, "test"); err != nil {
+		t.Fatalf("WriteConfig: %v", err)
+	}
+
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("config file not created: %v", err)
+	}
+}
+
+func TestWriteConfig_Atomic(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "haproxy.cfg")
+
+	// Write initial config
+	if err := WriteConfig(path, "initial"); err != nil {
+		t.Fatalf("WriteConfig initial: %v", err)
+	}
+
+	// Overwrite with new config
+	if err := WriteConfig(path, "updated"); err != nil {
+		t.Fatalf("WriteConfig updated: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	if string(data) != "updated" {
+		t.Errorf("config = %q, want %q", string(data), "updated")
+	}
+
+	// No temp file left behind
+	if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) {
+		t.Error("temp file should not exist after successful write")
+	}
+}
+
+func TestConfigAgeSeconds(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.cfg")
+
+	// Non-existent file
+	if age := configAgeSeconds(path); age != -1 {
+		t.Errorf("age of non-existent file = %d, want -1", age)
+	}
+
+	// Write file and check age
+	os.WriteFile(path, []byte("test"), 0o644)
+	age := configAgeSeconds(path)
+	if age < 0 || age > 2 {
+		t.Errorf("age = %d, expected 0-2 seconds", age)
+	}
+}
+
+func TestIsHAProxyRunning_NoPIDFile(t *testing.T) {
+	if isHAProxyRunning("/nonexistent/haproxy.pid") {
+		t.Error("expected false for non-existent PID file")
+	}
+}
+
+func TestReadPID_InvalidContent(t *testing.T) {
+	dir := t.TempDir()
+	pidFile := filepath.Join(dir, "haproxy.pid")
+
+	os.WriteFile(pidFile, []byte("not-a-number\n"), 0o644)
+	if pid := readPID(pidFile); pid != 0 {
+		t.Errorf("readPID = %d, want 0 for invalid content", pid)
+	}
+}
+
+func TestReadPID_DeadProcess(t *testing.T) {
+	dir := t.TempDir()
+	pidFile := filepath.Join(dir, "haproxy.pid")
+
+	// Use a very high PID that's unlikely to be running
+	os.WriteFile(pidFile, []byte("999999999\n"), 0o644)
+	if pid := readPID(pidFile); pid != 0 {
+		t.Errorf("readPID = %d, want 0 for dead process", pid)
+	}
+}
+
+func TestAgent_StartStop(t *testing.T) {
+	ns := startTestNATS(t)
+
+	agent, err := New("lb-test123", ns.ClientURL())
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+
+	// Verify agent is connected
+	if agent.nc == nil || !agent.nc.IsConnected() {
+		t.Fatal("agent not connected to NATS")
+	}
+
+	agent.Stop()
+
+	if agent.nc != nil {
+		t.Error("nc should be nil after Stop")
+	}
+}
+
+func TestAgent_HandleConfig(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-cfg123", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+
+	// Mock reload to avoid needing real HAProxy
+	reloadCalled := false
+	agent.reloadFn = func(configPath, pidPath string) error {
+		reloadCalled = true
+		return nil
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	// Send a config update via a separate NATS connection
+	nc, err := nats.Connect(ns.ClientURL())
+	if err != nil {
+		t.Fatalf("connect: %v", err)
+	}
+	defer nc.Close()
+
+	haproxyConfig := "global\n  log stdout\ndefaults\n  mode http\n"
+	reply, err := nc.Request(ConfigTopic("lb-cfg123"), []byte(haproxyConfig), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	// Check response
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "ok" {
+		t.Errorf("response status = %q, want %q", resp["status"], "ok")
+	}
+
+	// Verify config was written
+	data, err := os.ReadFile(agent.configPath)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	if string(data) != haproxyConfig {
+		t.Errorf("written config = %q, want %q", string(data), haproxyConfig)
+	}
+
+	if !reloadCalled {
+		t.Error("reload function was not called")
+	}
+}
+
+func TestAgent_HandleConfig_Empty(t *testing.T) {
+	ns := startTestNATS(t)
+
+	agent, _ := New("lb-empty", ns.ClientURL())
+	agent.reloadFn = func(_, _ string) error { return nil }
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(ConfigTopic("lb-empty"), []byte(""), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "error" {
+		t.Errorf("response status = %q, want %q for empty config", resp["status"], "error")
+	}
+}
+
+func TestAgent_HandlePing(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-ping123", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(PingTopic("lb-ping123"), nil, 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp PingResponse
+	if err := json.Unmarshal(reply.Data, &resp); err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+
+	if resp.Status != "ok" {
+		t.Errorf("status = %q, want %q", resp.Status, "ok")
+	}
+	if resp.LBID != "lb-ping123" {
+		t.Errorf("lb_id = %q, want %q", resp.LBID, "lb-ping123")
+	}
+	if resp.HAProxy {
+		t.Error("haproxy_running should be false (no PID file)")
+	}
+	if resp.ConfigAge != -1 {
+		t.Errorf("config_age = %d, want -1 (no config file)", resp.ConfigAge)
+	}
+}
+
+func TestAgent_HandleConfig_ReloadError(t *testing.T) {
+	ns := startTestNATS(t)
+	dir := t.TempDir()
+
+	agent, _ := New("lb-err", ns.ClientURL())
+	agent.configPath = filepath.Join(dir, "haproxy.cfg")
+	agent.pidPath = filepath.Join(dir, "haproxy.pid")
+	agent.reloadFn = func(_, _ string) error {
+		return fmt.Errorf("haproxy binary not found")
+	}
+
+	if err := agent.Start(); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	defer agent.Stop()
+
+	nc, _ := nats.Connect(ns.ClientURL())
+	defer nc.Close()
+
+	reply, err := nc.Request(ConfigTopic("lb-err"), []byte("some config"), 2*time.Second)
+	if err != nil {
+		t.Fatalf("Request: %v", err)
+	}
+
+	var resp map[string]string
+	json.Unmarshal(reply.Data, &resp)
+	if resp["status"] != "error" {
+		t.Errorf("response status = %q, want %q", resp["status"], "error")
+	}
+	if resp["error"] == "" {
+		t.Error("expected error message in response")
+	}
+}

spinifex/awsec2query/parser.go

@@ -186,10 +186,20 @@ func setFieldValue(field reflect.Value, value string) error {
 }
 
 func setSliceField(field reflect.Value, params map[string]string, prefix string) error {
-	// Find all indexed items for this slice
+	// Find all indexed items for this slice.
+	// Supports both EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N) formats.
 	indices := make(map[int]bool)
+	useMember := false
 	for key := range params {
-		if strings.HasPrefix(key, prefix+".") {
+		if strings.HasPrefix(key, prefix+".member.") {
+			parts := strings.Split(key[len(prefix)+len(".member."):], ".")
+			if len(parts) > 0 {
+				if idx, err := strconv.Atoi(parts[0]); err == nil {
+					indices[idx] = true
+					useMember = true
+				}
+			}
+		} else if strings.HasPrefix(key, prefix+".") {
 			parts := strings.Split(key[len(prefix)+1:], ".")
 			if len(parts) > 0 {
 				if idx, err := strconv.Atoi(parts[0]); err == nil {
@@ -221,7 +231,12 @@ func setSliceField(field reflect.Value, params map[string]string, prefix string)
 		}
 
 		elem := slice.Index(idx - 1)
-		indexPrefix := fmt.Sprintf("%s.%d", prefix, idx)
+		var indexPrefix string
+		if useMember {
+			indexPrefix = fmt.Sprintf("%s.member.%d", prefix, idx)
+		} else {
+			indexPrefix = fmt.Sprintf("%s.%d", prefix, idx)
+		}
 
 		// Handle different element types
 		switch elemType.Kind() {

spinifex/awsec2query/parser_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -509,3 +510,81 @@ func TestQueryParamsToStruct_BlockDeviceMappingsWithIOPS(t *testing.T) {
 	assert.Equal(t, int64(500), aws.Int64Value(bdm.Ebs.Throughput))
 	assert.Equal(t, "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", aws.StringValue(bdm.Ebs.KmsKeyId))
 }
+
+// --- ELBv2 member-style list tests ---
+
+func TestQueryParamsToStruct_ELBv2RegisterTargets(t *testing.T) {
+	// ELBv2 uses Targets.member.N.Id format (IAM-style query protocol)
+	args := map[string]string{
+		"Action":              "RegisterTargets",
+		"TargetGroupArn":      "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+		"Targets.member.1.Id": "i-1234567890abcdef0",
+		"Targets.member.2.Id": "i-0987654321fedcba0",
+	}
+
+	input := &elbv2.RegisterTargetsInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123", aws.StringValue(input.TargetGroupArn))
+	assert.Len(t, input.Targets, 2)
+	assert.Equal(t, "i-1234567890abcdef0", aws.StringValue(input.Targets[0].Id))
+	assert.Equal(t, "i-0987654321fedcba0", aws.StringValue(input.Targets[1].Id))
+}
+
+func TestQueryParamsToStruct_ELBv2RegisterTargetsWithPort(t *testing.T) {
+	args := map[string]string{
+		"Action":                "RegisterTargets",
+		"TargetGroupArn":        "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+		"Targets.member.1.Id":   "i-1234567890abcdef0",
+		"Targets.member.1.Port": "8080",
+	}
+
+	input := &elbv2.RegisterTargetsInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Len(t, input.Targets, 1)
+	assert.Equal(t, "i-1234567890abcdef0", aws.StringValue(input.Targets[0].Id))
+	assert.Equal(t, int64(8080), aws.Int64Value(input.Targets[0].Port))
+}
+
+func TestQueryParamsToStruct_ELBv2CreateLoadBalancerMemberSubnets(t *testing.T) {
+	// ELBv2 uses Subnets.member.N format for string slices
+	args := map[string]string{
+		"Action":           "CreateLoadBalancer",
+		"Name":             "my-alb",
+		"Subnets.member.1": "subnet-abc123",
+		"Subnets.member.2": "subnet-def456",
+	}
+
+	input := &elbv2.CreateLoadBalancerInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "my-alb", aws.StringValue(input.Name))
+	assert.Len(t, input.Subnets, 2)
+	assert.Equal(t, "subnet-abc123", aws.StringValue(input.Subnets[0]))
+	assert.Equal(t, "subnet-def456", aws.StringValue(input.Subnets[1]))
+}
+
+func TestQueryParamsToStruct_ELBv2CreateListenerWithActions(t *testing.T) {
+	args := map[string]string{
+		"Action":                                 "CreateListener",
+		"LoadBalancerArn":                        "arn:aws:elasticloadbalancing:us-east-1:000000000001:loadbalancer/app/my-alb/lb-abc123",
+		"Protocol":                               "HTTP",
+		"Port":                                   "80",
+		"DefaultActions.member.1.Type":           "forward",
+		"DefaultActions.member.1.TargetGroupArn": "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123",
+	}
+
+	input := &elbv2.CreateListenerInput{}
+	err := QueryParamsToStruct(args, input)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "HTTP", aws.StringValue(input.Protocol))
+	assert.Equal(t, int64(80), aws.Int64Value(input.Port))
+	assert.Len(t, input.DefaultActions, 1)
+	assert.Equal(t, "forward", aws.StringValue(input.DefaultActions[0].Type))
+	assert.Equal(t, "arn:aws:elasticloadbalancing:us-east-1:000000000001:targetgroup/my-tg/tg-abc123", aws.StringValue(input.DefaultActions[0].TargetGroupArn))
+}

spinifex/awserrors/awserrors.go

@@ -343,6 +343,7 @@ var (
 	ErrorPendingVpcPeeringConnectionLimitExceeded              = "PendingVpcPeeringConnectionLimitExceeded"
 	ErrorPlacementGroupLimitExceeded                           = "PlacementGroupLimitExceeded"
 	ErrorPrivateIpAddressLimitExceeded                         = "PrivateIpAddressLimitExceeded"
+	ErrorRequestEntityTooLarge                                 = "RequestEntityTooLarge"
 	ErrorRequestExpired                                        = "RequestExpired"
 	ErrorRequestLimitExceeded                                  = "RequestLimitExceeded"
 	ErrorRequestResourceCountExceeded                          = "RequestResourceCountExceeded"
@@ -432,6 +433,24 @@ var (
 	ErrorIAMInvalidInput            = ErrorInvalidInput // alias for IAM usage
 	ErrorIAMMalformedPolicyDocument = "MalformedPolicyDocument"
 	ErrorAccessDenied               = "AccessDenied"
+
+	// ELBv2-specific error codes
+	ErrorELBv2LoadBalancerNotFound         = "LoadBalancerNotFound"
+	ErrorELBv2TargetGroupNotFound          = "TargetGroupNotFound"
+	ErrorELBv2ListenerNotFound             = "ListenerNotFound"
+	ErrorELBv2DuplicateLoadBalancer        = "DuplicateLoadBalancerName"
+	ErrorELBv2DuplicateTargetGroup         = "DuplicateTargetGroupName"
+	ErrorELBv2DuplicateListener            = "DuplicateListener"
+	ErrorELBv2TooManyLoadBalancers         = "TooManyLoadBalancers"
+	ErrorELBv2TooManyTargetGroups          = "TooManyTargetGroups"
+	ErrorELBv2TooManyListeners             = "TooManyListeners"
+	ErrorELBv2TooManyTargets               = "TooManyRegistrationsForTargetId"
+	ErrorELBv2InvalidTarget                = "InvalidTarget"
+	ErrorELBv2TargetGroupInUse             = "ResourceInUse"
+	ErrorELBv2InvalidSecurityGroup         = "InvalidSecurityGroup"
+	ErrorELBv2InvalidScheme                = "InvalidScheme"
+	ErrorELBv2SubnetNotFound               = "SubnetNotFound"
+	ErrorELBv2AvailabilityZoneNotSupported = "AvailabilityZoneNotSupported"
 )
 
 // ValidErrorCode returns the error code if it exists in ErrorLookup,
@@ -782,6 +801,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorPendingVpcPeeringConnectionLimitExceeded:              {HTTPCode: 409, Message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
 	ErrorPlacementGroupLimitExceeded:                           {HTTPCode: 400, Message: "You've reached the limit on the number of placement groups that you can have."},
 	ErrorPrivateIpAddressLimitExceeded:                         {HTTPCode: 400, Message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
+	ErrorRequestEntityTooLarge:                                 {HTTPCode: 413, Message: "Request body exceeds the maximum allowed size."},
 	ErrorRequestExpired:                                        {HTTPCode: 403, Message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
 	ErrorRequestLimitExceeded:                                  {HTTPCode: 500, Message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
 	ErrorRequestResourceCountExceeded:                          {HTTPCode: 409, Message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
@@ -870,4 +890,22 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorIAMLimitExceeded:           {HTTPCode: 409, Message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
 	ErrorIAMMalformedPolicyDocument: {HTTPCode: 400, Message: "The policy document is malformed."},
 	ErrorAccessDenied:               {HTTPCode: 403, Message: "User is not authorized to perform this action."},
+
+	// ELBv2 error codes
+	ErrorELBv2LoadBalancerNotFound:         {HTTPCode: 400, Message: "One or more load balancers not found."},
+	ErrorELBv2TargetGroupNotFound:          {HTTPCode: 400, Message: "One or more target groups not found."},
+	ErrorELBv2ListenerNotFound:             {HTTPCode: 400, Message: "One or more listeners not found."},
+	ErrorELBv2DuplicateLoadBalancer:        {HTTPCode: 400, Message: "A load balancer with the same name already exists."},
+	ErrorELBv2DuplicateTargetGroup:         {HTTPCode: 400, Message: "A target group with the same name already exists."},
+	ErrorELBv2DuplicateListener:            {HTTPCode: 400, Message: "A listener with the specified port already exists on this load balancer."},
+	ErrorELBv2TooManyLoadBalancers:         {HTTPCode: 400, Message: "You've reached the limit on the number of load balancers for your account."},
+	ErrorELBv2TooManyTargetGroups:          {HTTPCode: 400, Message: "You've reached the limit on the number of target groups for your account."},
+	ErrorELBv2TooManyListeners:             {HTTPCode: 400, Message: "You've reached the limit on the number of listeners per load balancer."},
+	ErrorELBv2TooManyTargets:               {HTTPCode: 400, Message: "You've reached the limit on the number of times a target can be registered with a target group."},
+	ErrorELBv2InvalidTarget:                {HTTPCode: 400, Message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
+	ErrorELBv2TargetGroupInUse:             {HTTPCode: 400, Message: "The target group is currently in use by a listener or rule."},
+	ErrorELBv2InvalidSecurityGroup:         {HTTPCode: 400, Message: "The specified security group does not exist."},
+	ErrorELBv2InvalidScheme:                {HTTPCode: 400, Message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
+	ErrorELBv2SubnetNotFound:               {HTTPCode: 400, Message: "The specified subnet does not exist."},
+	ErrorELBv2AvailabilityZoneNotSupported: {HTTPCode: 400, Message: "The specified Availability Zone is not supported."},
 }

spinifex/awserrors/awserrors_test.go

@@ -345,6 +345,7 @@ func TestErrorLookup(t *testing.T) {
 		{code: "PendingVpcPeeringConnectionLimitExceeded", http: 409, message: "You've reached the limit on the number of pending VPC peering connections that you can have."},
 		{code: "PlacementGroupLimitExceeded", http: 400, message: "You've reached the limit on the number of placement groups that you can have."},
 		{code: "PrivateIpAddressLimitExceeded", http: 400, message: "You've reached the limit on the number of private IP addresses that you can assign to the specified network interface for that type of instance. For more information, see IP addresses per network interface."},
+		{code: "RequestEntityTooLarge", http: 413, message: "Request body exceeds the maximum allowed size."},
 		{code: "RequestExpired", http: 403, message: "The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for presigned URLs), or the date stamp on the request is more than 15 minutes in the future. If you're using temporary security credentials, this error can also occur if the credentials have expired. For more information, see Temporary security credentials in the IAM User Guide."},
 		{code: "RequestLimitExceeded", http: 500, message: "The maximum request rate permitted by the Amazon EC2 APIs has been exceeded for your account. For best results, use an increasing or variable sleep interval between requests. For more information, see Query API request rate."},
 		{code: "RequestResourceCountExceeded", http: 409, message: "Details in your Spot request exceed the numbers allowed by the Spot service in one of the following ways, depending on the action that generated the error: \u2014If you get this error when you submitted a request for Spot Instances, check the number of Spot Instances specified in your request. The number shouldn't exceed the 3,000 maximum allowed per request. Resend your Spot Instance request and specify a number less than 3,000. If your account's regional Spot request limit is greater than 3,000 instances, you can access these instances by submitting multiple smaller requests. \u2014If you get this error when you sent Describe Spot Instance requests, check the number of requests for Spot Instance data, the amount of data you requested, and how often you sent the request. The frequency with which you requested the data combined with the amount of data exceeds the levels allowed by the Spot service. Try again and submit fewer large Describe requests over longer intervals."},
@@ -433,6 +434,23 @@ func TestErrorLookup(t *testing.T) {
 		{code: "LimitExceeded", http: 409, message: "The request was rejected because it attempted to create resources beyond the current AWS account limits."},
 		{code: "MalformedPolicyDocument", http: 400, message: "The policy document is malformed."},
 		{code: "AccessDenied", http: 403, message: "User is not authorized to perform this action."},
+		// ELBv2 error codes
+		{code: "LoadBalancerNotFound", http: 400, message: "One or more load balancers not found."},
+		{code: "TargetGroupNotFound", http: 400, message: "One or more target groups not found."},
+		{code: "ListenerNotFound", http: 400, message: "One or more listeners not found."},
+		{code: "DuplicateLoadBalancerName", http: 400, message: "A load balancer with the same name already exists."},
+		{code: "DuplicateTargetGroupName", http: 400, message: "A target group with the same name already exists."},
+		{code: "DuplicateListener", http: 400, message: "A listener with the specified port already exists on this load balancer."},
+		{code: "TooManyLoadBalancers", http: 400, message: "You've reached the limit on the number of load balancers for your account."},
+		{code: "TooManyTargetGroups", http: 400, message: "You've reached the limit on the number of target groups for your account."},
+		{code: "TooManyListeners", http: 400, message: "You've reached the limit on the number of listeners per load balancer."},
+		{code: "TooManyRegistrationsForTargetId", http: 400, message: "You've reached the limit on the number of times a target can be registered with a target group."},
+		{code: "InvalidTarget", http: 400, message: "The specified target does not exist, is not in the same VPC as the target group, or has an unsupported instance type."},
+		{code: "ResourceInUse", http: 400, message: "The target group is currently in use by a listener or rule."},
+		{code: "InvalidSecurityGroup", http: 400, message: "The specified security group does not exist."},
+		{code: "InvalidScheme", http: 400, message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
+		{code: "SubnetNotFound", http: 400, message: "The specified subnet does not exist."},
+		{code: "AvailabilityZoneNotSupported", http: 400, message: "The specified Availability Zone is not supported."},
 	}
 
 	if len(ErrorLookup) != len(expected) {

spinifex/daemon/daemon.go

@@ -38,11 +38,14 @@ import (
 	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+	handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw"
 	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
+	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
@@ -111,6 +114,9 @@ type Daemon struct {
 	placementGroupService *handlers_ec2_placementgroup.PlacementGroupServiceImpl
 	vpcService            *handlers_ec2_vpc.VPCServiceImpl
 	eipService            *handlers_ec2_eip.EIPServiceImpl
+	elbv2Service          *handlers_elbv2.ELBv2ServiceImpl
+	routeTableService     *handlers_ec2_routetable.RouteTableServiceImpl
+	natGatewayService     *handlers_ec2_natgw.NatGatewayServiceImpl
 	externalIPAM          *handlers_ec2_vpc.ExternalIPAM
 	ctx                   context.Context
 	cancel                context.CancelFunc
@@ -402,6 +408,18 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.RemoveInstanceFromPlacementGroup", d.handleEC2RemoveInstanceFromPlacementGroup, "spinifex-workers"},
 		{"ec2.ReserveClusterNode", d.handleEC2ReserveClusterNode, "spinifex-workers"},
 		{"ec2.FinalizeClusterInstances", d.handleEC2FinalizeClusterInstances, "spinifex-workers"},
+		{"ec2.CreateNatGateway", d.handleEC2CreateNatGateway, "spinifex-workers"},
+		{"ec2.DeleteNatGateway", d.handleEC2DeleteNatGateway, "spinifex-workers"},
+		{"ec2.DescribeNatGateways", d.handleEC2DescribeNatGateways, "spinifex-workers"},
+		{"ec2.CreateRouteTable", d.handleEC2CreateRouteTable, "spinifex-workers"},
+		{"ec2.DeleteRouteTable", d.handleEC2DeleteRouteTable, "spinifex-workers"},
+		{"ec2.DescribeRouteTables", d.handleEC2DescribeRouteTables, "spinifex-workers"},
+		{"ec2.CreateRoute", d.handleEC2CreateRoute, "spinifex-workers"},
+		{"ec2.DeleteRoute", d.handleEC2DeleteRoute, "spinifex-workers"},
+		{"ec2.ReplaceRoute", d.handleEC2ReplaceRoute, "spinifex-workers"},
+		{"ec2.AssociateRouteTable", d.handleEC2AssociateRouteTable, "spinifex-workers"},
+		{"ec2.DisassociateRouteTable", d.handleEC2DisassociateRouteTable, "spinifex-workers"},
+		{"ec2.ReplaceRouteTableAssociation", d.handleEC2ReplaceRouteTableAssociation, "spinifex-workers"},
 		{"ec2.CreateVpc", d.handleEC2CreateVpc, "spinifex-workers"},
 		{"ec2.DeleteVpc", d.handleEC2DeleteVpc, "spinifex-workers"},
 		{"ec2.DescribeVpcs", d.handleEC2DescribeVpcs, "spinifex-workers"},
@@ -409,9 +427,12 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.DeleteSubnet", d.handleEC2DeleteSubnet, "spinifex-workers"},
 		{"ec2.DescribeSubnets", d.handleEC2DescribeSubnets, "spinifex-workers"},
 		{"ec2.ModifySubnetAttribute", d.handleEC2ModifySubnetAttribute, "spinifex-workers"},
+		{"ec2.ModifyVpcAttribute", d.handleEC2ModifyVpcAttribute, "spinifex-workers"},
+		{"ec2.DescribeVpcAttribute", d.handleEC2DescribeVpcAttribute, "spinifex-workers"},
 		{"ec2.CreateNetworkInterface", d.handleEC2CreateNetworkInterface, "spinifex-workers"},
 		{"ec2.DeleteNetworkInterface", d.handleEC2DeleteNetworkInterface, "spinifex-workers"},
 		{"ec2.DescribeNetworkInterfaces", d.handleEC2DescribeNetworkInterfaces, "spinifex-workers"},
+		{"ec2.ModifyNetworkInterfaceAttribute", d.handleEC2ModifyNetworkInterfaceAttribute, "spinifex-workers"},
 		{"ec2.CreateSecurityGroup", d.handleEC2CreateSecurityGroup, "spinifex-workers"},
 		{"ec2.DeleteSecurityGroup", d.handleEC2DeleteSecurityGroup, "spinifex-workers"},
 		{"ec2.DescribeSecurityGroups", d.handleEC2DescribeSecurityGroups, "spinifex-workers"},
@@ -419,12 +440,8 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.AuthorizeSecurityGroupEgress", d.handleEC2AuthorizeSecurityGroupEgress, "spinifex-workers"},
 		{"ec2.RevokeSecurityGroupIngress", d.handleEC2RevokeSecurityGroupIngress, "spinifex-workers"},
 		{"ec2.RevokeSecurityGroupEgress", d.handleEC2RevokeSecurityGroupEgress, "spinifex-workers"},
-		{"ec2.AllocateAddress", d.handleEC2AllocateAddress, "spinifex-workers"},
-		{"ec2.ReleaseAddress", d.handleEC2ReleaseAddress, "spinifex-workers"},
-		{"ec2.AssociateAddress", d.handleEC2AssociateAddress, "spinifex-workers"},
-		{"ec2.DisassociateAddress", d.handleEC2DisassociateAddress, "spinifex-workers"},
-		{"ec2.DescribeAddresses", d.handleEC2DescribeAddresses, "spinifex-workers"},
 		{"ec2.ModifyInstanceAttribute", d.handleEC2ModifyInstanceAttribute, "spinifex-workers"},
+		{"ec2.DescribeInstanceAttribute", d.handleEC2DescribeInstanceAttribute, "spinifex-workers"},
 		{"ec2.start", d.handleEC2StartStoppedInstance, "spinifex-workers"},
 		{"ec2.terminate", d.handleEC2TerminateStoppedInstance, "spinifex-workers"},
 		{"ec2.DescribeStoppedInstances", d.handleEC2DescribeStoppedInstances, "spinifex-workers"},
@@ -438,10 +455,24 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.GetSerialConsoleAccessStatus", d.handleEC2GetSerialConsoleAccessStatus, "spinifex-workers"},
 		{"ec2.EnableSerialConsoleAccess", d.handleEC2EnableSerialConsoleAccess, "spinifex-workers"},
 		{"ec2.DisableSerialConsoleAccess", d.handleEC2DisableSerialConsoleAccess, "spinifex-workers"},
+		// ELBv2 operations
+		{"elbv2.CreateLoadBalancer", d.handleELBv2CreateLoadBalancer, "spinifex-workers"},
+		{"elbv2.DeleteLoadBalancer", d.handleELBv2DeleteLoadBalancer, "spinifex-workers"},
+		{"elbv2.DescribeLoadBalancers", d.handleELBv2DescribeLoadBalancers, "spinifex-workers"},
+		{"elbv2.CreateTargetGroup", d.handleELBv2CreateTargetGroup, "spinifex-workers"},
+		{"elbv2.DeleteTargetGroup", d.handleELBv2DeleteTargetGroup, "spinifex-workers"},
+		{"elbv2.DescribeTargetGroups", d.handleELBv2DescribeTargetGroups, "spinifex-workers"},
+		{"elbv2.RegisterTargets", d.handleELBv2RegisterTargets, "spinifex-workers"},
+		{"elbv2.DeregisterTargets", d.handleELBv2DeregisterTargets, "spinifex-workers"},
+		{"elbv2.DescribeTargetHealth", d.handleELBv2DescribeTargetHealth, "spinifex-workers"},
+		{"elbv2.CreateListener", d.handleELBv2CreateListener, "spinifex-workers"},
+		{"elbv2.DeleteListener", d.handleELBv2DeleteListener, "spinifex-workers"},
+		{"elbv2.DescribeListeners", d.handleELBv2DescribeListeners, "spinifex-workers"},
 		{fmt.Sprintf("spinifex.admin.%s.health", d.node), d.handleHealthCheck, ""},
 		{"spinifex.nodes.discover", d.handleNodeDiscover, ""},
 		{"spinifex.node.status", d.handleNodeStatus, ""},
 		{"spinifex.node.vms", d.handleNodeVMs, ""},
+		{"spinifex.storage.config", d.handleStorageConfig, ""},
 		// Account creation → create default VPC for new account
 		{"iam.account.created", d.handleAccountCreated, "spinifex-workers"},
 		// Coordinated cluster shutdown phases (fan-out, no queue group)
@@ -452,6 +483,19 @@ func (d *Daemon) subscribeAll() error {
 		{"spinifex.cluster.shutdown.infra", d.handleShutdownInfra, ""},
 	}
 
+	// EIP operations require external IPAM (pool mode). Only subscribe when available;
+	// without a subscriber the gateway returns a NATS timeout → clean error to the client.
+	if d.eipService != nil {
+		subs = append(subs,
+			natsSub{"ec2.AllocateAddress", d.handleEC2AllocateAddress, "spinifex-workers"},
+			natsSub{"ec2.ReleaseAddress", d.handleEC2ReleaseAddress, "spinifex-workers"},
+			natsSub{"ec2.AssociateAddress", d.handleEC2AssociateAddress, "spinifex-workers"},
+			natsSub{"ec2.DisassociateAddress", d.handleEC2DisassociateAddress, "spinifex-workers"},
+			natsSub{"ec2.DescribeAddresses", d.handleEC2DescribeAddresses, "spinifex-workers"},
+			natsSub{"ec2.DescribeAddressesAttribute", d.handleEC2DescribeAddressesAttribute, "spinifex-workers"},
+		)
+	}
+
 	for _, s := range subs {
 		var sub *nats.Subscription
 		var err error
@@ -547,6 +591,20 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize VPC service: %w", err)
 	}
 
+	d.routeTableService, err = initServiceWithRetry("RouteTable service", func() (*handlers_ec2_routetable.RouteTableServiceImpl, error) {
+		return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize RouteTable service: %w", err)
+	}
+
+	d.natGatewayService, err = initServiceWithRetry("NatGateway service", func() (*handlers_ec2_natgw.NatGatewayServiceImpl, error) {
+		return handlers_ec2_natgw.NewNatGatewayServiceImplWithNATS(d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize NatGateway service: %w", err)
+	}
+
 	// Initialize external IPAM if pool mode is configured (per-VM public IPs).
 	// NAT mode only uses SNAT via a shared gateway IP — no per-VM allocation needed.
 	if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode == "pool" {
@@ -601,6 +659,45 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize account settings service: %w", err)
 	}
 
+	d.elbv2Service, err = initServiceWithRetry("ELBv2 service", func() (*handlers_elbv2.ELBv2ServiceImpl, error) {
+		return handlers_elbv2.NewELBv2ServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize ELBv2 service: %w", err)
+	}
+	if d.vpcService != nil {
+		d.elbv2Service.SetVPCService(d.vpcService)
+	}
+
+	// Wire ALB VM lifecycle: instance launcher + NATS URL for cloud-init.
+	// In dev mode (user-mode networking), VMs reach the host at 10.0.2.2
+	// (QEMU's default gateway). Replace 0.0.0.0 with this address.
+	d.elbv2Service.SetInstanceLauncher(d)
+	if d.config != nil && d.config.NATS.Host != "" {
+		natsHost := d.config.NATS.Host
+		if d.config.Daemon.DevNetworking && strings.HasPrefix(natsHost, "0.0.0.0") {
+			natsHost = strings.Replace(natsHost, "0.0.0.0", "10.0.2.2", 1)
+		}
+		d.elbv2Service.SetNATSURL("nats://" + natsHost)
+	}
+	// Discover system AMI for ALB VMs. Prefer an image with "alb" in the
+	// name (pre-baked Alpine with HAProxy), fall back to first available.
+	if d.imageService != nil {
+		imagesOut, imgErr := d.imageService.DescribeImages(&ec2.DescribeImagesInput{}, utils.GlobalAccountID)
+		if imgErr == nil && len(imagesOut.Images) > 0 {
+			systemAMI := aws.StringValue(imagesOut.Images[0].ImageId)
+			for _, img := range imagesOut.Images {
+				name := aws.StringValue(img.Name)
+				if strings.Contains(name, "alb") {
+					systemAMI = aws.StringValue(img.ImageId)
+					break
+				}
+			}
+			d.elbv2Service.SetSystemAMI(systemAMI)
+			slog.Info("System AMI set for ALB VMs", "amiId", systemAMI)
+		}
+	}
+
 	// Ensure default VPC exists for system and admin accounts
 	// (matches AWS: every account has a default VPC with IGW + default SG)
 	if d.vpcService != nil {
@@ -1669,10 +1766,16 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 				// Publish vpc.delete-nat to remove dnat_and_snat rule
 				portName := "port-" + instance.ENIId
 				vpcId := ""
-				if instance.Instance != nil && instance.Instance.VpcId != nil {
-					vpcId = *instance.Instance.VpcId
+				logicalIP := ""
+				if instance.Instance != nil {
+					if instance.Instance.VpcId != nil {
+						vpcId = *instance.Instance.VpcId
+					}
+					if instance.Instance.PrivateIpAddress != nil {
+						logicalIP = *instance.Instance.PrivateIpAddress
+					}
 				}
-				d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
+				d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, logicalIP, portName, "")
 
 				// Release IP back to pool
 				if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
@@ -1756,6 +1859,11 @@ func (d *Daemon) setupShutdown() {
 			}
 		}
 
+		// Stop ELBv2 background goroutines
+		if d.elbv2Service != nil {
+			d.elbv2Service.Close()
+		}
+
 		// Final cleanup
 		for _, sub := range d.natsSubscriptions {
 			// Unsubscribe from each subscription
@@ -2107,74 +2215,18 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 	consoleLogPath := filepath.Join(runtimeDir, fmt.Sprintf("console-%s.log", instance.ID))
 	serialSocket := filepath.Join(runtimeDir, fmt.Sprintf("serial-%s.sock", instance.ID))
 
-	instance.Config = vm.Config{
-		Name:           instance.ID,
-		PIDFile:        pidFile,
-		EnableKVM:      true, // If available, if kvm fails, will use cpu max
-		NoGraphic:      true,
-		MachineType:    "q35",
-		ConsoleLogPath: consoleLogPath,
-		SerialSocket:   serialSocket,
-		CPUType:        "host", // If available, if kvm fails, will use cpu max
-		Memory:         int(memoryMiB),
-		CPUCount:       vCPUs,
-		Architecture:   architecture,
-	}
+	instance.Config = buildBaseVMConfig(instance.ID, pidFile, consoleLogPath, serialSocket, architecture, vCPUs, int(memoryMiB))
 
-	// Add PCIe root ports for volume hotplug (Q35 requires explicit root ports).
-	// 11 ports for /dev/sd[f-p] hotplug slots, starting at chassis 1.
-	for i := 1; i <= 11; i++ {
-		instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-			Value: fmt.Sprintf("pcie-root-port,id=hotplug%d,chassis=%d,slot=0", i, i),
-		})
-	}
-
-	// Loop through each volume in volumes
+	// Build QEMU drives from EBS volume requests.
 	instance.EBSRequests.Mu.Lock()
-
-	for _, v := range instance.EBSRequests.Requests {
-		drive := vm.Drive{}
-
-		// Use the NBDURI from mount response - contains socket path or TCP address
-		// NBDURI format: "nbd:unix:/path/to/socket.sock" or "nbd://host:port"
-		if v.NBDURI == "" {
-			slog.Error("NBDURI not set for volume", "volume", v.Name)
-			return fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", v.Name)
-		}
-		drive.File = v.NBDURI
-		slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
-
-		if v.Boot {
-			drive.Format = "raw"
-			drive.If = "none"
-			drive.Media = "disk"
-			drive.ID = "os"
-			drive.Cache = "none"
-
-			iothreadID := "ioth-os"
-			instance.Config.IOThreads = append(instance.Config.IOThreads, vm.IOThread{ID: iothreadID})
-
-			instance.Config.Devices = append(instance.Config.Devices, vm.Device{
-				Value: fmt.Sprintf("virtio-blk-pci,drive=%s,iothread=%s,num-queues=%d,bootindex=1",
-					drive.ID, iothreadID, instance.Config.CPUCount),
-			})
-		}
-
-		if v.CloudInit {
-			drive.Format = "raw"
-			drive.If = "virtio"
-			drive.Media = "cdrom"
-			drive.ID = "cloudinit"
-		}
-
-		// TODO: Add EFI support
-		if v.EFI {
-			continue
-		}
-
-		instance.Config.Drives = append(instance.Config.Drives, drive)
-	}
+	drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, vCPUs)
 	instance.EBSRequests.Mu.Unlock()
+	if err != nil {
+		return err
+	}
+	instance.Config.Drives = append(instance.Config.Drives, drives...)
+	instance.Config.IOThreads = append(instance.Config.IOThreads, iothreads...)
+	instance.Config.Devices = append(instance.Config.Devices, devices...)
 
 	// VPC tap networking vs user-mode fallback
 	if instance.ENIId != "" && d.networkPlumber != nil {
@@ -2208,8 +2260,32 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 					if bindIP == "" || bindIP == "0.0.0.0" {
 						bindIP = "127.0.0.1"
 					}
+					netdevVal := fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort)
+					// Add extra hostfwd rules for system instances (e.g. ALB VMs forwarding HTTP ports)
+					if instance.ExtraHostfwd != nil {
+						for guestPort := range instance.ExtraHostfwd {
+							fwdAddr, fwdErr := viperblock.FindFreePort()
+							if fwdErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to find free port for extra hostfwd", "guestPort", guestPort, "err", fwdErr)
+								continue
+							}
+							_, hostPort, splitErr := net.SplitHostPort(fwdAddr)
+							if splitErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to parse extra hostfwd address", "fwdAddr", fwdAddr, "err", splitErr)
+								continue
+							}
+							hostPortInt, convErr := strconv.Atoi(hostPort)
+							if convErr != nil {
+								slog.Warn("DEV_NETWORKING: failed to convert extra hostfwd port", "hostPort", hostPort, "err", convErr)
+								continue
+							}
+							netdevVal += fmt.Sprintf(",hostfwd=tcp:%s:%s-:%d", bindIP, hostPort, guestPort)
+							instance.ExtraHostfwd[guestPort] = hostPortInt
+							slog.Info("DEV_NETWORKING: extra hostfwd", "guestPort", guestPort, "hostPort", hostPort, "instanceId", instance.ID)
+						}
+					}
 					instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-						Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
+						Value: netdevVal,
 					})
 					devMac := generateDevMAC(instance.ID)
 					instance.Config.Devices = append(instance.Config.Devices, vm.Device{
@@ -2391,6 +2467,83 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 	return nil
 }
 
+// buildBaseVMConfig creates a vm.Config with base QEMU settings and PCIe
+// hotplug root ports. Architecture, vCPU, and memory come from the caller
+// (resolved from instance type info).
+func buildBaseVMConfig(instanceID, pidFile, consoleLogPath, serialSocket, architecture string, vCPUs, memoryMiB int) vm.Config {
+	cfg := vm.Config{
+		Name:           instanceID,
+		PIDFile:        pidFile,
+		EnableKVM:      true,
+		NoGraphic:      true,
+		MachineType:    "q35",
+		ConsoleLogPath: consoleLogPath,
+		SerialSocket:   serialSocket,
+		CPUType:        "host",
+		Memory:         memoryMiB,
+		CPUCount:       vCPUs,
+		Architecture:   architecture,
+	}
+
+	// Add PCIe root ports for volume hotplug (Q35 requires explicit root ports).
+	// 11 ports for /dev/sd[f-p] hotplug slots, starting at chassis 1.
+	for i := 1; i <= 11; i++ {
+		cfg.Devices = append(cfg.Devices, vm.Device{
+			Value: fmt.Sprintf("pcie-root-port,id=hotplug%d,chassis=%d,slot=0", i, i),
+		})
+	}
+
+	return cfg
+}
+
+// buildDrives converts EBS volume requests into QEMU drive, iothread, and device
+// configurations. Returns an error if any non-EFI volume is missing its NBDURI.
+func buildDrives(requests []types.EBSRequest, cpuCount int) ([]vm.Drive, []vm.IOThread, []vm.Device, error) {
+	var drives []vm.Drive
+	var iothreads []vm.IOThread
+	var devices []vm.Device
+
+	for _, v := range requests {
+		// TODO: Add EFI support
+		if v.EFI {
+			continue
+		}
+
+		if v.NBDURI == "" {
+			return nil, nil, nil, fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", v.Name)
+		}
+
+		drive := vm.Drive{File: v.NBDURI}
+
+		if v.Boot {
+			drive.Format = "raw"
+			drive.If = "none"
+			drive.Media = "disk"
+			drive.ID = "os"
+			drive.Cache = "none"
+
+			iothreadID := "ioth-os"
+			iothreads = append(iothreads, vm.IOThread{ID: iothreadID})
+			devices = append(devices, vm.Device{
+				Value: fmt.Sprintf("virtio-blk-pci,drive=%s,iothread=%s,num-queues=%d,bootindex=1",
+					drive.ID, iothreadID, cpuCount),
+			})
+		}
+
+		if v.CloudInit {
+			drive.Format = "raw"
+			drive.If = "virtio"
+			drive.Media = "cdrom"
+			drive.ID = "cloudinit"
+		}
+
+		slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
+		drives = append(drives, drive)
+	}
+
+	return drives, iothreads, devices, nil
+}
+
 // ebsTopic returns a node-specific EBS NATS topic, e.g. "ebs.node1.mount".
 // This ensures mount/unmount requests are routed to the viperblock instance
 // running on the same node as the daemon (NBD sockets are local).

spinifex/daemon/daemon_handlers_eip.go

@@ -21,3 +21,7 @@ func (d *Daemon) handleEC2DisassociateAddress(msg *nats.Msg) {
 func (d *Daemon) handleEC2DescribeAddresses(msg *nats.Msg) {
 	handleNATSRequest(msg, d.eipService.DescribeAddresses)
 }
+
+func (d *Daemon) handleEC2DescribeAddressesAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eipService.DescribeAddressesAttribute)
+}

spinifex/daemon/daemon_handlers_elbv2.go

@@ -0,0 +1,51 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleELBv2CreateLoadBalancer(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateLoadBalancer)
+}
+
+func (d *Daemon) handleELBv2DeleteLoadBalancer(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteLoadBalancer)
+}
+
+func (d *Daemon) handleELBv2DescribeLoadBalancers(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeLoadBalancers)
+}
+
+func (d *Daemon) handleELBv2CreateTargetGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateTargetGroup)
+}
+
+func (d *Daemon) handleELBv2DeleteTargetGroup(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteTargetGroup)
+}
+
+func (d *Daemon) handleELBv2DescribeTargetGroups(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTargetGroups)
+}
+
+func (d *Daemon) handleELBv2RegisterTargets(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.RegisterTargets)
+}
+
+func (d *Daemon) handleELBv2DeregisterTargets(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeregisterTargets)
+}
+
+func (d *Daemon) handleELBv2DescribeTargetHealth(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTargetHealth)
+}
+
+func (d *Daemon) handleELBv2CreateListener(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.CreateListener)
+}
+
+func (d *Daemon) handleELBv2DeleteListener(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DeleteListener)
+}
+
+func (d *Daemon) handleELBv2DescribeListeners(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeListeners)
+}

spinifex/daemon/daemon_handlers_instance.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/filterutil"
 	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
 	"github.com/mulgadc/spinifex/spinifex/types"
@@ -157,6 +158,20 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 			continue
 		}
 
+		// When Terraform sets associate_public_ip_address, it sends the subnet
+		// and security groups inside NetworkInterfaces[0] instead of the top-level
+		// fields. Extract them so the rest of the handler works uniformly.
+		if (runInstancesInput.SubnetId == nil || *runInstancesInput.SubnetId == "") &&
+			len(runInstancesInput.NetworkInterfaces) > 0 && runInstancesInput.NetworkInterfaces[0] != nil {
+			nic := runInstancesInput.NetworkInterfaces[0]
+			if nic.SubnetId != nil && *nic.SubnetId != "" {
+				runInstancesInput.SubnetId = nic.SubnetId
+			}
+			if len(runInstancesInput.SecurityGroupIds) == 0 && len(nic.Groups) > 0 {
+				runInstancesInput.SecurityGroupIds = nic.Groups
+			}
+		}
+
 		// Resolve default subnet when none specified (matches AWS behavior)
 		if (runInstancesInput.SubnetId == nil || *runInstancesInput.SubnetId == "") && d.vpcService != nil {
 			defaultSubnet, dsErr := d.vpcService.GetDefaultSubnet(accountID)
@@ -182,6 +197,12 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 			eni := eniOut.NetworkInterface
 			instance.ENIId = *eni.NetworkInterfaceId
 			instance.ENIMac = *eni.MacAddress
+
+			// Mark ENI as attached to this instance so attachment.instance-id
+			// filter works (used by ELBv2 RegisterTargets to resolve target IPs).
+			if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+				slog.Error("Failed to attach ENI to instance record — ELBv2 target IP resolution will fail", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
+			}
 			ec2Instance.SetPrivateIpAddress(*eni.PrivateIpAddress)
 			ec2Instance.SetSubnetId(*runInstancesInput.SubnetId)
 			ec2Instance.SetVpcId(*eni.VpcId)
@@ -612,6 +633,89 @@ func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command types.EC2I
 	}(instance, command.Attributes)
 }
 
+// describeInstancesValidFilters defines the set of filter names accepted by DescribeInstances.
+var describeInstancesValidFilters = map[string]bool{
+	"instance-state-name": true,
+	"instance-id":         true,
+	"instance-type":       true,
+	"vpc-id":              true,
+	"subnet-id":           true,
+	"tag-key":             true,
+	"tag-value":           true,
+}
+
+// instanceMatchesFilters checks whether a VM + its built ec2.Instance copy satisfy all parsed filters.
+func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]string) bool {
+	for name, values := range filters {
+		if strings.HasPrefix(name, "tag:") {
+			// tag:Key filters are handled after all field filters.
+			continue
+		}
+
+		var field string
+		switch name {
+		case "instance-state-name":
+			if ic.State != nil && ic.State.Name != nil {
+				field = *ic.State.Name
+			}
+		case "instance-id":
+			field = inst.ID
+		case "instance-type":
+			field = inst.InstanceType
+		case "vpc-id":
+			if ic.VpcId != nil {
+				field = *ic.VpcId
+			}
+		case "subnet-id":
+			if ic.SubnetId != nil {
+				field = *ic.SubnetId
+			}
+		case "tag-key":
+			if !matchTagKey(ic.Tags, values) {
+				return false
+			}
+			continue
+		case "tag-value":
+			if !matchTagValue(ic.Tags, values) {
+				return false
+			}
+			continue
+		default:
+			// Filter name passed ParseFilters but has no case — treat as non-match
+			// to avoid silently ignoring it.
+			return false
+		}
+
+		if !filterutil.MatchesAny(values, field) {
+			return false
+		}
+	}
+
+	// Check tag:Key filters via the instance's Tag slice.
+	tags := filterutil.EC2TagsToMap(ic.Tags)
+	return filterutil.MatchesTags(filters, tags)
+}
+
+// matchTagKey returns true if any tag key on the resource matches any of the filter values.
+func matchTagKey(tags []*ec2.Tag, values []string) bool {
+	for _, t := range tags {
+		if t.Key != nil && filterutil.MatchesAny(values, *t.Key) {
+			return true
+		}
+	}
+	return false
+}
+
+// matchTagValue returns true if any tag value on the resource matches any of the filter values.
+func matchTagValue(tags []*ec2.Tag, values []string) bool {
+	for _, t := range tags {
+		if t.Value != nil && filterutil.MatchesAny(values, *t.Value) {
+			return true
+		}
+	}
+	return false
+}
+
 // handleEC2DescribeInstances processes incoming EC2 DescribeInstances requests
 // This handler responds with instances running on this node owned by the caller's account
 func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
@@ -651,6 +755,14 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 		}
 	}
 
+	// Parse filters (returns error for unknown filter names)
+	parsedFilters, err := filterutil.ParseFilters(describeInstancesInput.Filters, describeInstancesValidFilters)
+	if err != nil {
+		slog.Warn("DescribeInstances: invalid filter", "err", err)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
 	// Group instances by reservation ID (AWS returns instances grouped by reservation)
 	reservationMap := make(map[string]*ec2.Reservation)
 
@@ -713,6 +825,11 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 				}
 			}
 
+			// Apply filters against the fully-built instance copy
+			if len(parsedFilters) > 0 && !instanceMatchesFilters(instance, &instanceCopy, parsedFilters) {
+				continue
+			}
+
 			// Add instance to its reservation
 			reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
 		}
@@ -977,10 +1094,16 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
 	if instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
 		portName := "port-" + instance.ENIId
 		vpcId := ""
-		if instance.Instance != nil && instance.Instance.VpcId != nil {
-			vpcId = *instance.Instance.VpcId
+		logicalIP := ""
+		if instance.Instance != nil {
+			if instance.Instance.VpcId != nil {
+				vpcId = *instance.Instance.VpcId
+			}
+			if instance.Instance.PrivateIpAddress != nil {
+				logicalIP = *instance.Instance.PrivateIpAddress
+			}
 		}
-		d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
+		d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, logicalIP, portName, "")
 
 		if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
 			slog.Warn("handleEC2TerminateStoppedInstance: failed to release public IP", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
@@ -1065,6 +1188,13 @@ func (d *Daemon) describeInstancesFromKV(msg *nats.Msg, listFn func() ([]*vm.VM,
 		}
 	}
 
+	parsedFilters, filterErr := filterutil.ParseFilters(describeInput.Filters, describeInstancesValidFilters)
+	if filterErr != nil {
+		slog.Warn(handlerName+": invalid filter", "err", filterErr)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
 	instances, err := listFn()
 	if err != nil {
 		slog.Error(handlerName+": failed to list instances", "err", err)
@@ -1112,6 +1242,10 @@ func (d *Daemon) describeInstancesFromKV(msg *nats.Msg, listFn func() ([]*vm.VM,
 			instanceCopy.State.SetName(fallbackName)
 		}
 
+		if len(parsedFilters) > 0 && !instanceMatchesFilters(instance, &instanceCopy, parsedFilters) {
+			continue
+		}
+
 		reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy)
 	}
 
@@ -1142,6 +1276,18 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
 
 	instanceID := *input.InstanceId
 
+	// SourceDestCheck is a networking concept that doesn't apply to bare-metal VMs.
+	// Accept the call as a no-op so Terraform and the AWS CLI don't error out.
+	// Unlike InstanceType/UserData, AWS allows this on running instances, so handle
+	// it before the stopped-state gate.
+	if input.SourceDestCheck != nil {
+		slog.Info("handleEC2ModifyInstanceAttribute: accepting SourceDestCheck (no-op on bare metal)", "instanceId", instanceID)
+		if err := msg.Respond([]byte(`{}`)); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
+		return
+	}
+
 	if d.jsManager == nil {
 		slog.Error("handleEC2ModifyInstanceAttribute: JetStream not available")
 		respondWithError(msg, awserrors.ErrorServerInternal)
@@ -1218,6 +1364,123 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
 	}
 }
 
+// handleEC2DescribeInstanceAttribute returns a single requested attribute for an instance.
+// It checks running instances first (in-memory), then falls back to stopped instances in KV.
+func (d *Daemon) handleEC2DescribeInstanceAttribute(msg *nats.Msg) {
+	var input ec2.DescribeInstanceAttributeInput
+	if err := json.Unmarshal(msg.Data, &input); err != nil {
+		slog.Error("handleEC2DescribeInstanceAttribute: failed to unmarshal request", "err", err)
+		respondWithError(msg, awserrors.ErrorServerInternal)
+		return
+	}
+
+	if input.InstanceId == nil || *input.InstanceId == "" {
+		slog.Error("handleEC2DescribeInstanceAttribute: missing instance_id")
+		respondWithError(msg, awserrors.ErrorMissingParameter)
+		return
+	}
+	if input.Attribute == nil || *input.Attribute == "" {
+		slog.Error("handleEC2DescribeInstanceAttribute: missing attribute")
+		respondWithError(msg, awserrors.ErrorMissingParameter)
+		return
+	}
+
+	instanceID := *input.InstanceId
+	attribute := *input.Attribute
+	accountID := utils.AccountIDFromMsg(msg)
+
+	// Look up instance: running first, then stopped KV.
+	var instance *vm.VM
+
+	d.Instances.Mu.Lock()
+	if running, ok := d.Instances.VMS[instanceID]; ok {
+		instance = running
+	}
+	d.Instances.Mu.Unlock()
+
+	if instance == nil {
+		if d.jsManager == nil {
+			slog.Error("handleEC2DescribeInstanceAttribute: JetStream not available")
+			respondWithError(msg, awserrors.ErrorServerInternal)
+			return
+		}
+		stopped, err := d.jsManager.LoadStoppedInstance(instanceID)
+		if err != nil {
+			slog.Error("handleEC2DescribeInstanceAttribute: failed to load stopped instance",
+				"instanceId", instanceID, "err", err)
+			respondWithError(msg, awserrors.ErrorServerInternal)
+			return
+		}
+		instance = stopped
+	}
+
+	if instance == nil {
+		slog.Warn("handleEC2DescribeInstanceAttribute: instance not found",
+			"instanceId", instanceID)
+		respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
+		return
+	}
+
+	if !checkInstanceOwnership(msg, instanceID, instance.AccountID) {
+		return
+	}
+
+	output := &ec2.DescribeInstanceAttributeOutput{
+		InstanceId: &instanceID,
+	}
+
+	switch attribute {
+	case ec2.InstanceAttributeNameInstanceType:
+		val := instance.InstanceType
+		output.InstanceType = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameUserData:
+		val := instance.UserData
+		output.UserData = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameDisableApiTermination:
+		val := false
+		output.DisableApiTermination = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameDisableApiStop:
+		val := false
+		output.DisableApiStop = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameInstanceInitiatedShutdownBehavior:
+		val := ec2.ShutdownBehaviorStop
+		output.InstanceInitiatedShutdownBehavior = &ec2.AttributeValue{Value: &val}
+
+	case ec2.InstanceAttributeNameEbsOptimized:
+		val := false
+		output.EbsOptimized = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameEnaSupport:
+		val := true
+		output.EnaSupport = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameSourceDestCheck:
+		val := true
+		output.SourceDestCheck = &ec2.AttributeBooleanValue{Value: &val}
+
+	case ec2.InstanceAttributeNameGroupSet:
+		if instance.Instance != nil && len(instance.Instance.SecurityGroups) > 0 {
+			output.Groups = instance.Instance.SecurityGroups
+		} else {
+			output.Groups = []*ec2.GroupIdentifier{}
+		}
+
+	default:
+		slog.Warn("handleEC2DescribeInstanceAttribute: unsupported attribute",
+			"instanceId", instanceID, "attribute", attribute)
+		respondWithError(msg, awserrors.ErrorInvalidParameterValue)
+		return
+	}
+
+	slog.Info("handleEC2DescribeInstanceAttribute: completed",
+		"instanceId", instanceID, "attribute", attribute, "accountID", accountID)
+	respondWithJSON(msg, output)
+}
+
 // publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
 func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) {
 	if d.natsConn == nil {

spinifex/daemon/daemon_handlers_natgw.go

@@ -0,0 +1,17 @@
+package daemon
+
+import (
+	"github.com/nats-io/nats.go"
+)
+
+func (d *Daemon) handleEC2CreateNatGateway(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.CreateNatGateway)
+}
+
+func (d *Daemon) handleEC2DeleteNatGateway(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.DeleteNatGateway)
+}
+
+func (d *Daemon) handleEC2DescribeNatGateways(msg *nats.Msg) {
+	handleNATSRequest(msg, d.natGatewayService.DescribeNatGateways)
+}

spinifex/daemon/daemon_handlers_routetable.go

@@ -0,0 +1,41 @@
+package daemon
+
+import (
+	"github.com/nats-io/nats.go"
+)
+
+func (d *Daemon) handleEC2CreateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.CreateRouteTable)
+}
+
+func (d *Daemon) handleEC2DeleteRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DeleteRouteTable)
+}
+
+func (d *Daemon) handleEC2DescribeRouteTables(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DescribeRouteTables)
+}
+
+func (d *Daemon) handleEC2CreateRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.CreateRoute)
+}
+
+func (d *Daemon) handleEC2DeleteRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DeleteRoute)
+}
+
+func (d *Daemon) handleEC2ReplaceRoute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.ReplaceRoute)
+}
+
+func (d *Daemon) handleEC2AssociateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.AssociateRouteTable)
+}
+
+func (d *Daemon) handleEC2DisassociateRouteTable(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.DisassociateRouteTable)
+}
+
+func (d *Daemon) handleEC2ReplaceRouteTableAssociation(msg *nats.Msg) {
+	handleNATSRequest(msg, d.routeTableService.ReplaceRouteTableAssociation)
+}

spinifex/daemon/daemon_handlers_storage.go

@@ -0,0 +1,106 @@
+package daemon
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/nats-io/nats.go"
+	toml "github.com/pelletier/go-toml/v2"
+)
+
+// predastoreTOML is a minimal representation of the predastore config file,
+// containing only the fields needed for storage metrics (no credentials).
+type predastoreTOML struct {
+	RS      predastoreRS       `toml:"rs"`
+	DB      []predastoreDBNode `toml:"db"`
+	Nodes   []predastoreNode   `toml:"nodes"`
+	Buckets []predastoreBucket `toml:"buckets"`
+}
+
+type predastoreRS struct {
+	Data   int `toml:"data"`
+	Parity int `toml:"parity"`
+}
+
+type predastoreDBNode struct {
+	ID   int    `toml:"id"`
+	Host string `toml:"host"`
+	Port int    `toml:"port"`
+}
+
+type predastoreNode struct {
+	ID   int    `toml:"id"`
+	Host string `toml:"host"`
+	Port int    `toml:"port"`
+}
+
+type predastoreBucket struct {
+	Name   string `toml:"name"`
+	Type   string `toml:"type"`
+	Region string `toml:"region"`
+}
+
+// handleStorageConfig responds with parsed predastore config (topology only, no creds).
+// Used by the gateway to build the GetStorageStatus response.
+func (d *Daemon) handleStorageConfig(msg *nats.Msg) {
+	configDir := filepath.Dir(d.configPath)
+	predastorePath := filepath.Join(configDir, "predastore", "predastore.toml")
+
+	data, err := os.ReadFile(predastorePath)
+	if err != nil {
+		slog.Debug("handleStorageConfig: failed to read predastore config", "path", predastorePath, "err", err)
+		respondWithError(msg, "InternalError")
+		return
+	}
+
+	var cfg predastoreTOML
+	if err := toml.Unmarshal(data, &cfg); err != nil {
+		slog.Error("handleStorageConfig: failed to parse predastore config", "err", err)
+		respondWithError(msg, "InternalError")
+		return
+	}
+
+	resp := types.StorageConfigResponse{
+		Encoding: types.StorageEncoding{
+			DataShards:   cfg.RS.Data,
+			ParityShards: cfg.RS.Parity,
+		},
+	}
+
+	for _, db := range cfg.DB {
+		resp.DBNodes = append(resp.DBNodes, types.StorageDBNode{
+			ID:   db.ID,
+			Host: db.Host,
+			Port: db.Port,
+		})
+	}
+	if resp.DBNodes == nil {
+		resp.DBNodes = []types.StorageDBNode{}
+	}
+
+	for _, n := range cfg.Nodes {
+		resp.ShardNodes = append(resp.ShardNodes, types.StorageShardNode{
+			ID:   n.ID,
+			Host: n.Host,
+			Port: n.Port,
+		})
+	}
+	if resp.ShardNodes == nil {
+		resp.ShardNodes = []types.StorageShardNode{}
+	}
+
+	for _, b := range cfg.Buckets {
+		resp.Buckets = append(resp.Buckets, types.StorageBucket{
+			Name:   b.Name,
+			Type:   b.Type,
+			Region: b.Region,
+		})
+	}
+	if resp.Buckets == nil {
+		resp.Buckets = []types.StorageBucket{}
+	}
+
+	respondWithJSON(msg, resp)
+}

spinifex/daemon/daemon_handlers_test.go

@@ -16,10 +16,13 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
 	handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
+	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
 	handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw"
 	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
+	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
@@ -1733,6 +1736,27 @@ func TestHandleEC2ModifyInstanceAttribute_ChangeUserData(t *testing.T) {
 	assert.Equal(t, "IyEvYmluL2Jhc2g=", *updated.RunInstancesInput.UserData)
 }
 
+func TestHandleEC2ModifyInstanceAttribute_SourceDestCheck(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.ModifyInstanceAttribute", "spinifex-workers", daemon.handleEC2ModifyInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	// SourceDestCheck is a no-op that succeeds without requiring a stopped instance
+	// in KV — Terraform sends this on running instances right after creation.
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:      aws.String("i-modify-sdc-001"),
+		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.ModifyInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, `{}`, string(reply.Data))
+}
+
 func TestHandleEC2ModifyInstanceAttribute_InstanceNotFound(t *testing.T) {
 	natsURL := sharedJSNATSURL
 
@@ -1918,6 +1942,373 @@ func TestHandleEC2ModifyInstanceAttribute_InvalidJSON(t *testing.T) {
 	assert.Equal(t, "ServerInternal", errResp["Code"])
 }
 
+// --- DescribeInstanceAttribute daemon tests ---
+
+func TestHandleEC2DescribeInstanceAttribute_RunningInstance_InstanceType(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-run-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String(instanceID),
+			InstanceType: aws.String("t3.micro"),
+		},
+	}
+
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instanceID] = instance
+	daemon.Instances.Mu.Unlock()
+	t.Cleanup(func() {
+		daemon.Instances.Mu.Lock()
+		delete(daemon.Instances.VMS, instanceID)
+		daemon.Instances.Mu.Unlock()
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.InstanceType)
+	assert.Equal(t, "t3.micro", *output.InstanceType.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_StoppedInstance_InstanceType(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-stop-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.medium",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId:   aws.String(instanceID),
+			InstanceType: aws.String("t3.medium"),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.InstanceType)
+	assert.Equal(t, "t3.medium", *output.InstanceType.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_UserData(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-ud-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		UserData:     "#!/bin/bash",
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameUserData),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.UserData)
+	assert.Equal(t, "#!/bin/bash", *output.UserData.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTermination(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-def-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameDisableApiTermination),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.DisableApiTermination)
+	assert.Equal(t, false, *output.DisableApiTermination.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_ShutdownBehavior(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-shut-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceInitiatedShutdownBehavior),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	require.NotNil(t, output.InstanceInitiatedShutdownBehavior)
+	assert.Equal(t, "stop", *output.InstanceInitiatedShutdownBehavior.Value)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_GroupSet_WithSecurityGroups(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-gs-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+			SecurityGroups: []*ec2.GroupIdentifier{
+				{GroupId: aws.String("sg-111"), GroupName: aws.String("default")},
+				{GroupId: aws.String("sg-222"), GroupName: aws.String("web")},
+			},
+		},
+	}
+
+	daemon.Instances.Mu.Lock()
+	daemon.Instances.VMS[instanceID] = instance
+	daemon.Instances.Mu.Unlock()
+	t.Cleanup(func() {
+		daemon.Instances.Mu.Lock()
+		delete(daemon.Instances.VMS, instanceID)
+		daemon.Instances.Mu.Unlock()
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameGroupSet),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.Len(t, output.Groups, 2)
+	assert.Equal(t, "sg-111", *output.Groups[0].GroupId)
+	assert.Equal(t, "sg-222", *output.Groups[1].GroupId)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_GroupSet_NilInstance(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-gs-nil-001"
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateStopped,
+		InstanceType: "t3.micro",
+		AccountID:    testAccountID,
+		Instance:     nil,
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameGroupSet),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.DescribeInstanceAttributeOutput
+	err = json.Unmarshal(reply.Data, &output)
+	require.NoError(t, err)
+	assert.Equal(t, instanceID, *output.InstanceId)
+	require.NotNil(t, output.Groups, "Groups should be empty slice, not nil")
+	assert.Empty(t, output.Groups)
+}
+
+func TestHandleEC2DescribeInstanceAttribute_InstanceNotFound(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-nonexistent"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "InvalidInstanceID.NotFound", errResp["Code"])
+}
+
+func TestHandleEC2DescribeInstanceAttribute_UnsupportedAttribute(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	instanceID := "i-describe-unsup-001"
+	instance := &vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateStopped,
+		AccountID: testAccountID,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(instanceID),
+		},
+	}
+	err = daemon.jsManager.WriteStoppedInstance(instanceID, instance)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(instanceID) })
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(ec2.InstanceAttributeNameBlockDeviceMapping),
+	}
+	reqData, _ := json.Marshal(input)
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstanceAttribute", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "InvalidParameterValue", errResp["Code"])
+}
+
+func TestHandleEC2DescribeInstanceAttribute_InvalidJSON(t *testing.T) {
+	natsURL := sharedJSNATSURL
+
+	daemon := createFullTestDaemonWithJetStream(t, natsURL)
+
+	sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", daemon.handleEC2DescribeInstanceAttribute)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := daemon.natsConn.Request("ec2.DescribeInstanceAttribute", []byte(`{invalid`), 5*time.Second)
+	require.NoError(t, err)
+
+	var errResp map[string]any
+	err = json.Unmarshal(reply.Data, &errResp)
+	require.NoError(t, err)
+	assert.Equal(t, "ServerInternal", errResp["Code"])
+}
+
 // --- Delegate handler round-trip tests (table-driven) ---
 // Each of these handlers is a single line delegating to handleNATSRequest.
 // This test verifies the wiring is correct by sending a NATS request and
@@ -3482,3 +3873,505 @@ func TestHandleEC2TerminateStoppedInstance_WritesToTerminatedKV(t *testing.T) {
 
 	_ = daemon.jsManager.DeleteTerminatedInstance(stoppedVM.ID)
 }
+
+// --- Bead 5: EIP daemon handler tests ---
+
+func TestDelegateHandlers_EIP(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+	ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []handlers_ec2_vpc.ExternalPoolConfig{
+		{Name: "test-pool", RangeStart: "192.168.100.2", RangeEnd: "192.168.100.254", Gateway: "192.168.100.1", PrefixLen: 24},
+	})
+	require.NoError(t, err)
+
+	eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(nc, ipam, daemon.vpcService)
+	require.NoError(t, err)
+	daemon.eipService = eipSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"AllocateAddress",
+			"ec2.test.AllocateAddress",
+			daemon.handleEC2AllocateAddress,
+			&ec2.AllocateAddressInput{},
+		},
+		{
+			"ReleaseAddress",
+			"ec2.test.ReleaseAddress",
+			daemon.handleEC2ReleaseAddress,
+			&ec2.ReleaseAddressInput{AllocationId: aws.String("eipalloc-nonexistent")},
+		},
+		{
+			"AssociateAddress",
+			"ec2.test.AssociateAddress",
+			daemon.handleEC2AssociateAddress,
+			&ec2.AssociateAddressInput{AllocationId: aws.String("eipalloc-nonexistent")},
+		},
+		{
+			"DisassociateAddress",
+			"ec2.test.DisassociateAddress",
+			daemon.handleEC2DisassociateAddress,
+			&ec2.DisassociateAddressInput{AssociationId: aws.String("eipassoc-nonexistent")},
+		},
+		{
+			"DescribeAddresses",
+			"ec2.test.DescribeAddresses",
+			daemon.handleEC2DescribeAddresses,
+			&ec2.DescribeAddressesInput{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "EIP response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 6: Security Group daemon handler tests ---
+
+func TestDelegateHandlers_SecurityGroup(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Create a VPC first so SG operations have a valid target
+	createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", daemon.handleEC2CreateVpc)
+	require.NoError(t, err)
+	defer createVpcSub.Unsubscribe()
+
+	vpcInput := &ec2.CreateVpcInput{CidrBlock: aws.String("10.50.0.0/16")}
+	reqData, _ := json.Marshal(vpcInput)
+	reply, err := natsRequest(daemon.natsConn, "ec2.CreateVpc", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var vpcOut ec2.CreateVpcOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &vpcOut))
+	vpcID := *vpcOut.Vpc.VpcId
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreateSecurityGroup",
+			"ec2.test.CreateSecurityGroup",
+			daemon.handleEC2CreateSecurityGroup,
+			&ec2.CreateSecurityGroupInput{
+				GroupName:   aws.String("test-sg"),
+				Description: aws.String("test security group"),
+				VpcId:       aws.String(vpcID),
+			},
+		},
+		{
+			"DescribeSecurityGroups",
+			"ec2.test.DescribeSecurityGroups",
+			daemon.handleEC2DescribeSecurityGroups,
+			&ec2.DescribeSecurityGroupsInput{},
+		},
+		{
+			"AuthorizeSecurityGroupIngress",
+			"ec2.test.AuthorizeSecurityGroupIngress",
+			daemon.handleEC2AuthorizeSecurityGroupIngress,
+			&ec2.AuthorizeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"AuthorizeSecurityGroupEgress",
+			"ec2.test.AuthorizeSecurityGroupEgress",
+			daemon.handleEC2AuthorizeSecurityGroupEgress,
+			&ec2.AuthorizeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"RevokeSecurityGroupIngress",
+			"ec2.test.RevokeSecurityGroupIngress",
+			daemon.handleEC2RevokeSecurityGroupIngress,
+			&ec2.RevokeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"RevokeSecurityGroupEgress",
+			"ec2.test.RevokeSecurityGroupEgress",
+			daemon.handleEC2RevokeSecurityGroupEgress,
+			&ec2.RevokeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")},
+		},
+		{
+			"DeleteSecurityGroup",
+			"ec2.test.DeleteSecurityGroup",
+			daemon.handleEC2DeleteSecurityGroup,
+			&ec2.DeleteSecurityGroupInput{GroupId: aws.String("sg-nonexistent")},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "SG response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 7: Route Table daemon handler tests ---
+
+func TestDelegateHandlers_RouteTable(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Route table service needs its own JetStream for KV buckets
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(daemon.config, nc)
+	require.NoError(t, err)
+	daemon.routeTableService = rtbSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreateRouteTable",
+			"ec2.test.CreateRouteTable",
+			daemon.handleEC2CreateRouteTable,
+			&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-nonexistent")},
+		},
+		{
+			"DeleteRouteTable",
+			"ec2.test.DeleteRouteTable",
+			daemon.handleEC2DeleteRouteTable,
+			&ec2.DeleteRouteTableInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DescribeRouteTables",
+			"ec2.test.DescribeRouteTables",
+			daemon.handleEC2DescribeRouteTables,
+			&ec2.DescribeRouteTablesInput{},
+		},
+		{
+			"CreateRoute",
+			"ec2.test.CreateRoute",
+			daemon.handleEC2CreateRoute,
+			&ec2.CreateRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DeleteRoute",
+			"ec2.test.DeleteRoute",
+			daemon.handleEC2DeleteRoute,
+			&ec2.DeleteRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"ReplaceRoute",
+			"ec2.test.ReplaceRoute",
+			daemon.handleEC2ReplaceRoute,
+			&ec2.ReplaceRouteInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"AssociateRouteTable",
+			"ec2.test.AssociateRouteTable",
+			daemon.handleEC2AssociateRouteTable,
+			&ec2.AssociateRouteTableInput{RouteTableId: aws.String("rtb-nonexistent")},
+		},
+		{
+			"DisassociateRouteTable",
+			"ec2.test.DisassociateRouteTable",
+			daemon.handleEC2DisassociateRouteTable,
+			&ec2.DisassociateRouteTableInput{AssociationId: aws.String("rtbassoc-nonexistent")},
+		},
+		{
+			"ReplaceRouteTableAssociation",
+			"ec2.test.ReplaceRouteTableAssociation",
+			daemon.handleEC2ReplaceRouteTableAssociation,
+			&ec2.ReplaceRouteTableAssociationInput{
+				AssociationId: aws.String("rtbassoc-nonexistent"),
+				RouteTableId:  aws.String("rtb-nonexistent"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "RouteTable response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 8: Placement Group daemon handler tests ---
+
+func TestDelegateHandlers_PlacementGroup(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	ns, err := server.NewServer(&server.Options{
+		Host:      "127.0.0.1",
+		Port:      -1,
+		JetStream: true,
+		StoreDir:  t.TempDir(),
+		NoLog:     true,
+		NoSigs:    true,
+	})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(func() { ns.Shutdown() })
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(func() { nc.Close() })
+
+	pgSvc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(daemon.config, nc)
+	require.NoError(t, err)
+	daemon.placementGroupService = pgSvc
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"CreatePlacementGroup",
+			"ec2.test.CreatePlacementGroup",
+			daemon.handleEC2CreatePlacementGroup,
+			&ec2.CreatePlacementGroupInput{
+				GroupName: aws.String("test-pg"),
+				Strategy:  aws.String("spread"),
+			},
+		},
+		{
+			"DescribePlacementGroups",
+			"ec2.test.DescribePlacementGroups",
+			daemon.handleEC2DescribePlacementGroups,
+			&ec2.DescribePlacementGroupsInput{},
+		},
+		{
+			"DeletePlacementGroup",
+			"ec2.test.DeletePlacementGroup",
+			daemon.handleEC2DeletePlacementGroup,
+			&ec2.DeletePlacementGroupInput{GroupName: aws.String("pg-nonexistent")},
+		},
+		{
+			"ReserveSpreadNodes",
+			"ec2.test.ReserveSpreadNodes",
+			daemon.handleEC2ReserveSpreadNodes,
+			&handlers_ec2_placementgroup.ReserveSpreadNodesInput{
+				GroupName:     "pg-nonexistent",
+				EligibleNodes: []string{"node-1"},
+				MinCount:      1,
+				MaxCount:      1,
+			},
+		},
+		{
+			"FinalizeSpreadInstances",
+			"ec2.test.FinalizeSpreadInstances",
+			daemon.handleEC2FinalizeSpreadInstances,
+			&handlers_ec2_placementgroup.FinalizeSpreadInstancesInput{
+				GroupName:     "pg-nonexistent",
+				NodeInstances: map[string][]string{"node-1": {"i-123"}},
+			},
+		},
+		{
+			"ReleaseSpreadNodes",
+			"ec2.test.ReleaseSpreadNodes",
+			daemon.handleEC2ReleaseSpreadNodes,
+			&handlers_ec2_placementgroup.ReleaseSpreadNodesInput{
+				GroupName: "pg-nonexistent",
+				Nodes:     []string{"node-1"},
+			},
+		},
+		{
+			"RemoveInstanceFromPlacementGroup",
+			"ec2.test.RemoveInstanceFromPlacementGroup",
+			daemon.handleEC2RemoveInstanceFromPlacementGroup,
+			&handlers_ec2_placementgroup.RemoveInstanceInput{
+				GroupName:  "pg-nonexistent",
+				NodeName:   "node-1",
+				InstanceID: "i-123",
+			},
+		},
+		{
+			"ReserveClusterNode",
+			"ec2.test.ReserveClusterNode",
+			daemon.handleEC2ReserveClusterNode,
+			&handlers_ec2_placementgroup.ReserveClusterNodeInput{
+				GroupName:     "pg-nonexistent",
+				EligibleNodes: []string{"node-1"},
+			},
+		},
+		{
+			"FinalizeClusterInstances",
+			"ec2.test.FinalizeClusterInstances",
+			daemon.handleEC2FinalizeClusterInstances,
+			&handlers_ec2_placementgroup.FinalizeClusterInstancesInput{
+				GroupName:     "pg-nonexistent",
+				NodeInstances: map[string][]string{"node-1": {"i-123"}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "PlacementGroup response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}
+
+// --- Bead 9: VPC attribute daemon handler tests (untested handlers) ---
+
+func TestDelegateHandlers_VPCAttributes(t *testing.T) {
+	daemon := createVPCTestDaemon(t)
+
+	// Create a VPC first
+	createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", daemon.handleEC2CreateVpc)
+	require.NoError(t, err)
+	defer createVpcSub.Unsubscribe()
+
+	vpcInput := &ec2.CreateVpcInput{CidrBlock: aws.String("10.60.0.0/16")}
+	reqData, _ := json.Marshal(vpcInput)
+	reply, err := natsRequest(daemon.natsConn, "ec2.CreateVpc", reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var vpcOut ec2.CreateVpcOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &vpcOut))
+	vpcID := *vpcOut.Vpc.VpcId
+
+	tests := []struct {
+		name    string
+		topic   string
+		handler func(*nats.Msg)
+		input   any
+	}{
+		{
+			"ModifySubnetAttribute",
+			"ec2.test.ModifySubnetAttribute",
+			daemon.handleEC2ModifySubnetAttribute,
+			&ec2.ModifySubnetAttributeInput{SubnetId: aws.String("subnet-nonexistent")},
+		},
+		{
+			"ModifyVpcAttribute",
+			"ec2.test.ModifyVpcAttribute",
+			daemon.handleEC2ModifyVpcAttribute,
+			&ec2.ModifyVpcAttributeInput{VpcId: aws.String(vpcID)},
+		},
+		{
+			"DescribeVpcAttribute",
+			"ec2.test.DescribeVpcAttribute",
+			daemon.handleEC2DescribeVpcAttribute,
+			&ec2.DescribeVpcAttributeInput{
+				VpcId:     aws.String(vpcID),
+				Attribute: aws.String("enableDnsSupport"),
+			},
+		},
+		{
+			"ModifyNetworkInterfaceAttribute",
+			"ec2.test.ModifyNetworkInterfaceAttribute",
+			daemon.handleEC2ModifyNetworkInterfaceAttribute,
+			&ec2.ModifyNetworkInterfaceAttributeInput{NetworkInterfaceId: aws.String("eni-nonexistent")},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sub, err := daemon.natsConn.QueueSubscribe(tt.topic, "spinifex-workers", tt.handler)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			reqData, err := json.Marshal(tt.input)
+			require.NoError(t, err)
+
+			reply, err := natsRequest(daemon.natsConn, tt.topic, reqData, 5*time.Second)
+			require.NoError(t, err)
+			require.NotNil(t, reply)
+
+			var resp json.RawMessage
+			err = json.Unmarshal(reply.Data, &resp)
+			require.NoError(t, err, "VPC attribute response should be valid JSON: %s", string(reply.Data))
+		})
+	}
+}

spinifex/daemon/daemon_handlers_vpc.go

@@ -38,6 +38,14 @@ func (d *Daemon) handleEC2ModifySubnetAttribute(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.ModifySubnetAttribute)
 }
 
+func (d *Daemon) handleEC2ModifyVpcAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.ModifyVpcAttribute)
+}
+
+func (d *Daemon) handleEC2DescribeVpcAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.DescribeVpcAttribute)
+}
+
 func (d *Daemon) handleEC2CreateNetworkInterface(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.CreateNetworkInterface)
 }
@@ -46,6 +54,10 @@ func (d *Daemon) handleEC2DeleteNetworkInterface(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DeleteNetworkInterface)
 }
 
+func (d *Daemon) handleEC2ModifyNetworkInterfaceAttribute(msg *nats.Msg) {
+	handleNATSRequest(msg, d.vpcService.ModifyNetworkInterfaceAttribute)
+}
+
 func (d *Daemon) handleEC2DescribeNetworkInterfaces(msg *nats.Msg) {
 	handleNATSRequest(msg, d.vpcService.DescribeNetworkInterfaces)
 }
@@ -134,6 +146,11 @@ func (d *Daemon) ensureDefaultVPCInfrastructure() {
 			}
 		}
 
+		// Add 0.0.0.0/0 → IGW route to the main route table (if not already present)
+		if d.routeTableService != nil {
+			d.ensureDefaultIGWRoute(accountID, defaultVpcId)
+		}
+
 		// Check if default security group exists for this VPC
 		sgOut, err := d.vpcService.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{}, accountID)
 		if err != nil {
@@ -188,3 +205,67 @@ func (d *Daemon) ensureDefaultVPCInfrastructure() {
 		}
 	}
 }
+
+// ensureDefaultIGWRoute adds 0.0.0.0/0 → igw-xxx to the main route table if not present
+func (d *Daemon) ensureDefaultIGWRoute(accountID, vpcID string) {
+	// Find the main route table for this VPC
+	vpcFilter := "vpc-id"
+	mainFilter := "association.main"
+	trueVal := "true"
+	descOut, err := d.routeTableService.DescribeRouteTables(&ec2.DescribeRouteTablesInput{
+		Filters: []*ec2.Filter{
+			{Name: &vpcFilter, Values: []*string{&vpcID}},
+			{Name: &mainFilter, Values: []*string{&trueVal}},
+		},
+	}, accountID)
+	if err != nil {
+		slog.Warn("Failed to query main route table for default IGW route", "vpcId", vpcID, "err", err)
+		return
+	}
+	if len(descOut.RouteTables) == 0 {
+		slog.Debug("No main route table found for default IGW route", "vpcId", vpcID, "accountID", accountID)
+		return
+	}
+
+	mainRtb := descOut.RouteTables[0]
+
+	// Check if 0.0.0.0/0 route already exists
+	for _, r := range mainRtb.Routes {
+		if r.DestinationCidrBlock != nil && *r.DestinationCidrBlock == "0.0.0.0/0" {
+			return // Already has a default route
+		}
+	}
+
+	// Find the attached IGW for this VPC
+	igwOut, err := d.igwService.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{}, accountID)
+	if err != nil {
+		slog.Warn("Failed to query IGWs for default route", "vpcId", vpcID, "err", err)
+		return
+	}
+	var igwID string
+	for _, igw := range igwOut.InternetGateways {
+		for _, att := range igw.Attachments {
+			if att.VpcId != nil && *att.VpcId == vpcID {
+				igwID = *igw.InternetGatewayId
+				break
+			}
+		}
+	}
+	if igwID == "" {
+		return // No IGW attached
+	}
+
+	// Add the default route
+	dest := "0.0.0.0/0"
+	_, err = d.routeTableService.CreateRoute(&ec2.CreateRouteInput{
+		RouteTableId:         mainRtb.RouteTableId,
+		DestinationCidrBlock: &dest,
+		GatewayId:            &igwID,
+	}, accountID)
+	if err != nil {
+		slog.Warn("Failed to add default IGW route to main route table", "err", err)
+	} else {
+		slog.Info("Added default IGW route to main route table",
+			"routeTableId", *mainRtb.RouteTableId, "igwId", igwID, "vpcId", vpcID)
+	}
+}

spinifex/daemon/daemon_system_instance.go

@@ -0,0 +1,294 @@
+package daemon
+
+import (
+	"encoding/base64"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+// Compile-time check that Daemon implements SystemInstanceLauncher.
+var _ handlers_elbv2.SystemInstanceLauncher = (*Daemon)(nil)
+
+// LaunchSystemInstance creates and starts a system-managed VM. The VM is owned
+// by the system account (GlobalAccountID) and is not visible to customer
+// DescribeInstances calls.
+//
+// This is the internal equivalent of RunInstances — it reuses the same
+// instance service, resource manager, volume preparation, and QEMU launch
+// path, but skips the NATS request/response envelope and key pair validation.
+func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) {
+	accountID := utils.GlobalAccountID
+
+	// Validate instance type
+	instanceType, exists := d.resourceMgr.instanceTypes[input.InstanceType]
+	if !exists {
+		return nil, fmt.Errorf("unknown instance type: %s", input.InstanceType)
+	}
+
+	// Validate AMI
+	if d.imageService == nil {
+		return nil, fmt.Errorf("image service not initialized")
+	}
+	if _, err := d.imageService.GetAMIConfig(input.ImageID); err != nil {
+		return nil, fmt.Errorf("AMI %s not found: %w", input.ImageID, err)
+	}
+
+	// Allocate resources
+	if err := d.resourceMgr.allocate(instanceType); err != nil {
+		return nil, fmt.Errorf("insufficient capacity for %s: %w", input.InstanceType, err)
+	}
+
+	// Build a RunInstancesInput for the instance service
+	runInput := &ec2.RunInstancesInput{
+		InstanceType: aws.String(input.InstanceType),
+		ImageId:      aws.String(input.ImageID),
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(1),
+	}
+	if input.SubnetID != "" {
+		runInput.SubnetId = aws.String(input.SubnetID)
+	}
+	if input.UserData != "" {
+		encoded := base64.StdEncoding.EncodeToString([]byte(input.UserData))
+		runInput.UserData = aws.String(encoded)
+	}
+
+	// Create VM via instance service
+	instance, ec2Instance, err := d.instanceService.RunInstance(runInput)
+	if err != nil {
+		d.resourceMgr.deallocate(instanceType)
+		return nil, fmt.Errorf("create instance: %w", err)
+	}
+
+	instance.AccountID = accountID
+	instance.Reservation = &ec2.Reservation{}
+	instance.Reservation.SetReservationId(utils.GenerateResourceID("r"))
+	instance.Reservation.SetOwnerId(accountID)
+	instance.Reservation.Instances = []*ec2.Instance{ec2Instance}
+
+	// Attach ENI — either use pre-created one or auto-create
+	privateIP := ""
+	if input.ENIID != "" {
+		// Use pre-created ENI (e.g. ALB ENI)
+		instance.ENIId = input.ENIID
+		instance.ENIMac = input.ENIMac
+		privateIP = input.ENIIP
+		ec2Instance.SetPrivateIpAddress(privateIP)
+		if input.SubnetID != "" {
+			ec2Instance.SetSubnetId(input.SubnetID)
+		}
+		// Mark ENI as attached to this instance
+		if d.vpcService != nil {
+			if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+				slog.Warn("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
+			}
+		}
+	} else if input.SubnetID != "" && d.vpcService != nil {
+		// Auto-create ENI in subnet
+		eniOut, eniErr := d.vpcService.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
+			SubnetId:    aws.String(input.SubnetID),
+			Description: aws.String("System interface for " + instance.ID),
+		}, accountID)
+		if eniErr != nil {
+			d.resourceMgr.deallocate(instanceType)
+			return nil, fmt.Errorf("create ENI: %w", eniErr)
+		}
+		eni := eniOut.NetworkInterface
+		instance.ENIId = *eni.NetworkInterfaceId
+		instance.ENIMac = *eni.MacAddress
+		privateIP = *eni.PrivateIpAddress
+		ec2Instance.SetPrivateIpAddress(privateIP)
+		ec2Instance.SetSubnetId(input.SubnetID)
+		ec2Instance.SetVpcId(*eni.VpcId)
+		// Mark ENI as attached
+		if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+			slog.Warn("LaunchSystemInstance: failed to attach auto-created ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
+		}
+	}
+
+	// Add to daemon state so LaunchInstance can find it
+	d.Instances.Mu.Lock()
+	d.Instances.VMS[instance.ID] = instance
+	d.Instances.Mu.Unlock()
+
+	if err := d.WriteState(); err != nil {
+		slog.Warn("LaunchSystemInstance: failed to write state", "instanceId", instance.ID, "err", err)
+	}
+
+	// Subscribe to per-instance NATS topic for terminate commands
+	d.mu.Lock()
+	sub, subErr := d.natsConn.Subscribe(fmt.Sprintf("ec2.cmd.%s", instance.ID), d.handleEC2Events)
+	if subErr != nil {
+		slog.Warn("LaunchSystemInstance: failed to subscribe to instance topic", "instanceId", instance.ID, "err", subErr)
+	} else {
+		d.natsSubscriptions[instance.ID] = sub
+	}
+	d.mu.Unlock()
+
+	// Pre-compute dev MAC for dual-NIC cloud-init
+	if d.config.Daemon.DevNetworking && instance.ENIId != "" {
+		instance.DevMAC = generateDevMAC(instance.ID)
+	}
+
+	// Store extra hostfwd ports (filled in by StartInstance with actual host ports)
+	if len(input.HostfwdPorts) > 0 {
+		instance.ExtraHostfwd = make(map[int]int, len(input.HostfwdPorts))
+		for _, port := range input.HostfwdPorts {
+			instance.ExtraHostfwd[port] = 0 // host port assigned in StartInstance
+		}
+	}
+
+	// Prepare volumes (root volume clone from AMI, cloud-init ISO, etc.)
+	volumeInfos, err := d.instanceService.GenerateVolumes(runInput, instance)
+	if err != nil {
+		d.cleanupFailedSystemInstance(instance, instanceType)
+		return nil, fmt.Errorf("generate volumes: %w", err)
+	}
+
+	instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos))
+	for _, vi := range volumeInfos {
+		mapping := &ec2.InstanceBlockDeviceMapping{}
+		mapping.SetDeviceName(vi.DeviceName)
+		mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
+		mapping.Ebs.SetVolumeId(vi.VolumeId)
+		mapping.Ebs.SetAttachTime(vi.AttachTime)
+		mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination)
+		mapping.Ebs.SetStatus("attached")
+		instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
+	}
+
+	// Launch QEMU VM
+	if err := d.LaunchInstance(instance); err != nil {
+		d.cleanupFailedSystemInstance(instance, instanceType)
+		return nil, fmt.Errorf("launch instance: %w", err)
+	}
+
+	slog.Info("LaunchSystemInstance completed",
+		"instanceId", instance.ID,
+		"instanceType", input.InstanceType,
+		"privateIp", privateIP,
+	)
+
+	return &handlers_elbv2.SystemInstanceOutput{
+		InstanceID: instance.ID,
+		PrivateIP:  privateIP,
+		HostfwdMap: instance.ExtraHostfwd,
+	}, nil
+}
+
+// TerminateSystemInstance stops and cleans up a system-managed VM.
+func (d *Daemon) TerminateSystemInstance(instanceID string) error {
+	d.Instances.Mu.Lock()
+	instance, exists := d.Instances.VMS[instanceID]
+	d.Instances.Mu.Unlock()
+
+	if !exists {
+		return fmt.Errorf("instance %s not found", instanceID)
+	}
+
+	// Transition to shutting-down
+	if err := d.TransitionState(instance, vm.StateShuttingDown); err != nil {
+		return fmt.Errorf("transition to shutting-down: %w", err)
+	}
+
+	// Stop the VM (QEMU shutdown, volume unmount, tap cleanup)
+	if err := d.stopInstance(map[string]*vm.VM{instanceID: instance}, true); err != nil {
+		slog.Error("TerminateSystemInstance: stopInstance failed", "instanceId", instanceID, "err", err)
+		return fmt.Errorf("stop instance: %w", err)
+	}
+
+	if err := d.TransitionState(instance, vm.StateTerminated); err != nil {
+		slog.Warn("TerminateSystemInstance: failed to transition to terminated", "instanceId", instanceID, "err", err)
+	}
+
+	// Write to terminated KV bucket (auto-expires)
+	if d.jsManager != nil {
+		if err := d.jsManager.WriteTerminatedInstance(instanceID, instance); err != nil {
+			slog.Warn("TerminateSystemInstance: failed to write terminated state", "instanceId", instanceID, "err", err)
+		}
+	}
+
+	// Clean up local state
+	d.Instances.Mu.Lock()
+	delete(d.Instances.VMS, instanceID)
+	d.Instances.Mu.Unlock()
+
+	// Unsubscribe from per-instance NATS topic
+	d.mu.Lock()
+	if sub, ok := d.natsSubscriptions[instanceID]; ok {
+		if err := sub.Unsubscribe(); err != nil {
+			slog.Warn("TerminateSystemInstance: failed to unsubscribe", "instanceId", instanceID, "err", err)
+		}
+		delete(d.natsSubscriptions, instanceID)
+	}
+	d.mu.Unlock()
+
+	// Deallocate resources
+	if it, ok := d.resourceMgr.instanceTypes[instance.InstanceType]; ok {
+		d.resourceMgr.deallocate(it)
+	}
+
+	slog.Info("TerminateSystemInstance completed", "instanceId", instanceID)
+	return nil
+}
+
+// cleanupFailedSystemInstance handles cleanup when a system instance launch fails
+// after partial setup (state added, volumes created, etc).
+func (d *Daemon) cleanupFailedSystemInstance(instance *vm.VM, instanceType *ec2.InstanceTypeInfo) {
+	d.markInstanceFailed(instance, "system_instance_launch_failed")
+	d.resourceMgr.deallocate(instanceType)
+
+	// Clean up ENI if we auto-created one (not pre-created)
+	// Pre-created ENIs are managed by the caller (e.g. ELBv2 service)
+	if instance.ENIId != "" && d.vpcService != nil {
+		// Only clean up ENIs we created (description starts with "System interface")
+		result, err := d.vpcService.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
+			NetworkInterfaceIds: []*string{aws.String(instance.ENIId)},
+		}, utils.GlobalAccountID)
+		if err == nil && len(result.NetworkInterfaces) > 0 {
+			desc := aws.StringValue(result.NetworkInterfaces[0].Description)
+			if len(desc) >= 16 && desc[:16] == "System interface" {
+				if _, delErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
+					NetworkInterfaceId: aws.String(instance.ENIId),
+				}, utils.GlobalAccountID); delErr != nil {
+					slog.Warn("Failed to cleanup system ENI", "eniId", instance.ENIId, "err", delErr)
+				}
+			}
+		}
+	}
+}
+
+// WaitForSystemInstance polls until the instance reaches running state or times out.
+func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		d.Instances.Mu.Lock()
+		inst, exists := d.Instances.VMS[instanceID]
+		var status vm.InstanceState
+		if exists {
+			status = inst.Status
+		}
+		d.Instances.Mu.Unlock()
+
+		if !exists {
+			return fmt.Errorf("instance %s disappeared", instanceID)
+		}
+
+		switch status {
+		case vm.StateRunning:
+			return nil
+		case vm.StateError, vm.StateShuttingDown, vm.StateTerminated:
+			return fmt.Errorf("instance %s in terminal state: %s", instanceID, status)
+		}
+
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("instance %s did not reach running state within %s", instanceID, timeout)
+}

spinifex/daemon/daemon_system_instance_test.go

@@ -0,0 +1,90 @@
+package daemon
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+func TestWaitForSystemInstance_AlreadyRunning(t *testing.T) {
+	d := &Daemon{
+		Instances: vm.Instances{
+			VMS: map[string]*vm.VM{
+				"i-test1": {ID: "i-test1", Status: vm.StateRunning},
+			},
+		},
+	}
+
+	err := d.WaitForSystemInstance("i-test1", 1*time.Second)
+	if err != nil {
+		t.Fatalf("expected no error for running instance, got: %v", err)
+	}
+}
+
+func TestWaitForSystemInstance_NotFound(t *testing.T) {
+	d := &Daemon{
+		Instances: vm.Instances{
+			VMS: map[string]*vm.VM{},
+		},
+	}
+
+	err := d.WaitForSystemInstance("i-nonexistent", 500*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected error for missing instance")
+	}
+}
+
+func TestWaitForSystemInstance_ErrorState(t *testing.T) {
+	d := &Daemon{
+		Instances: vm.Instances{
+			VMS: map[string]*vm.VM{
+				"i-failed": {ID: "i-failed", Status: vm.StateError},
+			},
+		},
+	}
+
+	err := d.WaitForSystemInstance("i-failed", 1*time.Second)
+	if err == nil {
+		t.Fatal("expected error for failed instance")
+	}
+}
+
+func TestWaitForSystemInstance_TransitionsToRunning(t *testing.T) {
+	inst := &vm.VM{ID: "i-pending", Status: vm.StateProvisioning}
+	d := &Daemon{
+		Instances: vm.Instances{
+			VMS: map[string]*vm.VM{
+				"i-pending": inst,
+			},
+		},
+	}
+
+	// Transition to running after a short delay
+	go func() {
+		time.Sleep(200 * time.Millisecond)
+		d.Instances.Mu.Lock()
+		inst.Status = vm.StateRunning
+		d.Instances.Mu.Unlock()
+	}()
+
+	err := d.WaitForSystemInstance("i-pending", 2*time.Second)
+	if err != nil {
+		t.Fatalf("expected instance to reach running, got: %v", err)
+	}
+}
+
+func TestWaitForSystemInstance_Timeout(t *testing.T) {
+	d := &Daemon{
+		Instances: vm.Instances{
+			VMS: map[string]*vm.VM{
+				"i-stuck": {ID: "i-stuck", Status: vm.StateProvisioning},
+			},
+		},
+	}
+
+	err := d.WaitForSystemInstance("i-stuck", 600*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected timeout error")
+	}
+}

spinifex/daemon/daemon_test.go

@@ -3838,3 +3838,191 @@ func TestDaemonReadyFlag(t *testing.T) {
 	daemon.ready.Store(true)
 	assert.True(t, daemon.ready.Load(), "daemon should be ready after setting flag")
 }
+
+func TestBuildBaseVMConfig(t *testing.T) {
+	tests := []struct {
+		name         string
+		instanceID   string
+		pidFile      string
+		consolePath  string
+		serialSocket string
+		architecture string
+		vCPUs        int
+		memoryMiB    int
+	}{
+		{
+			name:         "x86_64 instance",
+			instanceID:   "i-abc123",
+			pidFile:      "/tmp/qemu-i-abc123.pid",
+			consolePath:  "/run/console-i-abc123.log",
+			serialSocket: "/run/serial-i-abc123.sock",
+			architecture: "x86_64",
+			vCPUs:        4,
+			memoryMiB:    8192,
+		},
+		{
+			name:         "arm64 instance",
+			instanceID:   "i-def456",
+			pidFile:      "/tmp/qemu-i-def456.pid",
+			consolePath:  "/run/console-i-def456.log",
+			serialSocket: "/run/serial-i-def456.sock",
+			architecture: "arm64",
+			vCPUs:        2,
+			memoryMiB:    4096,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := buildBaseVMConfig(tt.instanceID, tt.pidFile, tt.consolePath, tt.serialSocket, tt.architecture, tt.vCPUs, tt.memoryMiB)
+
+			assert.Equal(t, tt.instanceID, cfg.Name)
+			assert.Equal(t, tt.pidFile, cfg.PIDFile)
+			assert.Equal(t, tt.consolePath, cfg.ConsoleLogPath)
+			assert.Equal(t, tt.serialSocket, cfg.SerialSocket)
+			assert.Equal(t, tt.architecture, cfg.Architecture)
+			assert.Equal(t, tt.vCPUs, cfg.CPUCount)
+			assert.Equal(t, tt.memoryMiB, cfg.Memory)
+			assert.True(t, cfg.EnableKVM)
+			assert.True(t, cfg.NoGraphic)
+			assert.Equal(t, "q35", cfg.MachineType)
+			assert.Equal(t, "host", cfg.CPUType)
+
+			// 11 PCIe root ports for hotplug slots
+			require.Len(t, cfg.Devices, 11)
+			for i, dev := range cfg.Devices {
+				expected := fmt.Sprintf("pcie-root-port,id=hotplug%d,chassis=%d,slot=0", i+1, i+1)
+				assert.Equal(t, expected, dev.Value)
+			}
+		})
+	}
+}
+
+func TestBuildDrives(t *testing.T) {
+	tests := []struct {
+		name          string
+		requests      []types.EBSRequest
+		cpuCount      int
+		wantDrives    int
+		wantIOThreads int
+		wantDevices   int
+		wantErr       string
+	}{
+		{
+			name: "boot volume",
+			requests: []types.EBSRequest{
+				{Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true},
+			},
+			cpuCount:      4,
+			wantDrives:    1,
+			wantIOThreads: 1,
+			wantDevices:   1,
+		},
+		{
+			name: "cloud-init volume",
+			requests: []types.EBSRequest{
+				{Name: "vol-ci", NBDURI: "nbd:unix:/tmp/ci.sock", CloudInit: true},
+			},
+			cpuCount:      2,
+			wantDrives:    1,
+			wantIOThreads: 0,
+			wantDevices:   0,
+		},
+		{
+			name: "EFI volume skipped",
+			requests: []types.EBSRequest{
+				{Name: "vol-efi", EFI: true},
+			},
+			cpuCount:      2,
+			wantDrives:    0,
+			wantIOThreads: 0,
+			wantDevices:   0,
+		},
+		{
+			name: "missing NBDURI returns error",
+			requests: []types.EBSRequest{
+				{Name: "vol-bad"},
+			},
+			cpuCount: 2,
+			wantErr:  "NBDURI not set for volume vol-bad",
+		},
+		{
+			name: "mixed boot + cloud-init + EFI",
+			requests: []types.EBSRequest{
+				{Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true},
+				{Name: "vol-ci", NBDURI: "nbd:unix:/tmp/ci.sock", CloudInit: true},
+				{Name: "vol-efi", EFI: true},
+			},
+			cpuCount:      4,
+			wantDrives:    2, // boot + cloud-init (EFI skipped)
+			wantIOThreads: 1, // only boot gets iothread
+			wantDevices:   1, // only boot gets virtio-blk device
+		},
+		{
+			name:          "empty requests",
+			requests:      []types.EBSRequest{},
+			cpuCount:      2,
+			wantDrives:    0,
+			wantIOThreads: 0,
+			wantDevices:   0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			drives, iothreads, devices, err := buildDrives(tt.requests, tt.cpuCount)
+
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Len(t, drives, tt.wantDrives)
+			assert.Len(t, iothreads, tt.wantIOThreads)
+			assert.Len(t, devices, tt.wantDevices)
+		})
+	}
+}
+
+func TestBuildDrives_BootVolume(t *testing.T) {
+	requests := []types.EBSRequest{
+		{Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true},
+	}
+
+	drives, iothreads, devices, err := buildDrives(requests, 4)
+	require.NoError(t, err)
+
+	require.Len(t, drives, 1)
+	d := drives[0]
+	assert.Equal(t, "nbd:unix:/tmp/boot.sock", d.File)
+	assert.Equal(t, "raw", d.Format)
+	assert.Equal(t, "none", d.If)
+	assert.Equal(t, "disk", d.Media)
+	assert.Equal(t, "os", d.ID)
+	assert.Equal(t, "none", d.Cache)
+
+	require.Len(t, iothreads, 1)
+	assert.Equal(t, "ioth-os", iothreads[0].ID)
+
+	require.Len(t, devices, 1)
+	assert.Equal(t, "virtio-blk-pci,drive=os,iothread=ioth-os,num-queues=4,bootindex=1", devices[0].Value)
+}
+
+func TestBuildDrives_CloudInitVolume(t *testing.T) {
+	requests := []types.EBSRequest{
+		{Name: "vol-ci", NBDURI: "nbd:unix:/tmp/ci.sock", CloudInit: true},
+	}
+
+	drives, _, _, err := buildDrives(requests, 2)
+	require.NoError(t, err)
+
+	require.Len(t, drives, 1)
+	d := drives[0]
+	assert.Equal(t, "nbd:unix:/tmp/ci.sock", d.File)
+	assert.Equal(t, "raw", d.Format)
+	assert.Equal(t, "virtio", d.If)
+	assert.Equal(t, "cdrom", d.Media)
+	assert.Equal(t, "cloudinit", d.ID)
+}

spinifex/daemon/instance_filter_test.go

@@ -0,0 +1,218 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/filterutil"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInstanceMatchesFilters_NoFilters(t *testing.T) {
+	inst := &vm.VM{ID: "i-123", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-123"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	assert.True(t, instanceMatchesFilters(inst, ic, nil))
+}
+
+func TestInstanceMatchesFilters_SingleFilter_Match(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{
+		"instance-state-name": {"running"},
+	}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_SingleFilter_NoMatch(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{
+		"instance-state-name": {"stopped"},
+	}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_MultipleValues_ORLogic(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{
+		"instance-state-name": {"running", "stopped"},
+	}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_MultipleFilters_ANDLogic(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		VpcId:        aws.String("vpc-111"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	// Both must match
+	filters := map[string][]string{
+		"instance-state-name": {"running"},
+		"vpc-id":              {"vpc-111"},
+	}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	// One doesn't match
+	filters["vpc-id"] = []string{"vpc-999"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_InstanceId(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{"instance-id": {"i-abc"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["instance-id"] = []string{"i-other"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_InstanceType(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "m5.large"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("m5.large"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{"instance-type": {"m5.large"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_SubnetId(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		SubnetId:   aws.String("subnet-123"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{"subnet-id": {"subnet-123"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["subnet-id"] = []string{"subnet-999"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_Wildcard(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	filters := map[string][]string{"instance-type": {"t3.*"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["instance-type"] = []string{"m5.*"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_TagFilter(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+		Tags: []*ec2.Tag{
+			{Key: aws.String("Environment"), Value: aws.String("prod")},
+			{Key: aws.String("Team"), Value: aws.String("backend")},
+		},
+	}
+
+	filters := map[string][]string{"tag:Environment": {"prod"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["tag:Environment"] = []string{"staging"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_TagKey(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+		Tags: []*ec2.Tag{
+			{Key: aws.String("Environment"), Value: aws.String("prod")},
+		},
+	}
+	filters := map[string][]string{"tag-key": {"Environment"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["tag-key"] = []string{"MissingKey"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_TagValue(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc"}
+	ic := &ec2.Instance{
+		InstanceId: aws.String("i-abc"),
+		State:      &ec2.InstanceState{Name: aws.String("running")},
+		Tags: []*ec2.Tag{
+			{Key: aws.String("Environment"), Value: aws.String("prod")},
+		},
+	}
+	filters := map[string][]string{"tag-value": {"prod"}}
+	assert.True(t, instanceMatchesFilters(inst, ic, filters))
+
+	filters["tag-value"] = []string{"staging"}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestInstanceMatchesFilters_EmptyResults(t *testing.T) {
+	inst := &vm.VM{ID: "i-abc", InstanceType: "t3.micro"}
+	ic := &ec2.Instance{
+		InstanceId:   aws.String("i-abc"),
+		InstanceType: aws.String("t3.micro"),
+		State:        &ec2.InstanceState{Name: aws.String("running")},
+	}
+	// Filter that matches nothing
+	filters := map[string][]string{"instance-type": {"c5.4xlarge"}}
+	assert.False(t, instanceMatchesFilters(inst, ic, filters))
+}
+
+func TestParseFilters_UnknownFilterName(t *testing.T) {
+	filters := []*ec2.Filter{
+		{Name: aws.String("bogus-filter"), Values: []*string{aws.String("val")}},
+	}
+	_, err := filterutil.ParseFilters(filters, describeInstancesValidFilters)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "InvalidParameterValue")
+}
+
+func TestEC2TagsToMap(t *testing.T) {
+	tags := []*ec2.Tag{
+		{Key: aws.String("a"), Value: aws.String("1")},
+		{Key: aws.String("b"), Value: aws.String("2")},
+	}
+	m := filterutil.EC2TagsToMap(tags)
+	assert.Equal(t, "1", m["a"])
+	assert.Equal(t, "2", m["b"])
+}
+
+func TestEC2TagsToMap_Nil(t *testing.T) {
+	assert.Nil(t, filterutil.EC2TagsToMap(nil))
+}

spinifex/daemon/state_test.go

@@ -10,10 +10,9 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/vm"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,23 +22,7 @@ import (
 func createDaemonWithJetStream(t *testing.T) *Daemon {
 	t.Helper()
 
-	jsTmpDir := t.TempDir()
-
-	opts := &server.Options{
-		Host:      "127.0.0.1",
-		Port:      -1,
-		JetStream: true,
-		StoreDir:  jsTmpDir,
-		NoLog:     true,
-		NoSigs:    true,
-	}
-	ns, err := server.NewServer(opts)
-	require.NoError(t, err)
-	go ns.Start()
-	require.True(t, ns.ReadyForConnections(5*time.Second), "NATS server failed to start")
-	t.Cleanup(func() { ns.Shutdown() })
-
-	natsURL := ns.ClientURL()
+	_, nc, _ := testutil.StartTestJetStream(t)
 
 	tmpDir := t.TempDir()
 
@@ -51,10 +34,6 @@ func createDaemonWithJetStream(t *testing.T) *Daemon {
 	require.NoError(t, err)
 	daemon.config = &config.Config{BaseDir: tmpDir}
 
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	t.Cleanup(func() { nc.Close() })
-
 	daemon.natsConn = nc
 	daemon.jsManager, err = NewJetStreamManager(nc, 1)
 	require.NoError(t, err)

spinifex/filterutil/filterutil.go

@@ -0,0 +1,130 @@
+package filterutil
+
+import (
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// ParseFilters converts AWS SDK filter types into a map[string][]string for
+// easier matching. validNames defines the set of accepted filter names.
+// Filter names starting with "tag:" are always accepted. Returns an
+// InvalidParameterValue-style error if an unsupported filter name is
+// encountered.
+func ParseFilters(filters []*ec2.Filter, validNames map[string]bool) (map[string][]string, error) {
+	if len(filters) == 0 {
+		return nil, nil
+	}
+
+	result := make(map[string][]string, len(filters))
+	for _, f := range filters {
+		if f.Name == nil {
+			slog.Warn("ParseFilters: skipping filter with nil Name")
+			continue
+		}
+		name := *f.Name
+
+		if !strings.HasPrefix(name, "tag:") && !validNames[name] {
+			return nil, fmt.Errorf("InvalidParameterValue: The filter '%s' is invalid", name)
+		}
+
+		for _, v := range f.Values {
+			if v != nil {
+				result[name] = append(result[name], *v)
+			}
+		}
+	}
+	return result, nil
+}
+
+// MatchesAny returns true if value matches any of the filter values.
+// Supports the AWS wildcard convention where * matches any substring.
+// Returns true if filterValues is empty.
+func MatchesAny(filterValues []string, value string) bool {
+	if len(filterValues) == 0 {
+		return true
+	}
+	for _, pattern := range filterValues {
+		if matchWildcard(pattern, value) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchesTags checks whether a resource's tags satisfy all tag:Key filters
+// present in the filter map. Each tag:Key filter uses OR logic across its
+// values — the resource must have the tag and its value must match at least
+// one filter value (with wildcard support).
+func MatchesTags(filters map[string][]string, tags map[string]string) bool {
+	for name, values := range filters {
+		if !strings.HasPrefix(name, "tag:") {
+			continue
+		}
+		tagKey := name[4:] // strip "tag:" prefix
+		tagValue, exists := tags[tagKey]
+		if !exists {
+			return false
+		}
+		if !MatchesAny(values, tagValue) {
+			return false
+		}
+	}
+	return true
+}
+
+// EC2TagsToMap converts []*ec2.Tag to map[string]string for MatchesTags.
+func EC2TagsToMap(tags []*ec2.Tag) map[string]string {
+	if len(tags) == 0 {
+		return nil
+	}
+	m := make(map[string]string, len(tags))
+	for _, t := range tags {
+		if t.Key != nil && t.Value != nil {
+			m[*t.Key] = *t.Value
+		}
+	}
+	return m
+}
+
+// matchWildcard matches value against a pattern where * matches zero or more
+// characters. This implements the AWS filter wildcard convention.
+func matchWildcard(pattern, value string) bool {
+	if pattern == "*" {
+		return true
+	}
+	if !strings.Contains(pattern, "*") {
+		return pattern == value
+	}
+
+	parts := strings.Split(pattern, "*")
+	last := len(parts) - 1
+
+	// Check that value starts with the first part and ends with the last.
+	if !strings.HasPrefix(value, parts[0]) {
+		return false
+	}
+	if !strings.HasSuffix(value, parts[last]) {
+		return false
+	}
+
+	// Trim prefix and suffix from the search region so middle parts
+	// cannot consume characters reserved for the anchored ends.
+	remaining := value[len(parts[0]):]
+	if len(remaining) < len(parts[last]) {
+		return false
+	}
+	remaining = remaining[:len(remaining)-len(parts[last])]
+
+	// Walk through middle parts in order.
+	for i := 1; i < last; i++ {
+		idx := strings.Index(remaining, parts[i])
+		if idx < 0 {
+			return false
+		}
+		remaining = remaining[idx+len(parts[i]):]
+	}
+	return true
+}

spinifex/filterutil/filterutil_test.go

@@ -0,0 +1,307 @@
+package filterutil
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+func TestParseFilters_NilInput(t *testing.T) {
+	result, err := ParseFilters(nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if result != nil {
+		t.Fatalf("expected nil, got %v", result)
+	}
+}
+
+func TestParseFilters_EmptyInput(t *testing.T) {
+	result, err := ParseFilters([]*ec2.Filter{}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if result != nil {
+		t.Fatalf("expected nil, got %v", result)
+	}
+}
+
+func TestParseFilters_ValidFilters(t *testing.T) {
+	valid := map[string]bool{"vpc-id": true, "state": true}
+	filters := []*ec2.Filter{
+		{Name: aws.String("vpc-id"), Values: []*string{aws.String("vpc-1"), aws.String("vpc-2")}},
+		{Name: aws.String("state"), Values: []*string{aws.String("running")}},
+	}
+
+	result, err := ParseFilters(filters, valid)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(result) != 2 {
+		t.Fatalf("expected 2 filter keys, got %d", len(result))
+	}
+	if len(result["vpc-id"]) != 2 {
+		t.Fatalf("expected 2 vpc-id values, got %d", len(result["vpc-id"]))
+	}
+	if result["state"][0] != "running" {
+		t.Fatalf("expected state=running, got %s", result["state"][0])
+	}
+}
+
+func TestParseFilters_InvalidFilterName(t *testing.T) {
+	valid := map[string]bool{"vpc-id": true}
+	filters := []*ec2.Filter{
+		{Name: aws.String("bogus-filter"), Values: []*string{aws.String("x")}},
+	}
+
+	_, err := ParseFilters(filters, valid)
+	if err == nil {
+		t.Fatal("expected error for invalid filter name")
+	}
+}
+
+func TestParseFilters_TagFiltersAlwaysAccepted(t *testing.T) {
+	valid := map[string]bool{"vpc-id": true}
+	filters := []*ec2.Filter{
+		{Name: aws.String("tag:Environment"), Values: []*string{aws.String("prod")}},
+	}
+
+	result, err := ParseFilters(filters, valid)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(result["tag:Environment"]) != 1 {
+		t.Fatalf("expected 1 tag filter value, got %d", len(result["tag:Environment"]))
+	}
+}
+
+func TestParseFilters_NilNameSkipped(t *testing.T) {
+	valid := map[string]bool{"vpc-id": true}
+	filters := []*ec2.Filter{
+		{Name: nil, Values: []*string{aws.String("x")}},
+		{Name: aws.String("vpc-id"), Values: []*string{aws.String("vpc-1")}},
+	}
+
+	result, err := ParseFilters(filters, valid)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(result) != 1 {
+		t.Fatalf("expected 1 filter key, got %d", len(result))
+	}
+}
+
+func TestParseFilters_NilValuesSkipped(t *testing.T) {
+	valid := map[string]bool{"state": true}
+	filters := []*ec2.Filter{
+		{Name: aws.String("state"), Values: []*string{aws.String("running"), nil, aws.String("stopped")}},
+	}
+
+	result, err := ParseFilters(filters, valid)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(result["state"]) != 2 {
+		t.Fatalf("expected 2 state values, got %d", len(result["state"]))
+	}
+}
+
+func TestMatchesAny_EmptyFilterValues(t *testing.T) {
+	if !MatchesAny(nil, "anything") {
+		t.Fatal("expected true for empty filter values")
+	}
+	if !MatchesAny([]string{}, "anything") {
+		t.Fatal("expected true for empty filter values")
+	}
+}
+
+func TestMatchesAny_ExactMatch(t *testing.T) {
+	if !MatchesAny([]string{"running", "stopped"}, "running") {
+		t.Fatal("expected match for 'running'")
+	}
+	if MatchesAny([]string{"running", "stopped"}, "terminated") {
+		t.Fatal("expected no match for 'terminated'")
+	}
+}
+
+func TestMatchesAny_WildcardStar(t *testing.T) {
+	if !MatchesAny([]string{"*"}, "anything") {
+		t.Fatal("expected * to match anything")
+	}
+}
+
+func TestMatchesAny_WildcardPrefix(t *testing.T) {
+	if !MatchesAny([]string{"prod-*"}, "prod-web") {
+		t.Fatal("expected prod-* to match prod-web")
+	}
+	if MatchesAny([]string{"prod-*"}, "staging-web") {
+		t.Fatal("expected prod-* to not match staging-web")
+	}
+}
+
+func TestMatchesAny_WildcardSuffix(t *testing.T) {
+	if !MatchesAny([]string{"*-web"}, "prod-web") {
+		t.Fatal("expected *-web to match prod-web")
+	}
+	if MatchesAny([]string{"*-web"}, "prod-api") {
+		t.Fatal("expected *-web to not match prod-api")
+	}
+}
+
+func TestMatchesAny_WildcardMiddle(t *testing.T) {
+	if !MatchesAny([]string{"prod-*-web"}, "prod-us-east-web") {
+		t.Fatal("expected prod-*-web to match prod-us-east-web")
+	}
+	if MatchesAny([]string{"prod-*-web"}, "prod-us-east-api") {
+		t.Fatal("expected prod-*-web to not match prod-us-east-api")
+	}
+}
+
+func TestMatchesAny_MultipleWildcards(t *testing.T) {
+	if !MatchesAny([]string{"*prod*web*"}, "my-prod-us-web-server") {
+		t.Fatal("expected *prod*web* to match")
+	}
+	if MatchesAny([]string{"*prod*web*"}, "my-staging-us-api-server") {
+		t.Fatal("expected *prod*web* to not match")
+	}
+}
+
+func TestMatchesAny_EmptyStringMatches(t *testing.T) {
+	if !MatchesAny([]string{""}, "") {
+		t.Fatal("expected empty string to match empty string")
+	}
+	if MatchesAny([]string{""}, "notempty") {
+		t.Fatal("expected empty string to not match non-empty")
+	}
+}
+
+func TestMatchesTags_NoTagFilters(t *testing.T) {
+	filters := map[string][]string{
+		"vpc-id": {"vpc-1"},
+	}
+	tags := map[string]string{"Environment": "prod"}
+	if !MatchesTags(filters, tags) {
+		t.Fatal("expected true when no tag: filters present")
+	}
+}
+
+func TestMatchesTags_MatchingSingleTag(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Environment": {"prod"},
+	}
+	tags := map[string]string{"Environment": "prod"}
+	if !MatchesTags(filters, tags) {
+		t.Fatal("expected tag match")
+	}
+}
+
+func TestMatchesTags_NonMatchingTag(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Environment": {"prod"},
+	}
+	tags := map[string]string{"Environment": "staging"}
+	if MatchesTags(filters, tags) {
+		t.Fatal("expected no tag match")
+	}
+}
+
+func TestMatchesTags_MissingTag(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Environment": {"prod"},
+	}
+	tags := map[string]string{"Team": "infra"}
+	if MatchesTags(filters, tags) {
+		t.Fatal("expected no match when tag key is missing")
+	}
+}
+
+func TestMatchesTags_NilTags(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Environment": {"prod"},
+	}
+	if MatchesTags(filters, nil) {
+		t.Fatal("expected no match with nil tags")
+	}
+}
+
+func TestMatchesTags_MultipleTagFiltersAND(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Environment": {"prod"},
+		"tag:Team":        {"infra", "platform"},
+	}
+	// Matches both: Environment=prod AND Team in {infra, platform}
+	tags := map[string]string{"Environment": "prod", "Team": "infra"}
+	if !MatchesTags(filters, tags) {
+		t.Fatal("expected match for both tag filters")
+	}
+
+	// Fails second filter
+	tags2 := map[string]string{"Environment": "prod", "Team": "frontend"}
+	if MatchesTags(filters, tags2) {
+		t.Fatal("expected no match when second tag doesn't match")
+	}
+}
+
+func TestMatchesTags_WildcardValues(t *testing.T) {
+	filters := map[string][]string{
+		"tag:Name": {"prod-*"},
+	}
+	tags := map[string]string{"Name": "prod-web-01"}
+	if !MatchesTags(filters, tags) {
+		t.Fatal("expected wildcard tag match")
+	}
+}
+
+func TestMatchesTags_EmptyFilters(t *testing.T) {
+	if !MatchesTags(nil, nil) {
+		t.Fatal("expected true for nil filters")
+	}
+	if !MatchesTags(map[string][]string{}, nil) {
+		t.Fatal("expected true for empty filters")
+	}
+}
+
+func TestEC2TagsToMap(t *testing.T) {
+	tags := []*ec2.Tag{
+		{Key: aws.String("Env"), Value: aws.String("prod")},
+		{Key: aws.String("Team"), Value: aws.String("infra")},
+	}
+	m := EC2TagsToMap(tags)
+	if m["Env"] != "prod" || m["Team"] != "infra" {
+		t.Fatalf("unexpected map: %v", m)
+	}
+}
+
+func TestEC2TagsToMap_NilAndEmpty(t *testing.T) {
+	if EC2TagsToMap(nil) != nil {
+		t.Fatal("expected nil for nil tags")
+	}
+	if EC2TagsToMap([]*ec2.Tag{}) != nil {
+		t.Fatal("expected nil for empty tags")
+	}
+}
+
+func TestMatchWildcard_OverlappingSuffixNotDoubled(t *testing.T) {
+	// Pattern "a*b*ab" — the suffix "ab" must not be consumed by the
+	// middle "b" match, leaving nothing for the suffix anchor.
+	if !MatchesAny([]string{"a*b*ab"}, "abab") {
+		t.Fatal("expected a*b*ab to match abab")
+	}
+	if MatchesAny([]string{"a*b*ab"}, "ab") {
+		t.Fatal("expected a*b*ab to NOT match ab (suffix overlap)")
+	}
+}
+
+func TestMatchWildcard_TwoPartPattern(t *testing.T) {
+	// Simple two-part: prefix + suffix, no middle parts.
+	if !MatchesAny([]string{"a*a"}, "aa") {
+		t.Fatal("expected a*a to match aa")
+	}
+	if !MatchesAny([]string{"a*a"}, "aXa") {
+		t.Fatal("expected a*a to match aXa")
+	}
+	if MatchesAny([]string{"a*a"}, "a") {
+		t.Fatal("expected a*a to NOT match a")
+	}
+}

spinifex/formation/formation.go

@@ -2,11 +2,11 @@ package formation
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"maps"
-	"net"
 	"net/http"
 	"sync"
 	"time"
@@ -190,8 +190,14 @@ func (fs *FormationServer) SetMasterKey(key string) {
 	fs.masterKey = key
 }
 
-// Start launches the HTTP server on the given address (e.g. "10.0.0.1:4432").
+// Start launches the HTTPS server on the given address (e.g. "10.0.0.1:4432")
+// using the cluster CA certificate and key for TLS.
 func (fs *FormationServer) Start(bindAddr string) error {
+	tlsCert, err := tls.X509KeyPair([]byte(fs.caCert), []byte(fs.caKey))
+	if err != nil {
+		return fmt.Errorf("formation server TLS keypair: %w", err)
+	}
+
 	mux := http.NewServeMux()
 	mux.HandleFunc("POST /formation/join", fs.handleJoin)
 	mux.HandleFunc("GET /formation/status", fs.handleStatus)
@@ -204,9 +210,12 @@ func (fs *FormationServer) Start(bindAddr string) error {
 		WriteTimeout:      10 * time.Second,
 		IdleTimeout:       30 * time.Second,
 		ReadHeaderTimeout: 5 * time.Second,
+		TLSConfig: &tls.Config{
+			Certificates: []tls.Certificate{tlsCert},
+		},
 	}
 
-	ln, err := net.Listen("tcp", bindAddr)
+	ln, err := tls.Listen("tcp", bindAddr, fs.server.TLSConfig)
 	if err != nil {
 		return fmt.Errorf("formation server listen: %w", err)
 	}
@@ -217,7 +226,7 @@ func (fs *FormationServer) Start(bindAddr string) error {
 		}
 	}()
 
-	slog.Info("Formation server started", "addr", bindAddr)
+	slog.Info("Formation server started (TLS)", "addr", bindAddr)
 	return nil
 }
 

spinifex/gateway/auth.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/subtle"
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -21,6 +22,9 @@ import (
 const (
 	// Maximum allowed clock skew for signature validation (5 minutes)
 	maxClockSkew = 5 * time.Minute
+
+	// Maximum request body size for signature validation (10 MB)
+	maxBodySize = 10 * 1024 * 1024
 )
 
 // SigV4AuthMiddleware returns stdlib middleware that validates AWS Signature V4 authentication.
@@ -139,9 +143,18 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 				return
 			}
 
+			// Limit request body size to prevent OOM from unauthenticated requests
+			r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
+
 			// Read body once and re-buffer for downstream handlers
 			body, err := io.ReadAll(r.Body)
 			if err != nil {
+				var maxBytesErr *http.MaxBytesError
+				if errors.As(err, &maxBytesErr) {
+					slog.Warn("Request body too large", "limit", maxBodySize)
+					gw.writeSigV4Error(w, r, awserrors.ErrorRequestEntityTooLarge)
+					return
+				}
 				slog.Error("Failed to read request body", "err", err)
 				gw.writeSigV4Error(w, r, awserrors.ErrorInternalError)
 				return

spinifex/gateway/auth_test.go

@@ -667,6 +667,34 @@ func TestSigV4Auth_DecryptFailure(t *testing.T) {
 	}
 }
 
+func TestSigV4Auth_RequestBodyTooLarge(t *testing.T) {
+	handler := setupTestApp(testAccessKey, testSecretKey)
+
+	// Create a body that exceeds maxBodySize (10 MB + 1 byte)
+	oversizedBody := strings.Repeat("x", maxBodySize+1)
+	authHeader, timestamp := generateTestAuthHeader(
+		"POST", "/", "", oversizedBody,
+		testAccessKey, testSecretKey, testRegion, testService,
+	)
+
+	req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(oversizedBody))
+	req.Host = "localhost:9999"
+	req.Header.Set("Authorization", authHeader)
+	req.Header.Set("X-Amz-Date", timestamp)
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp := doRequest(handler, req)
+
+	if resp.StatusCode != http.StatusRequestEntityTooLarge {
+		t.Errorf("Expected status 413, got %d", resp.StatusCode)
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "RequestEntityTooLarge") {
+		t.Errorf("Expected RequestEntityTooLarge error, got: %s", string(body))
+	}
+}
+
 func TestCanonicalHeaderName(t *testing.T) {
 	testCases := []struct {
 		input    string

spinifex/gateway/ec2.go

@@ -19,7 +19,9 @@ import (
 	gateway_ec2_image "github.com/mulgadc/spinifex/spinifex/gateway/ec2/image"
 	gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance"
 	gateway_ec2_key "github.com/mulgadc/spinifex/spinifex/gateway/ec2/key"
+	gateway_ec2_natgw "github.com/mulgadc/spinifex/spinifex/gateway/ec2/natgw"
 	gateway_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/gateway/ec2/placementgroup"
+	gateway_ec2_routetable "github.com/mulgadc/spinifex/spinifex/gateway/ec2/routetable"
 	gateway_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/gateway/ec2/snapshot"
 	gateway_ec2_tags "github.com/mulgadc/spinifex/spinifex/gateway/ec2/tags"
 	gateway_ec2_volume "github.com/mulgadc/spinifex/spinifex/gateway/ec2/volume"
@@ -82,6 +84,12 @@ var ec2Actions = map[string]EC2Handler{
 	"ModifyInstanceAttribute": ec2Handler(func(input *ec2.ModifyInstanceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_instance.ModifyInstanceAttribute(input, gw.NATSConn, accountID)
 	}),
+	"DescribeInstanceAttribute": ec2Handler(func(input *ec2.DescribeInstanceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_instance.DescribeInstanceAttribute(input, gw.NATSConn, accountID)
+	}),
+	"DescribeInstanceCreditSpecifications": ec2Handler(func(input *ec2.DescribeInstanceCreditSpecificationsInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_instance.DescribeInstanceCreditSpecifications(input)
+	}),
 	"CreateKeyPair": ec2Handler(func(input *ec2.CreateKeyPairInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_key.CreateKeyPair(input, gw.NATSConn, accountID)
 	}),
@@ -217,6 +225,12 @@ var ec2Actions = map[string]EC2Handler{
 	"DescribeVpcs": ec2Handler(func(input *ec2.DescribeVpcsInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.DescribeVpcs(input, gw.NATSConn, accountID)
 	}),
+	"ModifyVpcAttribute": ec2Handler(func(input *ec2.ModifyVpcAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_vpc.ModifyVpcAttribute(input, gw.NATSConn, accountID)
+	}),
+	"DescribeVpcAttribute": ec2Handler(func(input *ec2.DescribeVpcAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_vpc.DescribeVpcAttribute(input, gw.NATSConn, accountID)
+	}),
 	"CreateSubnet": ec2Handler(func(input *ec2.CreateSubnetInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.CreateSubnet(input, gw.NATSConn, accountID)
 	}),
@@ -229,6 +243,33 @@ var ec2Actions = map[string]EC2Handler{
 	"ModifySubnetAttribute": ec2Handler(func(input *ec2.ModifySubnetAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.ModifySubnetAttribute(input, gw.NATSConn, accountID)
 	}),
+	"CreateRouteTable": ec2Handler(func(input *ec2.CreateRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.CreateRouteTable(input, gw.NATSConn, accountID)
+	}),
+	"DeleteRouteTable": ec2Handler(func(input *ec2.DeleteRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.DeleteRouteTable(input, gw.NATSConn, accountID)
+	}),
+	"DescribeRouteTables": ec2Handler(func(input *ec2.DescribeRouteTablesInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.DescribeRouteTables(input, gw.NATSConn, accountID)
+	}),
+	"CreateRoute": ec2Handler(func(input *ec2.CreateRouteInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.CreateRoute(input, gw.NATSConn, accountID)
+	}),
+	"DeleteRoute": ec2Handler(func(input *ec2.DeleteRouteInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.DeleteRoute(input, gw.NATSConn, accountID)
+	}),
+	"ReplaceRoute": ec2Handler(func(input *ec2.ReplaceRouteInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.ReplaceRoute(input, gw.NATSConn, accountID)
+	}),
+	"AssociateRouteTable": ec2Handler(func(input *ec2.AssociateRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.AssociateRouteTable(input, gw.NATSConn, accountID)
+	}),
+	"DisassociateRouteTable": ec2Handler(func(input *ec2.DisassociateRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.DisassociateRouteTable(input, gw.NATSConn, accountID)
+	}),
+	"ReplaceRouteTableAssociation": ec2Handler(func(input *ec2.ReplaceRouteTableAssociationInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_routetable.ReplaceRouteTableAssociation(input, gw.NATSConn, accountID)
+	}),
 	"CreateNetworkInterface": ec2Handler(func(input *ec2.CreateNetworkInterfaceInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.CreateNetworkInterface(input, gw.NATSConn, accountID)
 	}),
@@ -238,6 +279,9 @@ var ec2Actions = map[string]EC2Handler{
 	"DescribeNetworkInterfaces": ec2Handler(func(input *ec2.DescribeNetworkInterfacesInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.DescribeNetworkInterfaces(input, gw.NATSConn, accountID)
 	}),
+	"ModifyNetworkInterfaceAttribute": ec2Handler(func(input *ec2.ModifyNetworkInterfaceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_vpc.ModifyNetworkInterfaceAttribute(input, gw.NATSConn, accountID)
+	}),
 	"CreateSecurityGroup": ec2Handler(func(input *ec2.CreateSecurityGroupInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_vpc.CreateSecurityGroup(input, gw.NATSConn, accountID)
 	}),
@@ -274,6 +318,18 @@ var ec2Actions = map[string]EC2Handler{
 	"DescribeAddresses": ec2Handler(func(input *ec2.DescribeAddressesInput, gw *GatewayConfig, accountID string) (any, error) {
 		return gateway_ec2_eip.DescribeAddresses(input, gw.NATSConn, accountID)
 	}),
+	"DescribeAddressesAttribute": ec2Handler(func(input *ec2.DescribeAddressesAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_eip.DescribeAddressesAttribute(input, gw.NATSConn, accountID)
+	}),
+	"CreateNatGateway": ec2Handler(func(input *ec2.CreateNatGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_natgw.CreateNatGateway(input, gw.NATSConn, accountID)
+	}),
+	"DeleteNatGateway": ec2Handler(func(input *ec2.DeleteNatGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_natgw.DeleteNatGateway(input, gw.NATSConn, accountID)
+	}),
+	"DescribeNatGateways": ec2Handler(func(input *ec2.DescribeNatGatewaysInput, gw *GatewayConfig, accountID string) (any, error) {
+		return gateway_ec2_natgw.DescribeNatGateways(input, gw.NATSConn, accountID)
+	}),
 }
 
 // ec2LocalActions are actions that don't require a NATS connection.

spinifex/gateway/ec2/eip/DescribeAddressesAttribute.go

@@ -0,0 +1,27 @@
+package gateway_ec2_eip
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
+	"github.com/nats-io/nats.go"
+)
+
+// DescribeAddressesAttribute handles the EC2 DescribeAddressesAttribute API call.
+func DescribeAddressesAttribute(input *ec2.DescribeAddressesAttributeInput, natsConn *nats.Conn, accountID string) (ec2.DescribeAddressesAttributeOutput, error) {
+	var output ec2.DescribeAddressesAttributeOutput
+
+	if input == nil {
+		return output, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	svc := handlers_ec2_eip.NewNATSEIPService(natsConn)
+	result, err := svc.DescribeAddressesAttribute(input, accountID)
+	if err != nil {
+		return output, err
+	}
+
+	return *result, nil
+}

spinifex/gateway/ec2/eip/eip_test.go

@@ -0,0 +1,114 @@
+package gateway_ec2_eip
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+)
+
+const testAccountID = "123456789012"
+
+// AllocateAddress tests
+
+func TestAllocateAddress_NilInput(t *testing.T) {
+	_, err := AllocateAddress(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestAllocateAddress_NilNATS(t *testing.T) {
+	_, err := AllocateAddress(&ec2.AllocateAddressInput{}, nil, testAccountID)
+	assert.Error(t, err)
+}
+
+// AssociateAddress tests
+
+func TestAssociateAddress_NilInput(t *testing.T) {
+	_, err := AssociateAddress(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestAssociateAddress_NilAllocationId(t *testing.T) {
+	_, err := AssociateAddress(&ec2.AssociateAddressInput{}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestAssociateAddress_EmptyAllocationId(t *testing.T) {
+	_, err := AssociateAddress(&ec2.AssociateAddressInput{AllocationId: aws.String("")}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestAssociateAddress_NilNATS(t *testing.T) {
+	_, err := AssociateAddress(&ec2.AssociateAddressInput{AllocationId: aws.String("eipalloc-123")}, nil, testAccountID)
+	assert.Error(t, err)
+}
+
+// DescribeAddresses tests
+
+func TestDescribeAddresses_NilInput(t *testing.T) {
+	_, err := DescribeAddresses(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestDescribeAddresses_NilNATS(t *testing.T) {
+	_, err := DescribeAddresses(&ec2.DescribeAddressesInput{}, nil, testAccountID)
+	assert.Error(t, err)
+}
+
+// DescribeAddressesAttribute tests
+
+func TestDescribeAddressesAttribute_NilInput(t *testing.T) {
+	_, err := DescribeAddressesAttribute(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestDescribeAddressesAttribute_NilNATS(t *testing.T) {
+	_, err := DescribeAddressesAttribute(&ec2.DescribeAddressesAttributeInput{}, nil, testAccountID)
+	assert.Error(t, err)
+}
+
+// DisassociateAddress tests
+
+func TestDisassociateAddress_NilInput(t *testing.T) {
+	_, err := DisassociateAddress(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestDisassociateAddress_NilAssociationId(t *testing.T) {
+	_, err := DisassociateAddress(&ec2.DisassociateAddressInput{}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestDisassociateAddress_EmptyAssociationId(t *testing.T) {
+	_, err := DisassociateAddress(&ec2.DisassociateAddressInput{AssociationId: aws.String("")}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestDisassociateAddress_NilNATS(t *testing.T) {
+	_, err := DisassociateAddress(&ec2.DisassociateAddressInput{AssociationId: aws.String("eipassoc-123")}, nil, testAccountID)
+	assert.Error(t, err)
+}
+
+// ReleaseAddress tests
+
+func TestReleaseAddress_NilInput(t *testing.T) {
+	_, err := ReleaseAddress(nil, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}
+
+func TestReleaseAddress_NilAllocationId(t *testing.T) {
+	_, err := ReleaseAddress(&ec2.ReleaseAddressInput{}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestReleaseAddress_EmptyAllocationId(t *testing.T) {
+	_, err := ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String("")}, nil, testAccountID)
+	assert.EqualError(t, err, awserrors.ErrorMissingParameter)
+}
+
+func TestReleaseAddress_NilNATS(t *testing.T) {
+	_, err := ReleaseAddress(&ec2.ReleaseAddressInput{AllocationId: aws.String("eipalloc-123")}, nil, testAccountID)
+	assert.Error(t, err)
+}

spinifex/gateway/ec2/instance/DescribeInstanceAttribute.go

@@ -0,0 +1,47 @@
+package gateway_ec2_instance
+
+import (
+	"errors"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+)
+
+// ValidateDescribeInstanceAttributeInput validates the input for DescribeInstanceAttribute.
+func ValidateDescribeInstanceAttributeInput(input *ec2.DescribeInstanceAttributeInput) error {
+	if input == nil {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if input.InstanceId == nil || *input.InstanceId == "" {
+		return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+	}
+	if !strings.HasPrefix(*input.InstanceId, "i-") {
+		return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+	}
+	if input.Attribute == nil || *input.Attribute == "" {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+	return nil
+}
+
+// DescribeInstanceAttribute sends a describe-attribute request to the daemon via NATS.
+func DescribeInstanceAttribute(input *ec2.DescribeInstanceAttributeInput, natsConn *nats.Conn, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
+	if err := ValidateDescribeInstanceAttributeInput(input); err != nil {
+		return nil, err
+	}
+
+	slog.Info("DescribeInstanceAttribute: Processing request", "instance_id", *input.InstanceId, "attribute", *input.Attribute)
+
+	output, err := utils.NATSRequest[ec2.DescribeInstanceAttributeOutput](natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("DescribeInstanceAttribute: Completed successfully", "instance_id", *input.InstanceId)
+	return output, nil
+}

spinifex/gateway/ec2/instance/DescribeInstanceAttribute_test.go

@@ -0,0 +1,129 @@
+package gateway_ec2_instance
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// --- Validation tests ---
+
+func TestValidateDescribeInstanceAttributeInput_NilInput(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(nil)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_MissingInstanceId(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		Attribute: aws.String(ec2.InstanceAttributeNameInstanceType),
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_EmptyInstanceId(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(""),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_BadPrefix(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("x-12345"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_MissingAttribute(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-abc123"),
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_EmptyAttribute(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-abc123"),
+		Attribute:  aws.String(""),
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
+}
+
+func TestValidateDescribeInstanceAttributeInput_Valid(t *testing.T) {
+	err := ValidateDescribeInstanceAttributeInput(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-abc123"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	})
+	assert.NoError(t, err)
+}
+
+// --- Gateway function tests ---
+
+func TestDescribeInstanceAttribute_Success(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", func(msg *nats.Msg) {
+		var input ec2.DescribeInstanceAttributeInput
+		err := json.Unmarshal(msg.Data, &input)
+		require.NoError(t, err)
+		assert.Equal(t, "i-test123", *input.InstanceId)
+		assert.Equal(t, "instanceType", *input.Attribute)
+
+		output := &ec2.DescribeInstanceAttributeOutput{
+			InstanceId:   aws.String("i-test123"),
+			InstanceType: &ec2.AttributeValue{Value: aws.String("t3.micro")},
+		}
+		resp, _ := json.Marshal(output)
+		msg.Respond(resp)
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-test123"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+
+	output, err := DescribeInstanceAttribute(input, nc, "123456789012")
+	require.NoError(t, err)
+	require.NotNil(t, output)
+	assert.Equal(t, "i-test123", *output.InstanceId)
+	assert.Equal(t, "t3.micro", *output.InstanceType.Value)
+}
+
+func TestDescribeInstanceAttribute_DaemonError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.QueueSubscribe("ec2.DescribeInstanceAttribute", "spinifex-workers", func(msg *nats.Msg) {
+		msg.Respond([]byte(`{"Code":"InvalidInstanceID.NotFound"}`))
+	})
+
+	input := &ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String("i-notfound"),
+		Attribute:  aws.String(ec2.InstanceAttributeNameInstanceType),
+	}
+
+	_, err := DescribeInstanceAttribute(input, nc, "123456789012")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+}
+
+func TestDescribeInstanceAttribute_ValidationFailure(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	_, err := DescribeInstanceAttribute(nil, nc, "123456789012")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}

spinifex/gateway/ec2/instance/DescribeInstanceCreditSpecifications.go

@@ -0,0 +1,31 @@
+package gateway_ec2_instance
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// DescribeInstanceCreditSpecifications returns CPU credit specifications for T-series instances.
+// Stub: always returns "standard" mode for each requested instance.
+func DescribeInstanceCreditSpecifications(input *ec2.DescribeInstanceCreditSpecificationsInput) (*ec2.DescribeInstanceCreditSpecificationsOutput, error) {
+	if input == nil {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+
+	var specs []*ec2.InstanceCreditSpecification
+	for _, id := range input.InstanceIds {
+		if id != nil && *id != "" {
+			specs = append(specs, &ec2.InstanceCreditSpecification{
+				InstanceId: id,
+				CpuCredits: aws.String("standard"),
+			})
+		}
+	}
+
+	return &ec2.DescribeInstanceCreditSpecificationsOutput{
+		InstanceCreditSpecifications: specs,
+	}, nil
+}

spinifex/gateway/ec2/instance/DescribeInstanceCreditSpecifications_test.go

@@ -0,0 +1,55 @@
+package gateway_ec2_instance
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDescribeInstanceCreditSpecifications_NilInput(t *testing.T) {
+	out, err := DescribeInstanceCreditSpecifications(nil)
+	require.Error(t, err)
+	assert.Nil(t, out)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}
+
+func TestDescribeInstanceCreditSpecifications_EmptyInstanceIds(t *testing.T) {
+	out, err := DescribeInstanceCreditSpecifications(&ec2.DescribeInstanceCreditSpecificationsInput{})
+	require.NoError(t, err)
+	assert.Empty(t, out.InstanceCreditSpecifications)
+}
+
+func TestDescribeInstanceCreditSpecifications_SingleInstance(t *testing.T) {
+	out, err := DescribeInstanceCreditSpecifications(&ec2.DescribeInstanceCreditSpecificationsInput{
+		InstanceIds: []*string{aws.String("i-abc123")},
+	})
+	require.NoError(t, err)
+	require.Len(t, out.InstanceCreditSpecifications, 1)
+	assert.Equal(t, "i-abc123", *out.InstanceCreditSpecifications[0].InstanceId)
+	assert.Equal(t, "standard", *out.InstanceCreditSpecifications[0].CpuCredits)
+}
+
+func TestDescribeInstanceCreditSpecifications_MultipleInstances(t *testing.T) {
+	out, err := DescribeInstanceCreditSpecifications(&ec2.DescribeInstanceCreditSpecificationsInput{
+		InstanceIds: []*string{aws.String("i-aaa"), aws.String("i-bbb"), aws.String("i-ccc")},
+	})
+	require.NoError(t, err)
+	require.Len(t, out.InstanceCreditSpecifications, 3)
+	for _, spec := range out.InstanceCreditSpecifications {
+		assert.Equal(t, "standard", *spec.CpuCredits)
+	}
+}
+
+func TestDescribeInstanceCreditSpecifications_SkipsNilAndEmpty(t *testing.T) {
+	out, err := DescribeInstanceCreditSpecifications(&ec2.DescribeInstanceCreditSpecificationsInput{
+		InstanceIds: []*string{nil, aws.String(""), aws.String("i-valid"), nil, aws.String("i-also")},
+	})
+	require.NoError(t, err)
+	require.Len(t, out.InstanceCreditSpecifications, 2)
+	assert.Equal(t, "i-valid", *out.InstanceCreditSpecifications[0].InstanceId)
+	assert.Equal(t, "i-also", *out.InstanceCreditSpecifications[1].InstanceId)
+}

spinifex/gateway/ec2/instance/DescribeInstanceTypes.go

@@ -113,6 +113,14 @@ func DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, natsConn *nats
 		}
 	}
 
+	// Build set of requested instance type names (if any) for filtering.
+	requestedTypes := make(map[string]bool)
+	for _, it := range input.InstanceTypes {
+		if it != nil {
+			requestedTypes[*it] = true
+		}
+	}
+
 	var finalInstanceTypes []*ec2.InstanceTypeInfo
 	if showCapacity {
 		finalInstanceTypes = allInstanceTypes
@@ -128,6 +136,17 @@ func DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, natsConn *nats
 		}
 	}
 
+	// Filter by requested instance type names if specified.
+	if len(requestedTypes) > 0 {
+		var filtered []*ec2.InstanceTypeInfo
+		for _, it := range finalInstanceTypes {
+			if it != nil && it.InstanceType != nil && requestedTypes[*it.InstanceType] {
+				filtered = append(filtered, it)
+			}
+		}
+		finalInstanceTypes = filtered
+	}
+
 	// Build final aggregated response
 	output := &ec2.DescribeInstanceTypesOutput{
 		InstanceTypes: finalInstanceTypes,

spinifex/gateway/ec2/instance/DescribeInstances.go

@@ -2,12 +2,14 @@ package gateway_ec2_instance
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -48,6 +50,7 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 	deadline := time.Now().Add(timeout)
 
 	var allReservations []*ec2.Reservation
+	var clientError string // first client error code from any node (e.g. InvalidParameterValue)
 	responsesReceived := 0
 
 	// If expectedNodes is not configured (0), fall back to timeout-based collection
@@ -86,8 +89,19 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 		// Check if response is an error
 		responseError, err := utils.ValidateErrorPayload(msg.Data)
 		if err != nil {
-			// Response is an error payload - log but continue collecting from other nodes
-			slog.Warn("DescribeInstances: Received error from node", "code", responseError.Code, "responses_received", responsesReceived)
+			code := ""
+			if responseError.Code != nil {
+				code = *responseError.Code
+			}
+			// Capture the first client error (e.g. InvalidParameterValue). Client errors
+			// are deterministic — all nodes return the same error for the same invalid
+			// request — so we propagate them to the caller after collection completes.
+			if clientError == "" && code != "" {
+				if info, known := awserrors.ErrorLookup[code]; known && info.HTTPCode >= 400 && info.HTTPCode < 500 {
+					clientError = code
+				}
+			}
+			slog.Warn("DescribeInstances: Received error from node", "code", code, "responses_received", responsesReceived)
 			continue
 		}
 
@@ -123,6 +137,12 @@ func DescribeInstances(input *ec2.DescribeInstancesInput, natsConn *nats.Conn, e
 	}
 	kvWg.Wait()
 
+	// If every node returned a client error and we collected no data, propagate
+	// the error to the caller so the HTTP response carries the correct status.
+	if clientError != "" && len(allReservations) == 0 {
+		return nil, errors.New(clientError)
+	}
+
 	// Build final aggregated response
 	output := &ec2.DescribeInstancesOutput{
 		Reservations: allReservations,

spinifex/gateway/ec2/instance/ModifyInstanceAttribute.go

@@ -35,6 +35,9 @@ func ValidateModifyInstanceAttributeInput(input *ec2.ModifyInstanceAttributeInpu
 	if input.UserData != nil {
 		count++
 	}
+	if input.SourceDestCheck != nil {
+		count++
+	}
 	if count != 1 {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}

spinifex/gateway/ec2/instance/ModifyInstanceAttribute_test.go

@@ -95,6 +95,24 @@ func TestValidateModifyInstanceAttributeInput_ValidUserData(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestValidateModifyInstanceAttributeInput_ValidSourceDestCheck(t *testing.T) {
+	err := ValidateModifyInstanceAttributeInput(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:      aws.String("i-abc123"),
+		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+	})
+	assert.NoError(t, err)
+}
+
+func TestValidateModifyInstanceAttributeInput_SourceDestCheckWithOtherAttribute(t *testing.T) {
+	err := ValidateModifyInstanceAttributeInput(&ec2.ModifyInstanceAttributeInput{
+		InstanceId:      aws.String("i-abc123"),
+		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(false)},
+		InstanceType:    &ec2.AttributeValue{Value: aws.String("t3.micro")},
+	})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+}
+
 // --- Gateway function tests ---
 
 func TestModifyInstanceAttribute_Success(t *testing.T) {
@@ -117,6 +135,22 @@ func TestModifyInstanceAttribute_Success(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestModifyInstanceAttribute_SourceDestCheck(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+
+	nc.QueueSubscribe("ec2.ModifyInstanceAttribute", "spinifex-workers", func(msg *nats.Msg) {
+		msg.Respond([]byte(`{}`))
+	})
+
+	input := &ec2.ModifyInstanceAttributeInput{
+		InstanceId:      aws.String("i-test123"),
+		SourceDestCheck: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
+	}
+
+	_, err := ModifyInstanceAttribute(input, nc, "123456789012")
+	assert.NoError(t, err)
+}
+
 func TestModifyInstanceAttribute_DaemonError(t *testing.T) {
 	_, nc := startTestNATSServer(t)
 

spinifex/gateway/ec2/instance/helpers_test.go

@@ -2,41 +2,14 @@ package gateway_ec2_instance
 
 import (
 	"testing"
-	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/require"
 )
 
 // startTestNATSServer starts an embedded NATS server for testing
 func startTestNATSServer(t *testing.T) (*server.Server, *nats.Conn) {
 	t.Helper()
-
-	opts := &server.Options{
-		Host:      "127.0.0.1",
-		Port:      -1,
-		JetStream: false,
-		NoLog:     true,
-		NoSigs:    true,
-	}
-
-	ns, err := server.NewServer(opts)
-	require.NoError(t, err, "Failed to create NATS server")
-
-	go ns.Start()
-
-	if !ns.ReadyForConnections(5 * time.Second) {
-		t.Fatal("NATS server failed to start")
-	}
-
-	nc, err := nats.Connect(ns.ClientURL())
-	require.NoError(t, err, "Failed to connect to NATS")
-
-	t.Cleanup(func() {
-		nc.Close()
-		ns.Shutdown()
-	})
-
-	return ns, nc
+	return testutil.StartTestNATS(t)
 }