chore: use ts config files

Replace fragile image-name substring match (-lb-/-alb-) with a
DescribeImages tag filter on spinifex:managed-by=elbv2. Remove the
silent fallback that picks the first image in the store when no LB
image exists — CreateLoadBalancer now returns ServerInternal with a
remediation hint instead of persisting a record in StateFailed.

- daemon: tag filter + error return from the resolver closure
- service_impl: resolver signature is func() (string, error); caches
  only successful results so the next call retries after an error
- CreateLoadBalancer gate now requires launcher + networking; AMI is
  resolved inside the gate and failures short-circuit before persist
- admin images import: add --name+--file combo (catalog metadata with
  a local file source) and a --tag key=value flag for explicit tagging
  of file-only imports
- build-system-image.sh + lb.conf: propagate SYSTEM_TAG via --tag so
  locally built system AMIs get the right tag even when the catalog
  is bypassed

Closes mulga-969

.githooks/pre-commit

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+branch=$(git rev-parse --abbrev-ref HEAD)
+
+# Only enforce preflight on dev and main
+if [ "$branch" != "dev" ] && [ "$branch" != "main" ]; then
+  exit 0
+fi
+
+# Only run preflight if Go files were staged
+if git diff --cached --name-only | grep -q '\.go$'; then
+  echo "Go files staged on $branch — running make preflight..."
+  make preflight
+else
+  echo "No Go files staged, skipping preflight."
+fi

.github/assets/architecture.svg

@@ -0,0 +1,225 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="620" viewBox="0 0 900 620">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#161b22"/>
+      <stop offset="45%" style="stop-color:#1f2028"/>
+      <stop offset="100%" style="stop-color:#161b22"/>
+    </linearGradient>
+    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262633"/>
+      <stop offset="100%" style="stop-color:#7F1D1D14"/>
+    </linearGradient>
+    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#F87171"/>
+      <stop offset="50%" style="stop-color:#DC2626"/>
+      <stop offset="100%" style="stop-color:#7F1D1D"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="620" fill="url(#bg)" rx="12"/>
+
+  <!-- Title bar -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ARCHITECTURE</text>
+  <text x="24" y="52" font-family="'Segoe UI', system-ui, sans-serif" font-size="16" fill="#f5f5f5" font-weight="500">AWS SDK → Gateway → NATS → Service daemons</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">stateless · message-driven · horizontally scalable</text>
+
+  <!-- ═══ CLIENT LAYER ════════════════════════════════════ -->
+  <rect x="16" y="72" width="868" height="78" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
+  <text x="32" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700" letter-spacing="1">CLIENTS</text>
+  <text x="32" y="108" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Standard AWS tooling — unmodified, no shims, no custom SDK</text>
+
+  <g transform="translate(32, 120)" font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <rect x="0" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="43" y="15" fill="#7ee787" text-anchor="middle">aws-cli</text>
+
+    <rect x="94" y="0" width="96" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="142" y="15" fill="#7ee787" text-anchor="middle">Terraform</text>
+
+    <rect x="198" y="0" width="76" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="236" y="15" fill="#7ee787" text-anchor="middle">boto3</text>
+
+    <rect x="282" y="0" width="98" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="331" y="15" fill="#7ee787" text-anchor="middle">aws-sdk-go</text>
+
+    <rect x="388" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="431" y="15" fill="#7ee787" text-anchor="middle">kubectl</text>
+
+    <rect x="482" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="525" y="15" fill="#7ee787" text-anchor="middle">Packer</text>
+
+    <rect x="576" y="0" width="120" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="636" y="15" fill="#7ee787" text-anchor="middle">your application</text>
+  </g>
+
+  <!-- ═══ HTTPS + SigV4 connector ═══════════════════════════ -->
+  <line x1="450" y1="150" x2="450" y2="178" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="390" y="154" width="120" height="18" rx="9" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="166" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#F87171" text-anchor="middle">HTTPS · SigV4 · :9999</text>
+
+  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
+  <rect x="16" y="178" width="868" height="68" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
+  <text x="32" y="200" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY</text>
+  <text x="32" y="215" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
+
+  <g transform="translate(32, 224)" font-family="'SF Mono', ui-monospace, monospace" font-size="9">
+    <rect x="0" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="39" y="11" fill="#FCA5A5" text-anchor="middle">TLS 1.3</text>
+
+    <rect x="86" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="125" y="11" fill="#FCA5A5" text-anchor="middle">SigV4 auth</text>
+
+    <rect x="172" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="221" y="11" fill="#FCA5A5" text-anchor="middle">policy eval</text>
+
+    <rect x="278" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="327" y="11" fill="#FCA5A5" text-anchor="middle">req routing</text>
+
+    <rect x="384" y="0" width="88" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="428" y="11" fill="#FCA5A5" text-anchor="middle">rate-limit</text>
+
+    <rect x="480" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="529" y="11" fill="#FCA5A5" text-anchor="middle">audit log</text>
+  </g>
+
+  <!-- ═══ NATS BUS ═══════════════════════════════════════ -->
+  <line x1="450" y1="246" x2="450" y2="272" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="16" y="272" width="868" height="34" rx="6" fill="url(#nats-grad)" opacity="0.92"/>
+  <text x="32" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS BUS</text>
+  <text x="140" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#fee2e2" opacity="0.95">ec2.*   ebs.*   s3.*   vpc.*   iam.*</text>
+  <text x="876" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">queue groups · fan-out · request/reply</text>
+
+  <!-- ═══ DAEMON FANOUT CONNECTORS ═══════════════════════ -->
+  <g stroke="#DC2626" stroke-width="0.9" stroke-dasharray="3,3" opacity="0.55" fill="none">
+    <line x1="100" y1="306" x2="100" y2="328"/>
+    <line x1="275" y1="306" x2="275" y2="328"/>
+    <line x1="450" y1="306" x2="450" y2="328"/>
+    <line x1="625" y1="306" x2="625" y2="328"/>
+    <line x1="800" y1="306" x2="800" y2="328"/>
+  </g>
+
+  <!-- ═══ SERVICE DAEMONS ROW ═══════════════════════════ -->
+  <!-- EC2 -->
+  <g transform="translate(28, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ec2.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">RunInstances</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">TerminateInstances</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">QMP · cloud-init</text>
+  </g>
+
+  <!-- EBS -->
+  <g transform="translate(204, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ebs.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">CreateVolume</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">AttachVolume</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">snapshots · WAL</text>
+  </g>
+
+  <!-- S3 -->
+  <g transform="translate(380, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">s3.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">PutObject · GetObject</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Multipart · presigned</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">erasure-coded</text>
+  </g>
+
+  <!-- VPC -->
+  <g transform="translate(556, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">vpc.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Subnets · SGs · RTs</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">ALB · NAT · IGW</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">OVN · OVS</text>
+  </g>
+
+  <!-- IAM -->
+  <g transform="translate(732, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">iam.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Users · Roles · Keys</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Policy evaluation</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">audit trail</text>
+  </g>
+
+  <!-- ═══ DAEMON → BACKEND CONNECTORS ═══════════════════ -->
+  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
+    <line x1="100" y1="410" x2="100" y2="440"/>
+    <line x1="276" y1="410" x2="276" y2="440"/>
+    <line x1="452" y1="410" x2="452" y2="440"/>
+    <line x1="628" y1="410" x2="628" y2="440"/>
+    <line x1="804" y1="410" x2="804" y2="440"/>
+  </g>
+
+  <!-- ═══ BACKEND RESOURCES BAR ═════════════════════════ -->
+  <rect x="16" y="440" width="868" height="26" rx="5" fill="url(#bar-grad)"/>
+  <text x="450" y="458" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">BACKEND RESOURCES · local hardware · no cloud dependency</text>
+
+  <!-- ═══ BACKEND ROW ═══════════════════════════════════ -->
+  <!-- QEMU/KVM -->
+  <g transform="translate(28, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">hypervisor</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">host kernel · virtio</text>
+  </g>
+
+  <!-- Viperblock -->
+  <g transform="translate(204, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Viperblock</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">block storage</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">NVMe WAL · NBD</text>
+  </g>
+
+  <!-- Predastore -->
+  <g transform="translate(380, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Predastore</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">object storage</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">Reed–Solomon</text>
+  </g>
+
+  <!-- OVN/OVS -->
+  <g transform="translate(556, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">OVN / OVS</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SDN fabric</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">overlays · flow rules</text>
+  </g>
+
+  <!-- IAM store -->
+  <g transform="translate(732, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Identity store</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">users · keys · rules</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">local · audit-logged</text>
+  </g>
+
+  <!-- ═══ FOOTER NOTES ══════════════════════════════════ -->
+  <g transform="translate(16, 558)">
+    <rect width="868" height="48" rx="6" fill="#0a0a0a" stroke="#27272a" stroke-width="0.8"/>
+    <text x="24" y="20" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
+      <tspan fill="#F87171" font-weight="600">Queue groups</tspan> load-balance ops across daemon replicas.
+      <tspan fill="#F87171" font-weight="600"> Fan-out</tspan> (no queue group) dispatches describe calls to every node.
+    </text>
+    <text x="24" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
+      Daemons are <tspan fill="#F87171" font-weight="600">stateless</tspan> — scale horizontally by starting more.
+      No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.
+    </text>
+  </g>
+</svg>

.github/assets/banner.svg

@@ -0,0 +1,154 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="300" viewBox="0 0 900 300">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a;stop-opacity:1" />
+      <stop offset="50%" style="stop-color:#0d1117;stop-opacity:1" />
+      <stop offset="100%" style="stop-color:#0a0a0a;stop-opacity:1" />
+    </linearGradient>
+    <radialGradient id="vignette" cx="50%" cy="50%" r="70%">
+      <stop offset="0%" style="stop-color:#7F1D1D;stop-opacity:0.08"/>
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0"/>
+    </radialGradient>
+    <linearGradient id="mulga-grad" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.95" />
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.95" />
+    </linearGradient>
+    <linearGradient id="rule" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0" />
+      <stop offset="50%" style="stop-color:#F87171;stop-opacity:0.45" />
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0" />
+    </linearGradient>
+    <filter id="node-glow">
+      <feGaussianBlur in="SourceGraphic" stdDeviation="3" />
+    </filter>
+  </defs>
+
+  <!-- Background -->
+  <rect width="900" height="300" fill="url(#bg)" rx="12"/>
+  <rect width="900" height="300" fill="url(#vignette)" rx="12"/>
+
+  <!-- Mesh network background (mulgadc style) -->
+  <g opacity="0.35">
+    <!-- Connections (drawn first, under nodes) -->
+    <g stroke="#7F1D1D" stroke-width="0.6" fill="none">
+      <line x1="80" y1="60" x2="200" y2="40"/>
+      <line x1="80" y1="60" x2="150" y2="140"/>
+      <line x1="200" y1="40" x2="330" y2="90"/>
+      <line x1="150" y1="140" x2="260" y2="180"/>
+      <line x1="150" y1="140" x2="60" y2="220"/>
+      <line x1="330" y1="90" x2="260" y2="180"/>
+      <line x1="260" y1="180" x2="120" y2="260"/>
+      <line x1="60" y1="220" x2="120" y2="260"/>
+
+      <line x1="620" y1="50" x2="760" y2="90"/>
+      <line x1="620" y1="50" x2="700" y2="160"/>
+      <line x1="760" y1="90" x2="850" y2="160"/>
+      <line x1="700" y1="160" x2="820" y2="220"/>
+      <line x1="850" y1="160" x2="820" y2="220"/>
+      <line x1="700" y1="160" x2="600" y2="240"/>
+      <line x1="600" y1="240" x2="820" y2="220"/>
+      <line x1="760" y1="90" x2="700" y2="160"/>
+    </g>
+    <!-- Nodes -->
+    <g fill="#DC2626">
+      <circle cx="80" cy="60" r="2.5"/>
+      <circle cx="200" cy="40" r="2"/>
+      <circle cx="330" cy="90" r="2.5"/>
+      <circle cx="150" cy="140" r="3"/>
+      <circle cx="260" cy="180" r="2"/>
+      <circle cx="60" cy="220" r="2"/>
+      <circle cx="120" cy="260" r="2.5"/>
+
+      <circle cx="620" cy="50" r="2"/>
+      <circle cx="760" cy="90" r="3"/>
+      <circle cx="700" cy="160" r="2.5"/>
+      <circle cx="850" cy="160" r="2"/>
+      <circle cx="820" cy="220" r="2.5"/>
+      <circle cx="600" cy="240" r="2"/>
+    </g>
+    <!-- Pulse rings (static "about-to-pulse" look) -->
+    <g fill="none" stroke="#F87171" stroke-width="0.8" opacity="0.5">
+      <circle cx="150" cy="140" r="6"/>
+      <circle cx="760" cy="90" r="6"/>
+      <circle cx="700" cy="160" r="8"/>
+    </g>
+  </g>
+
+  <!-- Main emblem: cluster-node stack with NATS bus -->
+  <g transform="translate(200, 150)">
+    <!-- Outer glow ring -->
+    <circle cx="0" cy="0" r="64" fill="none" stroke="url(#mulga-grad)" stroke-width="1.5" opacity="0.35" filter="url(#node-glow)"/>
+
+    <!-- Top node -->
+    <rect x="-24" y="-52" width="48" height="22" rx="3" fill="none" stroke="#DC2626" stroke-width="1.8"/>
+    <circle cx="-16" cy="-41" r="1.6" fill="#F87171"/>
+    <circle cx="-8" cy="-41" r="1.6" fill="#F87171"/>
+    <line x1="0" y1="-41" x2="16" y2="-41" stroke="#F87171" stroke-width="0.9" opacity="0.6"/>
+
+    <!-- Middle node (active) -->
+    <rect x="-30" y="-14" width="60" height="28" rx="3" fill="url(#mulga-grad)" opacity="0.22" stroke="#DC2626" stroke-width="2"/>
+    <circle cx="-20" cy="0" r="2" fill="#F87171"/>
+    <circle cx="-10" cy="0" r="2" fill="#F87171"/>
+    <line x1="-2" y1="0" x2="22" y2="0" stroke="#F87171" stroke-width="1.2" opacity="0.8"/>
+
+    <!-- Bottom node -->
+    <rect x="-24" y="30" width="48" height="22" rx="3" fill="none" stroke="#7F1D1D" stroke-width="1.8"/>
+    <circle cx="-16" cy="41" r="1.6" fill="#DC2626"/>
+    <circle cx="-8" cy="41" r="1.6" fill="#DC2626"/>
+    <line x1="0" y1="41" x2="16" y2="41" stroke="#DC2626" stroke-width="0.9" opacity="0.6"/>
+
+    <!-- Radiating mesh ports -->
+    <g stroke-width="0.9" opacity="0.55">
+      <line x1="-30" y1="0" x2="-58" y2="-26" stroke="#DC2626"/>
+      <line x1="-30" y1="0" x2="-64" y2="0" stroke="#DC2626"/>
+      <line x1="-30" y1="0" x2="-58" y2="26" stroke="#DC2626"/>
+      <line x1="30" y1="0" x2="58" y2="-26" stroke="#7F1D1D"/>
+      <line x1="30" y1="0" x2="64" y2="0" stroke="#7F1D1D"/>
+      <line x1="30" y1="0" x2="58" y2="26" stroke="#7F1D1D"/>
+    </g>
+    <g fill="#F87171">
+      <circle cx="-58" cy="-26" r="2.5"/>
+      <circle cx="-64" cy="0" r="2.5"/>
+      <circle cx="-58" cy="26" r="2.5"/>
+    </g>
+    <g fill="#DC2626">
+      <circle cx="58" cy="-26" r="2.5"/>
+      <circle cx="64" cy="0" r="2.5"/>
+      <circle cx="58" cy="26" r="2.5"/>
+    </g>
+  </g>
+
+  <!-- Title -->
+  <text x="340" y="120" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="56" font-weight="700" fill="#f5f5f5" letter-spacing="-1.5">Spinifex</text>
+
+  <!-- Subtitle -->
+  <text x="340" y="155" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="17" fill="#a1a1aa" letter-spacing="0.3">Open-source AWS-compatible platform</text>
+  <text x="340" y="180" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="17" fill="#a1a1aa" letter-spacing="0.3">for secure edge deployments</text>
+
+  <!-- Rule -->
+  <rect x="340" y="200" width="440" height="1" fill="url(#rule)" rx="1"/>
+
+  <!-- Service pills -->
+  <g font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="11" font-weight="500">
+    <rect x="340" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="365" y="231" fill="#F87171" text-anchor="middle">EC2</text>
+
+    <rect x="398" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="423" y="231" fill="#F87171" text-anchor="middle">EBS</text>
+
+    <rect x="456" y="215" width="45" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="478" y="231" fill="#F87171" text-anchor="middle">S3</text>
+
+    <rect x="509" y="215" width="55" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="536" y="231" fill="#F87171" text-anchor="middle">VPC</text>
+
+    <rect x="572" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="597" y="231" fill="#F87171" text-anchor="middle">IAM</text>
+
+    <rect x="630" y="215" width="78" height="24" rx="12" fill="#18181b" stroke="#52525b" stroke-width="0.5" stroke-opacity="0.8"/>
+    <text x="669" y="231" fill="#a1a1aa" text-anchor="middle">Air-Gapped</text>
+  </g>
+
+  <!-- Tagline -->
+  <text x="340" y="265" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="11" fill="#71717a" letter-spacing="1.8">YOUR INFRASTRUCTURE · YOUR HARDWARE · YOUR CLOUD</text>
+</svg>

.github/assets/design-architecture.svg

@@ -0,0 +1,175 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="720" viewBox="0 0 900 720">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#161b22"/>
+      <stop offset="45%" style="stop-color:#1f2028"/>
+      <stop offset="100%" style="stop-color:#161b22"/>
+    </linearGradient>
+    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262633"/>
+      <stop offset="100%" style="stop-color:#7F1D1D14"/>
+    </linearGradient>
+    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#F87171"/>
+      <stop offset="50%" style="stop-color:#DC2626"/>
+      <stop offset="100%" style="stop-color:#7F1D1D"/>
+    </linearGradient>
+    <linearGradient id="daemon-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D0a"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="720" fill="url(#bg)" rx="12"/>
+
+  <!-- Title -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">HIGH-LEVEL ARCHITECTURE</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">SDK → Gateway → NATS → Daemons → VMs</text>
+
+  <!-- ═══ AWS SDK / CLI ══════════════════════════════════ -->
+  <rect x="16" y="50" width="868" height="76" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
+  <text x="32" y="72" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700" letter-spacing="1">AWS SDK / CLI</text>
+  <text x="32" y="90" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Users interact with Spinifex using standard AWS SDKs and CLIs against a custom endpoint URL</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <rect x="32" y="98" width="480" height="20" rx="4" fill="#0a0a0a" stroke="#3fb950" stroke-width="0.6" stroke-opacity="0.4"/>
+    <text x="42" y="112" fill="#7ee787">$ aws ec2 run-instances --endpoint-url https://localhost:9999 ...</text>
+  </g>
+
+  <!-- Connector: HTTPS + SigV4 -->
+  <line x1="450" y1="126" x2="450" y2="154" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="380" y="128" width="140" height="20" rx="10" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="142" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" text-anchor="middle">HTTPS · SigV4 Auth</text>
+
+  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
+  <rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
+  <text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
+  <text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
+
+  <!-- Two-column feature bullets -->
+  <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+    <text x="40" y="200">● TLS termination</text>
+    <text x="40" y="220">● EC2 Query Protocol parsing</text>
+    <text x="40" y="240">● XML response generation</text>
+
+    <text x="460" y="200">● AWS SigV4 authentication</text>
+    <text x="460" y="220">● Routes to service handlers</text>
+    <text x="460" y="240">● NATS client connection</text>
+  </g>
+
+  <!-- Connector: JSON over NATS -->
+  <line x1="450" y1="258" x2="450" y2="286" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="380" y="260" width="140" height="20" rx="10" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="274" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" text-anchor="middle">JSON over NATS</text>
+
+  <!-- ═══ NATS BROKER ═══════════════════════════════════ -->
+  <rect x="16" y="286" width="868" height="124" rx="8" fill="#1a0a0a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <rect x="16" y="286" width="868" height="28" rx="8" fill="url(#nats-grad)" opacity="0.9"/>
+  <text x="32" y="305" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS MESSAGE BROKER</text>
+  <text x="876" y="305" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">request/reply · queue groups · fan-out</text>
+
+  <!-- Two-column topics/queue groups -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="40" y="332" fill="#F87171" font-weight="600">Topics</text>
+    <text x="40" y="350" fill="#f5f5f5">ec2.RunInstances</text>
+    <text x="40" y="368" fill="#f5f5f5">ec2.DescribeInstances</text>
+    <text x="40" y="386" fill="#f5f5f5">ec2.cmd.{instanceID}</text>
+    <text x="40" y="404" fill="#f5f5f5">ebs.mount  ·  ebs.unmount</text>
+
+    <text x="460" y="332" fill="#F87171" font-weight="600">Delivery mode</text>
+    <text x="460" y="350" fill="#a1a1aa">spinifex-workers <tspan fill="#71717a">(load-balanced)</tspan></text>
+    <text x="460" y="368" fill="#a1a1aa">no queue group <tspan fill="#71717a">(fan-out to all nodes)</tspan></text>
+    <text x="460" y="386" fill="#a1a1aa">per-instance <tspan fill="#71717a">(start / stop / terminate)</tspan></text>
+    <text x="460" y="404" fill="#a1a1aa">Viperblock request/reply</text>
+  </g>
+
+  <!-- Fan-out connectors to 3 daemons -->
+  <g stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6" fill="none">
+    <line x1="450" y1="410" x2="450" y2="430"/>
+    <line x1="450" y1="430" x2="155" y2="452"/>
+    <line x1="450" y1="430" x2="450" y2="452"/>
+    <line x1="450" y1="430" x2="745" y2="452"/>
+  </g>
+
+  <!-- ═══ DAEMON ROW ════════════════════════════════════ -->
+  <!-- Daemon 1 -->
+  <g transform="translate(32, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node 1</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon 2 -->
+  <g transform="translate(327, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node 2</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon N -->
+  <g transform="translate(622, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node N</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon → VM connectors -->
+  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
+    <line x1="155" y1="576" x2="155" y2="610"/>
+    <line x1="450" y1="576" x2="450" y2="610"/>
+    <line x1="745" y1="576" x2="745" y2="610"/>
+  </g>
+
+  <!-- ═══ VM BAR + ROW ═══════════════════════════════════ -->
+  <rect x="16" y="610" width="868" height="22" rx="5" fill="url(#bar-grad)"/>
+  <text x="450" y="625" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">WORKLOADS · QEMU/KVM virtual machines per node</text>
+
+  <!-- VM boxes -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <rect x="32" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="155" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="155" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="155" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node 1 workloads</text>
+
+    <rect x="327" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="450" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="450" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="450" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node 2 workloads</text>
+
+    <rect x="622" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="745" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="745" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="745" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node N workloads</text>
+  </g>
+</svg>

.github/assets/diagrams/tf-bastion.svg

@@ -0,0 +1,46 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="180" viewBox="0 0 900 180">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="180" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="28" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">BASTION · only entry point into a private subnet</text>
+
+  <!-- WAN -->
+  <g transform="translate(40, 56)">
+    <rect width="200" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">WAN</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">operator / laptop</text>
+  </g>
+
+  <!-- Arrow: SSH -->
+  <line x1="240" y1="86" x2="320" y2="86" stroke="#58a6ff" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="316,82 328,86 316,90" fill="#58a6ff" opacity="0.8"/>
+  <rect x="244" y="62" width="80" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="284" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">SSH</text>
+
+  <!-- Bastion -->
+  <g transform="translate(338, 56)">
+    <rect width="220" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="110" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Bastion</text>
+    <text x="110" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">public subnet</text>
+  </g>
+
+  <!-- Arrow: SSH (private IP) -->
+  <line x1="558" y1="86" x2="638" y2="86" stroke="#a855f7" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="634,82 646,86 634,90" fill="#a855f7" opacity="0.8"/>
+  <rect x="552" y="62" width="140" height="18" rx="9" fill="#0a0a0a" stroke="#a855f7" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="622" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#c084fc" text-anchor="middle">SSH (private IP)</text>
+
+  <!-- App server -->
+  <g transform="translate(656, 56)">
+    <rect width="220" height="60" rx="8" fill="#a855f71a" stroke="#a855f7" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="110" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">App Server</text>
+    <text x="110" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">private subnet · no internet</text>
+  </g>
+
+  <text x="450" y="156" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" text-anchor="middle" font-style="italic">App server has no route to the internet — all access hops through the bastion.</text>
+</svg>

.github/assets/diagrams/tf-nginx-alb.svg

@@ -0,0 +1,98 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="480" viewBox="0 0 900 480">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="priv-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f71f"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="480" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · VPC WITH PUBLIC AND PRIVATE SUBNETS</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · multi-AZ</text>
+
+  <!-- Internet -->
+  <rect x="350" y="54" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to IGW -->
+  <line x1="450" y1="94" x2="450" y2="116" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,112 454,112 450,120" fill="#71717a" opacity="0.7"/>
+
+  <!-- IGW -->
+  <rect x="380" y="122" width="140" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="450" y="142" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">IGW</text>
+  <text x="450" y="158" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">nginx-alb-igw</text>
+
+  <!-- Fan-out to two public subnets -->
+  <line x1="450" y1="166" x2="450" y2="184" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="230" y1="184" x2="670" y2="184" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="230" y1="184" x2="230" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="670" y1="184" x2="670" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <!-- public_a -->
+  <g transform="translate(108, 202)">
+    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_a</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+
+    <rect x="20" y="70" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="85" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">NAT GW</text>
+  </g>
+
+  <!-- public_b -->
+  <g transform="translate(548, 202)">
+    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_b</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="56" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="71" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+  </g>
+
+  <!-- SNAT arrow from public_a NAT → private row -->
+  <line x1="230" y1="306" x2="230" y2="340" stroke="#F87171" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.65"/>
+  <polygon points="226,336 234,336 230,344" fill="#F87171" opacity="0.8"/>
+  <rect x="256" y="314" width="240" height="20" rx="10" fill="#0a0a0a" stroke="#F87171" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="376" y="328" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">SNAT for private subnets (egress)</text>
+
+  <!-- private_a -->
+  <g transform="translate(108, 350)">
+    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_a</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_1</text>
+    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
+  </g>
+
+  <!-- private_b -->
+  <g transform="translate(548, 350)">
+    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_b</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_2</text>
+    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
+  </g>
+</svg>

.github/assets/diagrams/tf-s3-webapp.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="180" viewBox="0 0 900 180">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="180" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="28" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">S3 WEBAPP · Flask on EC2 with Predastore backend</text>
+
+  <!-- Browser -->
+  <g transform="translate(40, 56)">
+    <rect width="200" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Browser</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">end user</text>
+  </g>
+
+  <!-- Arrow: HTTP -->
+  <line x1="240" y1="86" x2="320" y2="86" stroke="#58a6ff" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="316,82 328,86 316,90" fill="#58a6ff" opacity="0.8"/>
+  <rect x="244" y="62" width="80" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="284" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">HTTP</text>
+
+  <!-- EC2 Instance -->
+  <g transform="translate(338, 56)">
+    <rect width="240" height="60" rx="8" fill="#DC26261a" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.65"/>
+    <text x="120" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">EC2 Instance</text>
+    <text x="120" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">Flask webapp · port 80</text>
+  </g>
+
+  <!-- Arrow: S3 API (boto3) -->
+  <line x1="578" y1="86" x2="658" y2="86" stroke="#a855f7" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="654,82 666,86 654,90" fill="#a855f7" opacity="0.8"/>
+  <rect x="570" y="62" width="144" height="18" rx="9" fill="#0a0a0a" stroke="#a855f7" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="642" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#c084fc" text-anchor="middle">S3 API (boto3)</text>
+
+  <!-- Predastore -->
+  <g transform="translate(676, 56)">
+    <rect width="200" height="60" rx="8" fill="#a855f71a" stroke="#a855f7" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">Predastore</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">S3 store · port 8443</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-config-layers.svg

@@ -0,0 +1,78 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="330" viewBox="0 0 900 330">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="cluster-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pool-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D08"/>
+    </linearGradient>
+    <linearGradient id="node-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="330" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">CONFIGURATION LEVELS · spinifex.toml</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">three scopes: cluster · pool · node</text>
+
+  <!-- File badge -->
+  <rect x="24" y="50" width="140" height="24" rx="6" fill="#161b22" stroke="#52525b" stroke-width="0.6"/>
+  <text x="94" y="67" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#e4e4e7" text-anchor="middle">spinifex.toml</text>
+
+  <!-- spine -->
+  <line x1="52" y1="74" x2="52" y2="298" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+
+  <!-- ═══ Level 1: [network] cluster-wide ═══════════════════ -->
+  <line x1="52" y1="108" x2="92" y2="108" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="86" width="764" height="56" rx="8" fill="url(#cluster-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="108" y="108" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">[network]</text>
+  <text x="220" y="108" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">Cluster-wide · which mode?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="128" fill="#a1a1aa">└─</text>
+    <text x="128" y="128" fill="#7ee787">external_mode</text>
+    <text x="224" y="128" fill="#a1a1aa">=</text>
+    <text x="238" y="128" fill="#f5f5f5">"pool"</text>
+  </g>
+
+  <!-- ═══ Level 2: [[network.external_pools]] ═══════════════ -->
+  <line x1="52" y1="176" x2="92" y2="176" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="154" width="764" height="78" rx="8" fill="url(#pool-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="108" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">[[network.external_pools]]</text>
+  <text x="328" y="176" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">IP ranges · what IPs to use?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="196" fill="#a1a1aa">├─</text>
+    <text x="128" y="196" fill="#FCA5A5">name</text>
+    <text x="158" y="196" fill="#71717a">·</text>
+    <text x="166" y="196" fill="#FCA5A5">range_start</text>
+    <text x="246" y="196" fill="#71717a">·</text>
+    <text x="254" y="196" fill="#FCA5A5">range_end</text>
+
+    <text x="108" y="212" fill="#a1a1aa">├─</text>
+    <text x="128" y="212" fill="#FCA5A5">gateway</text>
+    <text x="182" y="212" fill="#71717a">·</text>
+    <text x="190" y="212" fill="#FCA5A5">prefix_len</text>
+
+    <text x="108" y="228" fill="#a1a1aa">└─</text>
+    <text x="128" y="228" fill="#FCA5A5">region</text>
+    <text x="174" y="228" fill="#71717a">·</text>
+    <text x="182" y="228" fill="#FCA5A5">az</text>
+    <text x="202" y="228" fill="#71717a">(optional)</text>
+  </g>
+
+  <!-- ═══ Level 3: [nodes.NAME.vpcd] ════════════════════════ -->
+  <line x1="52" y1="268" x2="92" y2="268" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="246" width="764" height="56" rx="8" fill="url(#node-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+  <text x="108" y="268" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" font-weight="700">[nodes.NAME.vpcd]</text>
+  <text x="266" y="268" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">Per-node · which NIC?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="288" fill="#a1a1aa">└─</text>
+    <text x="128" y="288" fill="#8ab4f8">external_interface</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-datapath-localnet.svg

@@ -0,0 +1,63 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="110" viewBox="0 0 900 110">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="110" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="22" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">DATA PATH · VM to WAN via OVN localnet</text>
+
+  <!-- VM TAP -->
+  <g transform="translate(24, 40)">
+    <rect width="118" height="48" rx="7" fill="#a855f71a" stroke="#a855f7" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="59" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" text-anchor="middle" font-weight="700">VM TAP</text>
+    <text x="59" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">guest interface</text>
+  </g>
+  <polygon points="148,60 160,64 148,68" fill="#71717a" opacity="0.7"/>
+  <line x1="142" y1="64" x2="148" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-int -->
+  <g transform="translate(166, 40)">
+    <rect width="102" height="48" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="51" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" text-anchor="middle" font-weight="700">br-int</text>
+    <text x="51" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS bridge</text>
+  </g>
+  <polygon points="274,60 286,64 274,68" fill="#71717a" opacity="0.7"/>
+  <line x1="268" y1="64" x2="274" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- OVN logical pipeline -->
+  <g transform="translate(292, 40)">
+    <rect width="192" height="48" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="96" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">OVN logical pipeline</text>
+    <text x="96" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">switch → router → SNAT/DNAT</text>
+  </g>
+  <polygon points="490,60 502,64 490,68" fill="#71717a" opacity="0.7"/>
+  <line x1="484" y1="64" x2="490" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- localnet -->
+  <g transform="translate(508, 40)">
+    <rect width="120" height="48" rx="7" fill="#DC26260d" stroke="#DC2626" stroke-width="1" stroke-opacity="0.5" stroke-dasharray="4,3"/>
+    <text x="60" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">localnet</text>
+    <text x="60" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">OVN port type</text>
+  </g>
+  <polygon points="634,60 646,64 634,68" fill="#71717a" opacity="0.7"/>
+  <line x1="628" y1="64" x2="634" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-wan -->
+  <g transform="translate(652, 40)">
+    <rect width="110" height="48" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="55" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+    <text x="55" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS bridge</text>
+  </g>
+  <polygon points="768,60 780,64 768,68" fill="#71717a" opacity="0.7"/>
+  <line x1="762" y1="64" x2="768" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- WAN -->
+  <g transform="translate(786, 40)">
+    <rect width="90" height="48" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="45" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" text-anchor="middle" font-weight="700">WAN</text>
+    <text x="45" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">uplink</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-dhcp-conversations.svg

@@ -0,0 +1,74 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="360" viewBox="0 0 900 360">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="host-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+    <linearGradient id="vm-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="360" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">TWO DHCP CONVERSATIONS · completely independent</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">pool mode · static or dhcp source</text>
+
+  <!-- ═══ HOST ↔ ROUTER DHCP ══════════════════════════════ -->
+  <rect x="24" y="54" width="852" height="138" rx="10" fill="url(#host-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+  <text x="44" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" font-weight="700" letter-spacing="0.8">HOST ↔ ROUTER DHCP</text>
+  <text x="876" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="end">on br-ext · veth pair → br-wan</text>
+  <line x1="24" y1="88" x2="876" y2="88" stroke="#58a6ff" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <!-- step 1 -->
+    <circle cx="54" cy="108" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="112" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">1</text>
+    <text x="74" y="112" fill="#f5f5f5">Spinifex asks router:</text>
+    <text x="228" y="112" fill="#a1a1aa">"give me a public IP for this VM"</text>
+
+    <!-- step 2 -->
+    <circle cx="54" cy="132" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="136" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">2</text>
+    <text x="74" y="136" fill="#f5f5f5">Router responds:</text>
+    <text x="196" y="136" fill="#8ab4f8" font-weight="600">192.168.1.75</text>
+
+    <!-- step 3 -->
+    <circle cx="54" cy="156" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="160" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">3</text>
+    <text x="74" y="160" fill="#f5f5f5">Spinifex stores IP, tells OVN:</text>
+    <text x="274" y="160" fill="#F87171" font-weight="600">dnat_and_snat(192.168.1.75 ↔ 172.31.0.4)</text>
+  </g>
+
+  <text x="44" y="181" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" font-style="italic">Only used in DHCP source mode. Static mode picks from range.</text>
+
+  <!-- separator -->
+  <line x1="24" y1="206" x2="876" y2="206" stroke="#27272a" stroke-width="0.8"/>
+  <text x="450" y="220" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#52525b" text-anchor="middle" letter-spacing="2">INDEPENDENT · no packets cross between</text>
+
+  <!-- ═══ VM ↔ OVN DHCP ═══════════════════════════════════ -->
+  <rect x="24" y="234" width="852" height="110" rx="10" fill="url(#vm-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="44" y="258" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700" letter-spacing="0.8">VM ↔ OVN DHCP</text>
+  <text x="876" y="258" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#7ee787" text-anchor="end">on tap → br-int → OVN logical switch</text>
+  <line x1="24" y1="268" x2="876" y2="268" stroke="#3fb950" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <circle cx="54" cy="288" r="9" fill="#3fb950" fill-opacity="0.15" stroke="#3fb950" stroke-width="1"/>
+    <text x="54" y="292" fill="#3fb950" text-anchor="middle" font-weight="700" font-size="10">1</text>
+    <text x="74" y="292" fill="#f5f5f5">cloud-init asks:</text>
+    <text x="190" y="292" fill="#a1a1aa">"what's my IP?"</text>
+
+    <circle cx="54" cy="312" r="9" fill="#3fb950" fill-opacity="0.15" stroke="#3fb950" stroke-width="1"/>
+    <text x="54" y="316" fill="#3fb950" text-anchor="middle" font-weight="700" font-size="10">2</text>
+    <text x="74" y="316" fill="#f5f5f5">OVN DHCP responds:</text>
+    <text x="226" y="316" fill="#a1a1aa">"you are</text>
+    <text x="284" y="316" fill="#7ee787" font-weight="600">172.31.0.4</text>
+    <text x="356" y="316" fill="#a1a1aa">"</text>
+  </g>
+
+  <text x="44" y="336" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" font-style="italic">VM never sees 192.168.1.x or the router.</text>
+</svg>

.github/assets/diagrams/vpc-direct-bridge.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="170" viewBox="0 0 900 170">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="170" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">DIRECT BRIDGE MODE · dedicated WAN NIC</text>
+
+  <!-- Row 1: Management NIC (independent, no WAN bridge path) -->
+  <g transform="translate(24, 42)">
+    <rect width="852" height="44" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+    <text x="20" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">Management NIC</text>
+    <text x="20" y="36" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">host IP · SSH · spinifex services · overlay (Geneve)</text>
+    <text x="840" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#7ee787" text-anchor="end" font-weight="600">independent path</text>
+  </g>
+
+  <!-- Row 2: WAN NIC → br-wan → localnet -->
+  <g transform="translate(24, 100)">
+    <rect width="160" height="44" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="80" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">WAN NIC</text>
+    <text x="80" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">dedicated physical</text>
+  </g>
+  <polygon points="198,118 210,122 198,126" fill="#71717a" opacity="0.7"/>
+  <line x1="186" y1="122" x2="198" y2="122" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(214, 100)">
+    <rect width="220" height="44" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="110" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+    <text x="110" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS port (NIC added directly)</text>
+  </g>
+  <polygon points="448,118 460,122 448,126" fill="#71717a" opacity="0.7"/>
+  <line x1="434" y1="122" x2="448" y2="122" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(464, 100)">
+    <rect width="180" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">OVN localnet</text>
+    <text x="90" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">external switch port</text>
+  </g>
+
+  <text x="670" y="130" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" font-weight="600">distributed NAT · any MAC · all frames</text>
+</svg>

.github/assets/diagrams/vpc-host-flow.svg

@@ -0,0 +1,91 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="440" viewBox="0 0 900 440">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="host-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#27272a88"/>
+      <stop offset="100%" style="stop-color:#27272a22"/>
+    </linearGradient>
+    <linearGradient id="br-int-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f71f"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+    <linearGradient id="br-wan-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#58a6ff1f"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="440" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NETWORK FLOW · bare-metal host</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">br-int overlay · br-wan WAN uplink</text>
+
+  <!-- ═══ Bare-Metal Host container ═══════════════════════ -->
+  <rect x="40" y="60" width="820" height="284" rx="12" fill="url(#host-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.7"/>
+  <text x="58" y="84" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" font-weight="700" letter-spacing="1">BARE-METAL HOST</text>
+  <line x1="40" y1="96" x2="860" y2="96" stroke="#52525b" stroke-opacity="0.5" stroke-width="0.5"/>
+
+  <!-- br-int bridge (left) -->
+  <g transform="translate(80, 122)">
+    <rect width="260" height="176" rx="10" fill="url(#br-int-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" font-weight="700">br-int</text>
+    <text x="240" y="24" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="end">(overlay)</text>
+    <line x1="20" y1="34" x2="240" y2="34" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+      <rect x="30" y="52" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="70" fill="#d8b4fe" text-anchor="middle">tap-eni1</text>
+
+      <rect x="30" y="88" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="106" fill="#d8b4fe" text-anchor="middle">tap-eni2</text>
+
+      <rect x="30" y="124" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="142" fill="#d8b4fe" text-anchor="middle">tap-eni3</text>
+    </g>
+  </g>
+
+  <!-- OVN NAT pipeline arrow in middle -->
+  <g transform="translate(352, 200)">
+    <line x1="0" y1="10" x2="148" y2="10" stroke="#DC2626" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+    <polygon points="148,5 160,10 148,15" fill="#DC2626" opacity="0.8"/>
+    <text x="78" y="0" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">OVN</text>
+    <text x="78" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">NAT pipeline</text>
+  </g>
+
+  <!-- br-wan bridge (right) -->
+  <g transform="translate(520, 122)">
+    <rect width="260" height="176" rx="10" fill="url(#br-wan-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" font-weight="700">br-wan</text>
+    <text x="240" y="24" font-family="'Segoe UI', sans-serif" font-size="10" fill="#8ab4f8" text-anchor="end">(WAN)</text>
+    <line x1="20" y1="34" x2="240" y2="34" stroke="#58a6ff" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+      <rect x="30" y="52" width="200" height="60" rx="5" fill="#58a6ff1a" stroke="#58a6ff" stroke-opacity="0.45" stroke-width="0.8"/>
+      <text x="130" y="74" fill="#8ab4f8" text-anchor="middle" font-weight="600">spx-ext-{nic}</text>
+      <text x="130" y="92" fill="#a1a1aa" text-anchor="middle" font-size="10">(macvlan)</text>
+    </g>
+  </g>
+
+  <!-- macvlan bridge mode label -->
+  <text x="650" y="314" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="middle" font-style="italic">macvlan bridge mode</text>
+
+  <!-- WAN NIC inside host, bottom -->
+  <line x1="650" y1="298" x2="650" y2="320" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(500, 320)">
+    <rect width="300" height="20" rx="5" fill="#161b22" stroke="#52525b" stroke-width="0.6"/>
+    <text x="150" y="14" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">WAN NIC · host IP unchanged</text>
+  </g>
+
+  <!-- connector out -->
+  <line x1="650" y1="344" x2="650" y2="380" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="646,376 654,376 650,384" fill="#71717a" opacity="0.7"/>
+
+  <!-- Physical WAN -->
+  <g transform="translate(500, 388)">
+    <rect width="300" height="40" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+    <text x="150" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" text-anchor="middle" font-weight="700">Physical WAN</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-macvlan.svg

@@ -0,0 +1,57 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="320" viewBox="0 0 900 320">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="320" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">MACVLAN FALLBACK · single-NIC host</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">WAN NIC == SSH NIC · host IP preserved</text>
+
+  <!-- WAN NIC (parent) -->
+  <g transform="translate(280, 58)">
+    <rect width="340" height="54" rx="8" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.3" stroke-opacity="0.65"/>
+    <text x="170" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" text-anchor="middle" font-weight="700">WAN NIC</text>
+    <text x="170" y="42" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">host IP unchanged · parent interface</text>
+  </g>
+
+  <!-- Split point -->
+  <line x1="450" y1="112" x2="450" y2="132" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="260" y1="132" x2="640" y2="132" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="260" y1="132" x2="260" y2="148" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="640" y1="132" x2="640" y2="148" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- Child 1: host traffic -->
+  <g transform="translate(120, 148)">
+    <rect width="280" height="48" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="140" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" text-anchor="middle" font-weight="700">host traffic</text>
+    <text x="140" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SSH · management · overlay</text>
+  </g>
+
+  <!-- Child 2: macvlan -->
+  <g transform="translate(500, 148)">
+    <rect width="280" height="48" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="140" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">spx-ext-{nic}</text>
+    <text x="140" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">macvlan · bridge mode · no IP</text>
+  </g>
+
+  <!-- br-wan -->
+  <line x1="640" y1="196" x2="640" y2="216" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="636,212 644,212 640,220" fill="#71717a" opacity="0.7"/>
+
+  <g transform="translate(500, 220)">
+    <rect width="280" height="40" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="140" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+  </g>
+
+  <!-- localnet -->
+  <line x1="640" y1="260" x2="640" y2="276" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="636,272 644,272 640,280" fill="#71717a" opacity="0.7"/>
+
+  <g transform="translate(500, 280)">
+    <rect width="280" height="30" rx="6" fill="#DC26260d" stroke="#DC2626" stroke-width="1" stroke-opacity="0.5" stroke-dasharray="4,3"/>
+    <text x="140" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="600">localnet → OVN ext switch</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-network-path.svg

@@ -0,0 +1,131 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="wan-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="ovs-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff0a"/>
+    </linearGradient>
+    <linearGradient id="router-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D0a"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="priv-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f722"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="560" fill="url(#bg)" rx="12"/>
+
+  <!-- Title -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NETWORK PATH</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">OVN logical topology · bridge mapping · NAT</text>
+
+  <!-- WAN -->
+  <rect x="300" y="54" width="300" height="40" rx="8" fill="url(#wan-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.55"/>
+  <text x="450" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Physical Network (WAN)</text>
+
+  <!-- connector -->
+  <line x1="450" y1="94" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-wan bridge -->
+  <rect x="340" y="112" width="220" height="52" rx="8" fill="url(#ovs-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="450" y="134" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+  <text x="450" y="152" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">OVS bridge · physical uplink</text>
+
+  <!-- connector with bridge-mappings label -->
+  <line x1="450" y1="164" x2="450" y2="186" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <rect x="316" y="170" width="268" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="450" y="183" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">ovn-bridge-mappings: external:br-wan</text>
+
+  <!-- VPC Logical Router (the big box) -->
+  <rect x="60" y="208" width="780" height="196" rx="10" fill="url(#router-grad)" stroke="#DC2626" stroke-width="1.4" stroke-opacity="0.75"/>
+  <text x="78" y="232" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" font-weight="700" letter-spacing="1">VPC LOGICAL ROUTER</text>
+  <line x1="60" y1="242" x2="840" y2="242" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <!-- Router ports row -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <rect x="82" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="142" y="270" fill="#F87171" text-anchor="middle" font-weight="600">Gateway port</text>
+    <text x="142" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">public IP</text>
+
+    <text x="215" y="282" fill="#71717a">─────</text>
+
+    <rect x="260" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="320" y="270" fill="#F87171" text-anchor="middle" font-weight="600">ext switch</text>
+    <text x="320" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">OVN logical</text>
+
+    <text x="393" y="282" fill="#71717a">─────</text>
+
+    <rect x="438" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="498" y="270" fill="#F87171" text-anchor="middle" font-weight="600">localnet port</text>
+    <text x="498" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">→ br-wan</text>
+  </g>
+
+  <!-- NAT rules + Routes, two columns on right -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <text x="598" y="260" fill="#F87171" font-weight="600">NAT rules</text>
+    <text x="598" y="276" fill="#f5f5f5">SNAT:</text>
+    <text x="644" y="276" fill="#a1a1aa">VPC CIDR → gw IP</text>
+    <text x="598" y="292" fill="#f5f5f5">dnat_and_snat:</text>
+    <text x="704" y="292" fill="#a1a1aa">pub ↔ priv</text>
+  </g>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <text x="82" y="330" fill="#F87171" font-weight="600">Routes</text>
+    <text x="82" y="346" fill="#f5f5f5">0.0.0.0/0</text>
+    <text x="154" y="346" fill="#a1a1aa">→ WAN gateway</text>
+  </g>
+
+  <!-- Subnet branches inside router -->
+  <line x1="150" y1="362" x2="150" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="150" y1="386" x2="305" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="305" y1="386" x2="305" y2="404" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <line x1="150" y1="386" x2="515" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="515" y1="386" x2="515" y2="404" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <!-- Subnet A (public) -->
+  <g transform="translate(215, 404)">
+    <rect width="180" height="124" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">subnet-a</text>
+    <text x="90" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">(public)</text>
+    <line x1="16" y1="46" x2="164" y2="46" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="90" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#f5f5f5" text-anchor="middle">10.0.1.0/24</text>
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa">
+      <circle cx="40" cy="88" r="2.5" fill="#3fb950"/>
+      <text x="52" y="92">eni-1</text>
+      <circle cx="40" cy="108" r="2.5" fill="#3fb950"/>
+      <text x="52" y="112">eni-2</text>
+    </g>
+  </g>
+
+  <!-- Subnet B (private) -->
+  <g transform="translate(425, 404)">
+    <rect width="180" height="124" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">subnet-b</text>
+    <text x="90" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">(private)</text>
+    <line x1="16" y1="46" x2="164" y2="46" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="90" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#f5f5f5" text-anchor="middle">10.0.2.0/24</text>
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa">
+      <circle cx="40" cy="88" r="2.5" fill="#a855f7"/>
+      <text x="52" y="92">eni-3</text>
+      <circle cx="40" cy="108" r="2.5" fill="#a855f7"/>
+      <text x="52" y="112">eni-4</text>
+    </g>
+  </g>
+
+  <!-- Footnote -->
+  <text x="24" y="548" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a">Cross-host traffic uses Geneve tunnels (UDP 6081) over the overlay NIC · ovn-controller programs OpenFlow rules on br-int</text>
+</svg>

.github/assets/diagrams/vpc-private-subnet-flow.svg

@@ -0,0 +1,56 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="110" viewBox="0 0 900 110">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="110" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">PRIVATE SUBNET · no default route</text>
+
+  <!-- Instance -->
+  <g transform="translate(40, 42)">
+    <rect width="180" height="44" rx="7" fill="#a855f71a" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" text-anchor="middle" font-weight="700">Instance</text>
+    <text x="90" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">10.0.2.5</text>
+  </g>
+
+  <!-- arrow -->
+  <g stroke="#DC2626" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="220" y1="64" x2="260" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="260,60 268,64 260,68" fill="#DC2626" opacity="0.6"/>
+
+  <!-- Router -->
+  <g transform="translate(275, 42)">
+    <rect width="180" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">VPC Router</text>
+    <text x="90" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">lookup 0.0.0.0/0</text>
+  </g>
+
+  <!-- arrow -->
+  <g stroke="#DC2626" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="455" y1="64" x2="495" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="495,60 503,64 495,68" fill="#DC2626" opacity="0.6"/>
+
+  <!-- [no default route] -->
+  <g transform="translate(510, 42)">
+    <rect width="196" height="44" rx="7" fill="#0a0a0a" stroke="#71717a" stroke-width="1" stroke-opacity="0.7" stroke-dasharray="4,3"/>
+    <text x="98" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" text-anchor="middle" font-weight="600">[no default route]</text>
+    <text x="98" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="middle">no IGW attached</text>
+  </g>
+
+  <!-- arrow to dropped -->
+  <g stroke="#71717a" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="706" y1="64" x2="746" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="746,60 754,64 746,68" fill="#71717a" opacity="0.6"/>
+
+  <!-- dropped -->
+  <g transform="translate(760, 42)">
+    <rect width="116" height="44" rx="7" fill="#27272a" stroke="#52525b" stroke-width="1" stroke-opacity="0.7"/>
+    <text x="58" y="27" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" text-anchor="middle" font-weight="600">dropped</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-public-subnet-flow.svg

@@ -0,0 +1,74 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="170" viewBox="0 0 900 170">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="170" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">PUBLIC SUBNET · 1:1 NAT managed by OVN</text>
+
+  <!-- OUTBOUND row -->
+  <text x="24" y="52" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700">Outbound</text>
+
+  <g transform="translate(112, 38)">
+    <rect width="180" height="40" rx="6" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="90" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#7ee787" text-anchor="middle" font-weight="700">Instance</text>
+    <text x="90" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">10.0.1.5</text>
+  </g>
+
+  <g stroke="#3fb950" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="292" y1="58" x2="352" y2="58" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="352,54 360,58 352,62" fill="#3fb950" opacity="0.7"/>
+
+  <g transform="translate(368, 38)">
+    <rect width="130" height="40" rx="6" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="65" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">SNAT</text>
+  </g>
+
+  <g stroke="#3fb950" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="498" y1="58" x2="558" y2="58" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="558,54 566,58 558,62" fill="#3fb950" opacity="0.7"/>
+
+  <g transform="translate(574, 38)">
+    <rect width="272" height="40" rx="6" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="136" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#7ee787" text-anchor="middle" font-weight="700">exits as 203.0.113.10</text>
+    <text x="136" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">source IP rewritten</text>
+  </g>
+
+  <!-- INBOUND row -->
+  <text x="24" y="110" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" font-weight="700">Inbound</text>
+
+  <g transform="translate(112, 96)">
+    <rect width="180" height="40" rx="6" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="90" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#8ab4f8" text-anchor="middle" font-weight="700">WAN client</text>
+    <text x="90" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">→ 203.0.113.10</text>
+  </g>
+
+  <g stroke="#58a6ff" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="292" y1="116" x2="352" y2="116" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="352,112 360,116 352,120" fill="#58a6ff" opacity="0.7"/>
+
+  <g transform="translate(368, 96)">
+    <rect width="130" height="40" rx="6" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="65" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">DNAT</text>
+  </g>
+
+  <g stroke="#58a6ff" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="498" y1="116" x2="558" y2="116" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="558,112 566,116 558,120" fill="#58a6ff" opacity="0.7"/>
+
+  <g transform="translate(574, 96)">
+    <rect width="272" height="40" rx="6" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="136" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#8ab4f8" text-anchor="middle" font-weight="700">arrives at 10.0.1.5</text>
+    <text x="136" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">dest IP rewritten</text>
+  </g>
+
+  <!-- Footnote -->
+  <text x="24" y="158" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a">The instance OS only sees its private IP (10.0.1.5) — the public address is managed entirely by OVN's dnat_and_snat rule.</text>
+</svg>

.github/assets/platform.svg

@@ -0,0 +1,144 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
+  <defs>
+    <linearGradient id="pf-bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="hw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#52525b;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#52525b;stop-opacity:0.06"/>
+    </linearGradient>
+    <linearGradient id="os-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f7;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#a855f7;stop-opacity:0.05"/>
+    </linearGradient>
+    <linearGradient id="spx-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.28"/>
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.1"/>
+    </linearGradient>
+    <linearGradient id="app-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb950;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#3fb950;stop-opacity:0.05"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="560" fill="url(#pf-bg)" rx="12"/>
+
+  <!-- Subtle mesh dots background -->
+  <g opacity="0.12" fill="#7F1D1D">
+    <circle cx="40" cy="40" r="1"/><circle cx="120" cy="30" r="1"/><circle cx="220" cy="50" r="1"/>
+    <circle cx="320" cy="25" r="1"/><circle cx="440" cy="45" r="1"/><circle cx="560" cy="30" r="1"/>
+    <circle cx="680" cy="50" r="1"/><circle cx="780" cy="30" r="1"/><circle cx="860" cy="45" r="1"/>
+    <circle cx="60" cy="530" r="1"/><circle cx="160" cy="520" r="1"/><circle cx="260" cy="540" r="1"/>
+    <circle cx="380" cy="525" r="1"/><circle cx="500" cy="540" r="1"/><circle cx="620" cy="520" r="1"/>
+    <circle cx="740" cy="540" r="1"/><circle cx="840" cy="525" r="1"/>
+  </g>
+
+  <!-- Title -->
+  <text x="450" y="32" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" text-anchor="middle" letter-spacing="2.8">THE MULGA PLATFORM</text>
+  <text x="450" y="58" font-family="'Segoe UI', system-ui, sans-serif" font-size="20" font-weight="600" fill="#f5f5f5" text-anchor="middle">AWS-style workflows, on your own stack</text>
+
+  <!-- Layer 4: Applications (top) -->
+  <g transform="translate(60, 95)">
+    <rect width="780" height="88" rx="10" fill="url(#app-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" letter-spacing="1.5" font-weight="600">LAYER 4 · APPLICATIONS &amp; TOOLING</text>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">Your workloads — unmodified AWS tooling</text>
+
+    <g transform="translate(20, 60)" font-family="'Segoe UI', sans-serif" font-size="10.5">
+      <rect x="0" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="35" y="14" fill="#7ee787" text-anchor="middle">aws-cli</text>
+
+      <rect x="78" y="0" width="78" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="117" y="14" fill="#7ee787" text-anchor="middle">Terraform</text>
+
+      <rect x="164" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="208" y="14" fill="#7ee787" text-anchor="middle">AWS SDKs</text>
+
+      <rect x="260" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="304" y="14" fill="#7ee787" text-anchor="middle">Kubernetes</text>
+
+      <rect x="356" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="391" y="14" fill="#7ee787" text-anchor="middle">Packer</text>
+
+      <rect x="434" y="0" width="98" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="483" y="14" fill="#7ee787" text-anchor="middle">Your Services</text>
+    </g>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 188)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 3: Spinifex (highlighted) -->
+  <g transform="translate(60, 215)">
+    <rect width="780" height="140" rx="10" fill="url(#spx-grad)" stroke="#DC2626" stroke-width="1.5" stroke-opacity="0.75"/>
+    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" letter-spacing="1.5" font-weight="600">LAYER 3 · SPINIFEX · AWS ORCHESTRATION</text>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">AWS Gateway → NATS → Service Daemons · SigV4, IAM, VPC-aware</text>
+
+    <g transform="translate(20, 66)" font-family="'Segoe UI', sans-serif">
+      <!-- EC2 -->
+      <rect x="0" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="59" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EC2</text>
+      <text x="59" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">QEMU/KVM VMs</text>
+      <text x="59" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">cloud-init · AMIs</text>
+
+      <!-- EBS -->
+      <rect x="126" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="185" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EBS</text>
+      <text x="185" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Viperblock</text>
+      <text x="185" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">WAL · NBD · snapshots</text>
+
+      <!-- S3 -->
+      <rect x="252" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="311" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">S3</text>
+      <text x="311" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Predastore</text>
+      <text x="311" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Reed–Solomon EC</text>
+
+      <!-- VPC -->
+      <rect x="378" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
+      <text x="437" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">VPC</text>
+      <text x="437" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">OVN / OVS</text>
+      <text x="437" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">subnets · SG · ALB</text>
+
+      <!-- IAM -->
+      <rect x="504" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
+      <text x="563" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">IAM</text>
+      <text x="563" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">SigV4 · policies</text>
+      <text x="563" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">users · roles · keys</text>
+
+      <!-- Control -->
+      <rect x="630" y="0" width="110" height="58" rx="6" fill="#27272a" fill-opacity="0.8" stroke="#71717a" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="685" y="18" font-size="13" fill="#e4e4e7" text-anchor="middle" font-weight="600">Control</text>
+      <text x="685" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">NATS bus</text>
+      <text x="685" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">spx · daemons</text>
+    </g>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 360)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 2: OS -->
+  <g transform="translate(60, 387)">
+    <rect width="780" height="62" rx="10" fill="url(#os-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.5"/>
+    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" letter-spacing="1.5" font-weight="600">LAYER 2 · OPERATING SYSTEM</text>
+    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · headless autoinstall</text>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 454)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 1: Hardware -->
+  <g transform="translate(60, 481)">
+    <rect width="780" height="62" rx="10" fill="url(#hw-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" letter-spacing="1.5" font-weight="600">LAYER 1 · HARDWARE</text>
+    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Single node · multi-node cluster · edge appliance · air-gapped rack · commodity x86 / ARM</text>
+  </g>
+</svg>

.github/dependabot.yml

@@ -1,7 +1,7 @@
 version: 2
 updates:
   - package-ecosystem: gomod
-    target-branch: dev
+    target-branch: main
     directory: /
     schedule:
       interval: weekly
@@ -15,11 +15,14 @@ updates:
           - "*"
 
   - package-ecosystem: github-actions
+    target-branch: main
     directory: /
     schedule:
       interval: weekly
       day: monday
       timezone: Australia/Sydney
+    cooldown:
+      default-days: 1
     ignore:
       - dependency-name: "*"
         update-types:
@@ -31,7 +34,7 @@ updates:
           - "*"
 
   - package-ecosystem: npm
-    target-branch: dev
+    target-branch: main
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
@@ -39,6 +42,10 @@ updates:
       timezone: Australia/Sydney
     cooldown:
       default-days: 1
+    ignore:
+      - dependency-name: "@types/node"
+        update-types:
+          - "version-update:semver-major"
     groups:
       npm-dependencies:
         patterns:

.github/workflows/e2e.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
@@ -13,6 +12,13 @@ on:
       - tests/e2e/**
       - .github/workflows/e2e.yml
   workflow_dispatch:
+    inputs:
+      sha:
+        description: "PR head SHA to report status on"
+        required: false
+
+permissions:
+  statuses: write
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
@@ -21,17 +27,17 @@ jobs:
   e2e_single:
     name: Single-Node E2E
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 25
+    timeout-minutes: 30
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -42,9 +48,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - name: Clone and build
+      - name: Build install artifacts
         working-directory: mulga/scripts/tofu-cluster
-        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -56,13 +66,14 @@ jobs:
           ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
           tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-      - name: Bootstrap Cluster
+      - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          SPX_PREBUILT=true \
-          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
           SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
-          ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
+          ./bootstrap-install.sh env3 \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run E2E Tests
         working-directory: mulga/scripts/tofu-cluster
@@ -72,7 +83,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh'
+            'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh  && ./run-lb-e2e.sh'
 
       - name: Teardown
         if: always()
@@ -89,13 +100,13 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -148,17 +159,17 @@ jobs:
   e2e_real_multi:
     name: Real Multi-Node E2E
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 25
+    timeout-minutes: 35
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:
-      - name: Clean workspace
+      - name: Clean Workspace
         run: rm -rf "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -169,9 +180,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - name: Clone and build
+      - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
-        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
@@ -183,13 +198,14 @@ jobs:
           ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
           tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-      - name: Bootstrap Cluster
+      - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          SPX_PREBUILT=true \
-          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
           SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
-          ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
+          ./bootstrap-install.sh env2 \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run E2E Tests
         working-directory: mulga/scripts/tofu-cluster
@@ -206,8 +222,10 @@ jobs:
             NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
           done
 
+          NODE2_IP=$(echo "$INVENTORY" | jq -r '.nodes[1].wan_ip')
+
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
+            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
 
       - name: Teardown
         if: always()
@@ -216,3 +234,29 @@ jobs:
           tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
+
+  e2e_status:
+    name: Report E2E Status
+    if: always() && inputs.sha
+    needs: [e2e_single, e2e_pseudo_multi, e2e_real_multi]
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 5
+    steps:
+      - name: Report status to PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ needs.e2e_single.result }}" == "success" && \
+                "${{ needs.e2e_pseudo_multi.result }}" == "success" && \
+                "${{ needs.e2e_real_multi.result }}" == "success" ]]; then
+            STATE="success"
+            DESC="All E2E suites passed"
+          else
+            STATE="failure"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}"
+          fi
+          gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
+            -f state="$STATE" \
+            -f context=e2e \
+            -f description="$DESC" \
+            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/release.yml

@@ -98,7 +98,7 @@ jobs:
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
         with:
           draft: true
           files: |
@@ -161,28 +161,18 @@ jobs:
           TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
           ./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
 
-      - name: Run Smoke Test
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          INVENTORY=$(tofu output -json inventory)
-          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-            -i "$SSH_KEY" \
-            smoke-test-install.sh \
-            "tf-user@${WAN_IP}:/tmp/smoke-test.sh"
-          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-            -i "$SSH_KEY" "tf-user@${WAN_IP}" \
-            'bash /tmp/smoke-test.sh'
-
       - name: Promote Release
         if: success()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release edit "$TAG" --repo "$REPO" --draft=false
-          echo "Release ${TAG} promoted to latest"
+          if [[ "$TAG" =~ -(dev|rc|alpha|beta) ]]; then
+            gh release edit "$TAG" --repo "$REPO" --draft=false --prerelease
+            echo "Release ${TAG} promoted as prerelease"
+          else
+            gh release edit "$TAG" --repo "$REPO" --draft=false
+            echo "Release ${TAG} promoted to latest"
+          fi
 
       - name: Mark Release Failed
         if: failure()

.github/workflows/spinifex.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - "**/*.go"
       - go.mod
@@ -14,7 +13,6 @@ on:
   pull_request:
     branches:
       - main
-      - dev
   workflow_dispatch:
 
 jobs:

.github/workflows/ui.yml

@@ -4,14 +4,12 @@ on:
   push:
     branches:
       - main
-      - dev
     paths:
       - spinifex/services/spinifexui/frontend/**
       - .github/workflows/ui.yml
   pull_request:
     branches:
       - main
-      - dev
     paths:
       - spinifex/services/spinifexui/frontend/**
       - .github/workflows/ui.yml
@@ -32,7 +30,7 @@ jobs:
           sparse-checkout: spinifex/services/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v5
+        uses: pnpm/action-setup@v6
         with:
           package_json_file: spinifex/services/spinifexui/frontend/package.json
 

.gitignore

@@ -7,6 +7,9 @@
 !example.tfvars
 scripts/iac/proxmox/clusters/
 
+# Generated SSH private keys from terraform examples (e.g. local_file.nginx_pem)
+*.pem
+
 # Go workspace
 go.work
 go.work.sum
@@ -40,3 +43,4 @@ docs/PLAN.md
 docs/TODO.md
 docs/development/
 docs/internal/
+scripts/iac/aws/spinifex-test.pem

.golangci.yml

@@ -22,6 +22,7 @@ linters:
     - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
+    - forcetypeassert # flags unchecked type assertions that can panic
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
@@ -29,11 +30,14 @@ linters:
     - ineffassign # detects assignments to existing variables that are not used
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
     - nolintlint # reports ill-formed or insufficient nolint directives
+    - nosprintfhostport # flags fmt.Sprintf host:port — use net.JoinHostPort for IPv6 safety
+    - perfsprint # detects fmt.Sprintf calls replaceable with faster alternatives
     - predeclared # catches shadowing of Go builtins (len, cap, error, etc.)
     - reassign # checks that package variables are not reassigned
     - recvcheck # checks for receiver type consistency (pointer vs value)
@@ -91,6 +95,12 @@ linters:
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
     nakedret:
       max-func-lines: 0
 
@@ -124,5 +134,3 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
-      - text: "G402"
-        linters: [gosec] # TLS config — we handle TLS ourselves

Makefile

@@ -63,6 +63,19 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+build-lb-agent:
+	@echo -e "\n....Building lb-agent (static)"
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+
+build-system-image: ## Build a system image from manifest (use IMAGE=lb)
+ifndef IMAGE
+	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
+endif
+	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
+
+build-lb-image: ## Build LB Alpine image
+	$(MAKE) build-system-image IMAGE=lb
+
 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
 	$(GOPATH)/bin/$(GO_PROJECT_NAME)
@@ -103,6 +116,17 @@ run:
 	$(MAKE) go_build
 	$(MAKE) go_run
 
+# Fast iteration: build + install binary + restart all services
+deploy: build
+	sudo install -m 755 bin/spx /usr/local/bin/spx
+	sudo systemctl daemon-reload
+	sudo systemctl restart spinifex.target
+
+# Re-run setup.sh after changing systemd units, helper scripts, or logrotate config.
+# Not needed for code-only changes — use deploy for those.
+reinstall:
+	scripts/dev-install.sh
+
 clean:
 	rm -f ./bin/$(GO_PROJECT_NAME)
 	rm -rf spinifex/services/spinifexui/frontend/dist
@@ -119,9 +143,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.1 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.2 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.1.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -194,7 +218,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
+.PHONY: build build-ui build-lb-agent build-system-image build-lb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
+	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -1,25 +1,55 @@
+<p align="center">
+  <img src=".github/assets/banner.svg" alt="Spinifex — AWS-Compatible Infrastructure Stack" width="900">
+</p>
+
+<p align="center">
+  <a href="https://go.dev/"><img src="https://img.shields.io/badge/Go-1.22+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://github.com/mulgadc/spinifex/actions"><img src="https://img.shields.io/badge/CI-e2e-58a6ff?style=flat-square" alt="CI"></a>
+</p>
+
+<p align="center">
+  <a href="#what-is-spinifex">What is Spinifex?</a> ·
+  <a href="#the-platform">Platform</a> ·
+  <a href="#core-components">Components</a> ·
+  <a href="#architecture-at-a-glance">Architecture</a> ·
+  <a href="#installation">Installation</a> ·
+  <a href="https://docs.mulgadc.com">Docs</a>
+</p>
+
+---
+
 # Spinifex: An Open Source AWS-Compatible Stack for Bare-Metal, Edge, and On-Prem Deployments
 
-(Developer Preview) - Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com/) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
+Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
 
 ## What is Spinifex?
 
 Spinifex replicates essential AWS primitives—virtual machines, block volumes, and object storage—using lightweight, self-contained components that are easy to deploy and integrate.
 
-It’s designed for developers and operators who need:
+It's designed for developers and operators who need:
 
 - Cloud-like infrastructure without the cloud
 - Minimal dependencies, full control
 - Drop-in compatibility with tools like the AWS CLI, SDKs, and Terraform
 - A secure cloud environment you control and own the entire hardware, network and software stack
 
-You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren’t feasible.
+You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren't feasible.
+
+## The Platform
+
+<p align="center">
+  <img src=".github/assets/platform.svg" alt="The Mulga Platform — Hardware to Applications" width="900">
+</p>
+
+From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
 ## Core Components
 
 ### Spinifex (Compute Service – EC2 Alternative)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU’s QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
 
 - EC2-like VM management on bare metal
 - Launches with cloud-init metadata support
@@ -27,7 +57,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 ### Viperblock (Block Storage – EBS Alternative)
 
-[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It’s built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
+[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
 
 - Fast, durable virtual disks
 - Replication for resilience
@@ -43,11 +73,19 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+## Architecture at a Glance
+
+<p align="center">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture — AWS Gateway, NATS bus, service daemons" width="900">
+</p>
+
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless — scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
+
 ## Key Features
 
 AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, or SDKs you already use.
 
-- Designed for Control – Run on your own terms, whether that’s in a datacenter, remote site, or sensitive environment.
+- Designed for Control – Run on your own terms, whether that's in a datacenter, remote site, or sensitive environment.
 - Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
 - Works Offline – No reliance on centralized cloud services or external networks.
 - Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
@@ -61,11 +99,11 @@ Installation requires an Ubuntu / Debian system. See the detailed documentation
 The installation is straightforward to set up and running on a single node for testing purposes.
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 
 sudo /usr/local/share/spinifex/setup-ovn.sh --management
 
-sudo spx admin init
+sudo spx admin init --node node1 --nodes 1
 
 sudo systemctl start spinifex.target
 
@@ -76,7 +114,7 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/source-install/) documentation
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
 
 ### Component Repositories
 
@@ -87,6 +125,26 @@ Spinifex coordinates these independent components:
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
+## Spinifex UI
+
+Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+
+<p align="center">
+  <img src="docs/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+</p>
+
+The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+
+```bash
+open https://YOUR_NODE_IP:3000
+```
+
+- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
+- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+
+For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+
 ## Development Philosophy
 
 ### Built by Engineers, For Engineers

build/Dockerfile.distro

@@ -42,6 +42,7 @@ RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-s
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
+RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
 RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
 
 FROM scratch AS export

build/scripts/predastore-start.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Helper script for spinifex-predastore.service
+# Detects host, port, backend, and node-id from spinifex.toml
+# then execs spx. Falls back to single-node defaults if no config found.
+
+CONF=/etc/spinifex/spinifex.toml
+BIND="0.0.0.0:8443"
+BACKEND="filesystem"
+NODE_ID="0"
+
+if [ -f "$CONF" ]; then
+    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$H" ]; then
+        BIND="$H"
+        BACKEND="distributed"
+    fi
+
+    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
+    if [ -n "$N" ] && [ "$N" != "0" ]; then
+        NODE_ID="$N"
+    fi
+fi
+
+export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
+export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
+export SPINIFEX_PREDASTORE_BACKEND="$BACKEND"
+export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
+
+exec /usr/local/bin/spx service predastore start

build/scripts/wait-for-nats.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Wait for NATS to be reachable before starting a dependent service.
+# Reads the NATS address from spinifex.toml [nodes.*.nats] section,
+# falls back to nats.conf, then to 127.0.0.1:4222.
+
+CONF=/etc/spinifex/spinifex.toml
+NATS_CONF=/etc/spinifex/nats/nats.conf
+NATS_HOST=""
+NATS_PORT="4222"
+TIMEOUT=30
+
+# Try spinifex.toml first
+if [ -f "$CONF" ]; then
+    NATS_BIND=$(awk -F'"' '/\[nodes\..*\.nats\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$NATS_BIND" ]; then
+        NATS_HOST="${NATS_BIND%%:*}"
+        NATS_PORT="${NATS_BIND##*:}"
+    fi
+fi
+
+# Fall back to nats.conf
+if [ -z "$NATS_HOST" ] && [ -f "$NATS_CONF" ]; then
+    NATS_HOST=$(grep "^listen:" "$NATS_CONF" | sed 's/listen: *//;s/:.*//')
+fi
+
+# Final fallback
+NATS_HOST="${NATS_HOST:-127.0.0.1}"
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "Timed out waiting for NATS at $NATS_HOST:$NATS_PORT" >&2
+exit 1

build/systemd/spinifex-awsgw.service

@@ -2,21 +2,52 @@
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-gw
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
-Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-daemon.service

@@ -2,13 +2,13 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-daemon
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
@@ -17,5 +17,38 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 
+# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
+ProtectSystem=full
+PrivateDevices=no
+DevicePolicy=closed
+DeviceAllow=/dev/kvm rw
+DeviceAllow=/dev/net/tun rw
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+# AF_PACKET is required by dhcpcd (raw L2 BOOTP before the iface has an IP).
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-nats.service

@@ -5,7 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-nats
 Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
@@ -13,5 +13,36 @@ RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/nats
+ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-predastore.service

@@ -2,14 +2,14 @@
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-storage
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
-ExecStart=/usr/local/bin/spx service predastore start
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/var/lib/spinifex/predastore-start.sh
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
@@ -18,5 +18,36 @@ Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-ui.service

@@ -5,12 +5,45 @@ After=spinifex-awsgw.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-ui
 Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
+Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
 
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -2,13 +2,13 @@
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-predastore.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-viperblock
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
@@ -17,5 +17,36 @@ EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env
+ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-vpcd.service

@@ -2,18 +2,48 @@
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-vpcd
 Group=spinifex
-ExecStartPre=/bin/sh -c 'for i in $(seq 1 30); do nc -z 127.0.0.1 4222 && exit 0; sleep 1; done; exit 1'
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
+
+# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
+ProtectSystem=full
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml
+ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
 
 [Install]
 WantedBy=spinifex.target

cmd/lb-agent/main.go

@@ -0,0 +1,90 @@
+// lb-agent runs inside load balancer VMs and manages HAProxy configuration.
+//
+// It polls the AWS gateway for config updates and reports health via heartbeats.
+// All communication uses SigV4-signed HTTP requests.
+//
+// Usage:
+//
+//	lb-agent --lb-id=lb-xxxxx --gateway=https://192.168.1.33:9999 --access-key=AKIA... --secret-key=...
+//
+// Flags default to environment variables set by cloud-init:
+// LB_LB_ID, LB_GATEWAY_URL, LB_ACCESS_KEY, LB_SECRET_KEY, LB_REGION.
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mulgadc/spinifex/spinifex/lbagent"
+)
+
+func main() {
+	var (
+		lbID       string
+		gatewayURL string
+		accessKey  string
+		secretKey  string
+		region     string
+	)
+
+	hostname, _ := os.Hostname()
+
+	flag.StringVar(&lbID, "lb-id", envOrDefault("LB_LB_ID", hostname), "Load balancer ID")
+	flag.StringVar(&gatewayURL, "gateway", os.Getenv("LB_GATEWAY_URL"), "Gateway URL (e.g. https://192.168.1.33:9999)")
+	flag.StringVar(&accessKey, "access-key", os.Getenv("LB_ACCESS_KEY"), "AWS access key ID")
+	flag.StringVar(&secretKey, "secret-key", os.Getenv("LB_SECRET_KEY"), "AWS secret access key")
+	flag.StringVar(&region, "region", envOrDefault("LB_REGION", "us-east-1"), "AWS region for SigV4 signing")
+	flag.Parse()
+
+	if lbID == "" {
+		slog.Error("--lb-id is required (or set LB_LB_ID)")
+		os.Exit(1)
+	}
+	if gatewayURL == "" {
+		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
+		os.Exit(1)
+	}
+	if accessKey == "" || secretKey == "" {
+		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
+		os.Exit(1)
+	}
+
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+
+	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
+	if err != nil {
+		slog.Error("Failed to create agent", "err", err)
+		os.Exit(1)
+	}
+
+	// Start poll loop in a goroutine — it blocks until Stop is called.
+	errCh := make(chan error, 1)
+	go func() {
+		if err := agent.Start(); err != nil {
+			errCh <- err
+		}
+	}()
+
+	// Wait for shutdown signal or fatal error.
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+
+	select {
+	case sig := <-sigCh:
+		slog.Info("Received signal, shutting down", "signal", sig)
+	case err := <-errCh:
+		slog.Error("Agent error", "err", err)
+	}
+
+	agent.Stop()
+}
+
+func envOrDefault(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}

cmd/spinifex/cmd/admin.go

@@ -19,11 +19,13 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"maps"
 	"net"
 	"net/http"
 	"os"
@@ -31,7 +33,9 @@ import (
 	"path"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -158,6 +162,22 @@ var accountListCmd = &cobra.Command{
 	Run:   runAccountList,
 }
 
+var certCmd = &cobra.Command{
+	Use:   "cert",
+	Short: "Manage TLS certificates",
+	Long:  `Manage TLS certificates for the Spinifex platform.`,
+}
+
+var certRenewCmd = &cobra.Command{
+	Use:   "renew",
+	Short: "Regenerate the server certificate with current IPs",
+	Long: `Regenerate the server certificate signed by the existing CA.
+The new certificate will include all current network interface IPs
+and the machine hostname in its SANs. Use this after adding a new
+network interface or changing IP addresses.`,
+	Run: runCertRenew,
+}
+
 /*
 CLI ideas
 
@@ -195,6 +215,14 @@ func init() {
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
+
+	adminCmd.AddCommand(certCmd)
+	certCmd.AddCommand(certRenewCmd)
+
+	adminCmd.AddCommand(upgradeCmd)
+	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
+	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
 	accountCreateCmd.MarkFlagRequired("name")
 
@@ -215,12 +243,13 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
+	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
 
 	// External networking flags
 	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
-	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'static' (default, uses --external-pool range) or 'dhcp' (from router DHCP)")
+	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
 	adminInitCmd.Flags().String("external-pool", "", "External IP pool range as start-end (e.g., 192.168.1.150-192.168.1.250)")
 	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
@@ -238,6 +267,7 @@ func init() {
 	adminJoinCmd.Flags().String("cluster-bind", "", "IP address to bind NATS cluster services to (e.g., 10.11.12.1 for multi-node)")
 	adminJoinCmd.Flags().String("cluster-routes", "", "NATS cluster hosts for routing specify multiple with comma (e.g., 10.11.12.1:4248,10.11.12.2:4248 for multi-node)")
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
+	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 
@@ -249,6 +279,7 @@ func init() {
 	imagesImportCmd.Flags().String("version", "", "Specified distro version (e.g 12)")
 	imagesImportCmd.Flags().String("arch", "", "Specified distro arch (e.g aarch64, arm64, x86_64)")
 	imagesImportCmd.Flags().String("platform", "Linux/UNIX", "Specified platform (e.g Linux/UNIX, Windows)")
+	imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
 }
 
@@ -283,35 +314,54 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Determine, if name specified, or file
+	// --name pulls metadata (including Tags) from the catalog; --file supplies
+	// the local image source. When both are set, the catalog provides metadata
+	// and the file is used directly (no download). When only --file is set,
+	// metadata comes from flags and no catalog tags are applied.
 	imageName, _ := cmd.Flags().GetString("name")
+	localFile, _ := cmd.Flags().GetString("file")
+
+	if imageName == "" && localFile == "" {
+		fmt.Fprintf(os.Stderr, "Either --name or --file is required to import an image\n")
+		os.Exit(1)
+	}
 
 	if imageName != "" {
 		var exists bool
-		// Confirm the image exists
 		image, exists = utils.AvailableImages[imageName]
-
 		if !exists {
 			fmt.Fprintf(os.Stderr, "Image name not found in available images")
 			os.Exit(1)
 		}
-	} else {
-		imageFile, _ = cmd.Flags().GetString("file")
+	}
 
-		if imageFile == "" {
-			fmt.Fprintf(os.Stderr, "File required to import image")
-			os.Exit(1)
-		}
-
-		if _, err := os.Stat(imageFile); err != nil {
+	if localFile != "" {
+		if _, err := os.Stat(localFile); err != nil {
 			fmt.Fprintf(os.Stderr, "File could not be found: %s", err)
 			os.Exit(1)
 		}
+		imageFile = localFile
+		if imageName == "" {
+			image.Distro, _ = cmd.Flags().GetString("distro")
+			image.Version, _ = cmd.Flags().GetString("version")
+			image.Arch, _ = cmd.Flags().GetString("arch")
+			image.Platform, _ = cmd.Flags().GetString("platform")
+		}
+	}
 
-		image.Distro, _ = cmd.Flags().GetString("distro")
-		image.Version, _ = cmd.Flags().GetString("version")
-		image.Arch, _ = cmd.Flags().GetString("arch")
-		image.Platform, _ = cmd.Flags().GetString("platform")
+	// --tag k=v (repeatable) merges user-supplied tags into the AMI. Overrides
+	// catalog tags on key collision so operators can re-tag a known image.
+	tagFlags, _ := cmd.Flags().GetStringSlice("tag")
+	for _, kv := range tagFlags {
+		k, v, ok := strings.Cut(kv, "=")
+		if !ok || k == "" {
+			fmt.Fprintf(os.Stderr, "Invalid --tag %q: expected key=value\n", kv)
+			os.Exit(1)
+		}
+		if image.Tags == nil {
+			image.Tags = map[string]string{}
+		}
+		image.Tags[k] = v
 	}
 
 	if image.Distro == "" {
@@ -347,7 +397,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Created config directory: %s\n", imagePath)
 
 	// Next, if the file is selected to download, fetch it, extract disk image, and save to path
-	if imageName != "" {
+	if imageName != "" && localFile == "" {
 		// Download the file to the image path
 		filename := path.Base(image.URL)
 		imageFile = fmt.Sprintf("%s/%s", imagePath, filename)
@@ -412,6 +462,13 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	manifest.AMIMetadata.ImageOwnerAlias = "system"
 	manifest.AMIMetadata.VolumeSizeGiB = utils.SafeInt64ToUint64(imageStat.Size() / 1024 / 1024 / 1024)
 
+	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
+	// onto the imported AMI so the UI can filter them out.
+	if len(image.Tags) > 0 {
+		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+	}
+
 	// Volume Data
 	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
 	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
@@ -521,9 +578,9 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 // TODO: Move all logic to a module, use minimal application logic in viper commands
 func runAdminInit(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
-		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
-		fmt.Fprintln(os.Stderr, "   Run with sudo, or manually: sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates")
-		fmt.Fprintln(os.Stderr)
+		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin init' is not running as root.")
+		fmt.Fprintln(os.Stderr, "   Service user setup and CA certificate installation will be skipped.")
+		fmt.Fprintln(os.Stderr, "   For production deployments, run with sudo.")
 	}
 
 	force, _ := cmd.Flags().GetBool("force")
@@ -556,6 +613,31 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	noExternal, _ := cmd.Flags().GetBool("no-external")
 
+	// Fire telemetry in background (completes during init work, waited at end)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID:    admin.ReadMachineID(),
+				Event:        "init",
+				Region:       region,
+				AZ:           az,
+				Node:         node,
+				Nodes:        nodes,
+				BindIP:       bindIP,
+				Version:      Version,
+				ExternalMode: externalMode,
+			})
+		})
+	}
+
 	// Auto-detect network topology
 	var poolStart, poolEnd string
 	var detectedNet *admin.DetectedNetwork
@@ -613,6 +695,14 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 	if externalMode == "pool" {
+		// Resolve source: if not explicitly set, infer from whether a pool range was given
+		if externalSource == "" {
+			if externalPool != "" {
+				externalSource = "static"
+			} else {
+				externalSource = "dhcp"
+			}
+		}
 		if externalSource == "dhcp" {
 			// DHCP source: no static range needed, just gateway
 			if externalGateway == "" {
@@ -620,7 +710,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				os.Exit(1)
 			}
 		} else {
-			// Static source (default): need pool range
+			// Static source: need pool range
 			if externalPool == "" {
 				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n")
 				fmt.Fprintf(os.Stderr, "   Or use --external-source=dhcp to get IPs from router DHCP\n")
@@ -727,14 +817,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	bootstrapDir := filepath.Join(spxRoot, "awsgw")
+	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
+	if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Println("\n🔐 Generated IAM master key")
 	fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
+	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
 
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
@@ -815,7 +911,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
 	var predastoreNodeID int
@@ -875,6 +971,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   spxRoot,
+		LogDir:    LogDirFor(spxRoot),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -956,14 +1053,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(spxRoot)
 
-	// In production layout (running as root), fix ownership so the service user
-	// can read config and write data. SUDO_USER identifies the operator account.
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
 	if os.Getuid() == 0 {
-		sudoUser := os.Getenv("SUDO_USER")
-		if sudoUser != "" {
-			admin.ChownRecursive(configDir, sudoUser)
-			admin.ChownRecursive(spxRoot, sudoUser)
-		}
+		admin.SetServiceOwnership()
 	}
 
 	// Print success message
@@ -993,13 +1086,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		fmt.Fprintf(os.Stderr, "❌ Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	bootstrapDir := filepath.Join(spxRoot, "awsgw")
+	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
 	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
@@ -1083,7 +1177,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Generate multi-node predastore config
 	var predastoreNodeID int
@@ -1113,6 +1207,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   spxRoot,
+		LogDir:    LogDirFor(spxRoot),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -1188,6 +1283,12 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	admin.CreateServiceDirectories(spxRoot)
 
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
+	if os.Getuid() == 0 {
+		admin.SetServiceOwnership()
+	}
+
 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
 	time.Sleep(15 * time.Second)
@@ -1211,8 +1312,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 func runAdminJoin(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
-		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
-		fmt.Fprintln(os.Stderr)
+		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin join' is not running as root.")
+		fmt.Fprintln(os.Stderr, "   Service user setup and CA certificate installation will be skipped.")
+		fmt.Fprintln(os.Stderr, "   For production deployments, run with sudo.")
 	}
 
 	node, _ := cmd.Flags().GetString("node")
@@ -1291,9 +1393,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // formation server uses ephemeral self-signed cert
+		},
+	}
 
-	joinURL := fmt.Sprintf("http://%s/formation/join", leaderHost)
+	joinURL := fmt.Sprintf("https://%s/formation/join", leaderHost)
 	resp, err := client.Post(joinURL, "application/json", bytes.NewBuffer(reqBody))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error connecting to formation server: %v\n", err)
@@ -1322,7 +1429,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Registered with formation server (%d/%d nodes joined)\n", joinResp.Joined, joinResp.Expected)
 
 	// Poll status until formation is complete
-	statusURL := fmt.Sprintf("http://%s/formation/status", leaderHost)
+	statusURL := fmt.Sprintf("https://%s/formation/status", leaderHost)
 	var statusResp formation.StatusResponse
 
 	for {
@@ -1354,6 +1461,30 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	fmt.Printf("✅ Cluster formation complete! (%d nodes)\n\n", statusResp.Expected)
 
+	// Fire telemetry after formation (now we know the cluster topology)
+	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
+	if os.Getenv("SPX_NO_TELEMETRY") == "1" {
+		noTelemetry = true
+	}
+	var telemetryWg sync.WaitGroup
+	defer telemetryWg.Wait()
+	if !noTelemetry {
+		telemetryWg.Go(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			admin.SendTelemetry(ctx, admin.TelemetryPayload{
+				MachineID: admin.ReadMachineID(),
+				Event:     "join",
+				Region:    region,
+				AZ:        az,
+				Node:      node,
+				Nodes:     statusResp.Expected,
+				BindIP:    bindIP,
+				Version:   Version,
+			})
+		})
+	}
+
 	// Extract credentials and CA from formation status
 	creds := statusResp.Credentials
 	if creds == nil {
@@ -1403,12 +1534,17 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "❌ Error decoding master key: %v\n", err)
 		os.Exit(1)
 	}
-	if err := writeBootstrapFilesWithAdmin(configDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID, creds.AdminAccessKey, creds.AdminSecretKey); err != nil {
+	bootstrapDir := filepath.Join(dataDir, "awsgw")
+	if err := writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID, creds.AdminAccessKey, creds.AdminSecretKey); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
+	if err := writeSystemCredentials(configDir, creds.AccessKey, creds.SecretKey); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error writing system credentials: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Println("✅ IAM master key received from leader")
-	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
 	// Generate server cert signed by CA with this node's bind IP
 	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
@@ -1436,7 +1572,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Generate multi-node predastore config
 	var predastoreNodeID int
@@ -1472,6 +1608,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		Region:    creds.Region,
 		NatsToken: creds.NatsToken,
 		DataDir:   dataDir,
+		LogDir:    LogDirFor(dataDir),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -1551,6 +1688,12 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(dataDir)
 
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
+	if os.Getuid() == 0 {
+		admin.SetServiceOwnership()
+	}
+
 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
 	fmt.Printf("   Cluster: %s (%d nodes)\n", creds.ClusterName, len(statusResp.Nodes))
@@ -1558,10 +1701,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	for name, n := range statusResp.Nodes {
 		fmt.Printf("     - %s (%s)\n", name, n.BindIP)
 	}
-	fmt.Println("\n📋 Next steps:")
-	fmt.Println("   1. Start services:")
-	fmt.Println("      ./scripts/start-dev.sh")
-	fmt.Println()
 }
 
 // buildRemoteNodes converts formation NodeInfo into RemoteNode entries,
@@ -1694,7 +1833,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 			endpointHost = h
 		}
 	}
-	endpointURL := fmt.Sprintf("https://%s:9999", endpointHost)
+	endpointURL := "https://" + net.JoinHostPort(endpointHost, "9999")
 
 	credPath := filepath.Join(homeDir, ".aws", "credentials")
 	configPath := filepath.Join(homeDir, ".aws", "config")
@@ -1760,6 +1899,33 @@ func runAccountList(cmd *cobra.Command, args []string) {
 	}
 }
 
+func runCertRenew(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+
+	// Verify CA files exist.
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	if !admin.FileExists(caCertPath) || !admin.FileExists(caKeyPath) {
+		fmt.Fprintf(os.Stderr, "Error: CA files not found in %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	extraIPs, _ := cmd.Flags().GetStringSlice("extra-ip")
+	extraDNS, _ := cmd.Flags().GetStringSlice("extra-dns")
+
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("✅ Server certificate regenerated with current IPs and hostname")
+	fmt.Printf("   Certificate: %s\n", serverCertPath)
+	fmt.Println("\n⚠️  Restart awsgw and daemon services to pick up the new certificate.")
+}
+
 // writeBootstrapResult holds the admin credentials so callers can
 // write them to ~/.aws/credentials instead of the system credentials.
 type writeBootstrapResult struct {
@@ -1768,8 +1934,9 @@ type writeBootstrapResult struct {
 }
 
 // writeBootstrapFiles generates new admin credentials and writes the bootstrap
-// files (master.key + bootstrap.json). Used by init flows (single and multi-node).
-func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKey, accountID string) (*writeBootstrapResult, error) {
+// files (master.key to configDir, bootstrap.json to bootstrapDir).
+// Used by init flows (single and multi-node).
+func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, accessKey, secretKey, accountID string) (*writeBootstrapResult, error) {
 	adminAccessKey, err := admin.GenerateAWSAccessKey()
 	if err != nil {
 		return nil, fmt.Errorf("generate admin access key: %w", err)
@@ -1778,7 +1945,7 @@ func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKe
 	if err != nil {
 		return nil, fmt.Errorf("generate admin secret key: %w", err)
 	}
-	if err := writeBootstrapFilesWithAdmin(configDir, masterKey, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey); err != nil {
+	if err := writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey); err != nil {
 		return nil, err
 	}
 	return &writeBootstrapResult{
@@ -1787,9 +1954,31 @@ func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKe
 	}, nil
 }
 
+// writeSystemCredentials writes the system access key to a plaintext JSON file.
+// The daemon reads this at startup to inject credentials into ALB VM cloud-init
+// for SigV4-authenticated communication with the AWS gateway.
+func writeSystemCredentials(configDir, accessKey, secretKey string) error {
+	creds := struct {
+		AccessKeyID     string `json:"access_key_id"`
+		SecretAccessKey string `json:"secret_access_key"`
+	}{
+		AccessKeyID:     accessKey,
+		SecretAccessKey: secretKey,
+	}
+	data, err := json.MarshalIndent(creds, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshalling system credentials: %w", err)
+	}
+	return os.WriteFile(filepath.Join(configDir, "system-credentials.json"), data, 0600)
+}
+
 // writeBootstrapFilesWithAdmin writes the bootstrap files using the provided
-// admin credentials. Used by join flows where admin creds come from the leader.
-func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string) error {
+// admin credentials. master.key goes to configDir, bootstrap.json goes to
+// bootstrapDir (the awsgw data directory) so it stays outside /etc/spinifex.
+func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []byte, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string) error {
+	if err := os.MkdirAll(bootstrapDir, 0700); err != nil {
+		return fmt.Errorf("create bootstrap directory %s: %w", bootstrapDir, err)
+	}
 	if err := handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), masterKey); err != nil {
 		return fmt.Errorf("saving master key: %w", err)
 	}
@@ -1817,7 +2006,7 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey,
 		},
 	}
 
-	return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd)
+	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }
 
 // detectedWanBridge returns the OVS bridge name that OVN bridge-mappings should

cmd/spinifex/cmd/admin_upgrade.go

@@ -0,0 +1,92 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/spf13/cobra"
+)
+
+var upgradeCmd = &cobra.Command{
+	Use:   "upgrade",
+	Short: "Apply pending config migrations",
+	Long: `Apply pending configuration file migrations for upgrades between Spinifex versions.
+
+When run interactively (without --yes), shows pending migrations and prompts
+for confirmation. When called from setup.sh with --yes, applies immediately.
+
+Operators can skip migrations during install by setting INSTALL_SPINIFEX_SKIP_MIGRATE=1,
+then run 'spx admin upgrade' manually to review and apply.`,
+	Run: runAdminUpgrade,
+}
+
+func runAdminUpgrade(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+	dataDir, _ := cmd.Root().Flags().GetString("spinifex-dir")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	// Check that the installation exists.
+	spinifexToml := configDir + "/spinifex.toml"
+	if _, err := os.Stat(spinifexToml); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "No Spinifex installation found at %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	// Show current versions.
+	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading config versions: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Reading current config versions...")
+	for name, v := range versions {
+		if v == 0 {
+			fmt.Printf("  %-20s %d (no version marker)\n", name+":", v)
+		} else {
+			fmt.Printf("  %-20s %d\n", name+":", v)
+		}
+	}
+	fmt.Println()
+
+	// Check for pending migrations.
+	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking pending migrations: %v\n", err)
+		os.Exit(1)
+	}
+
+	if len(pending) == 0 {
+		fmt.Println("No pending config migrations.")
+		return
+	}
+
+	fmt.Println("Pending config migrations:")
+	for _, p := range pending {
+		fmt.Printf("  [config] %s %d → %d: %s\n", p.Target, p.FromVersion, p.ToVersion, p.Description)
+	}
+	fmt.Println()
+
+	// Prompt for confirmation unless --yes.
+	if !yes {
+		fmt.Print("Apply? [y/N] ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer != "y" && answer != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	// Apply migrations.
+	if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil {
+		fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("\nDone. Restart services to apply: sudo systemctl restart spinifex.target")
+}

cmd/spinifex/cmd/get.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -76,9 +77,15 @@ func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
 	}
 
 	// Backfill BaseDir from config file path when not set in the config.
-	// Config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
 	if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" {
-		nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
+		if isProductionLayout() {
+			// Production: BaseDir is /var/lib/spinifex (data dir).
+			// /var/lib/spinifex/config is a symlink to /etc/spinifex.
+			nodeConfig.BaseDir = DefaultDataDir()
+		} else {
+			// Dev: config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
+			nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
+		}
 		cfg.Nodes[cfg.Node] = nodeConfig
 	}
 
@@ -215,7 +222,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 				resp.Region,
 				resp.AZ,
 				formatUptime(resp.Uptime),
-				fmt.Sprintf("%d", resp.VMCount),
+				strconv.Itoa(resp.VMCount),
 				strings.Join(resp.Services, ","),
 			})
 		} else {
@@ -295,7 +302,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 			v.InstanceID,
 			v.Status,
 			v.InstanceType,
-			fmt.Sprintf("%d", v.VCPU),
+			strconv.Itoa(v.VCPU),
 			formatMemGB(v.MemoryGB),
 			v.Node,
 			v.Host,

cmd/spinifex/cmd/paths.go

@@ -7,7 +7,7 @@ import (
 )
 
 // DefaultConfigDir returns the default configuration directory.
-// Production: /etc/spinifex (when running as root AND /etc/spinifex exists)
+// Production: /etc/spinifex (when /etc/spinifex exists)
 // Development: ~/spinifex/config
 func DefaultConfigDir() string {
 	if isProductionLayout() {
@@ -26,6 +26,16 @@ func DefaultDataDir() string {
 	return filepath.Join(realUserHomeDir(), "spinifex")
 }
 
+// LogDirFor returns the log directory for a given data directory.
+// Production: /var/log/spinifex (matches systemd ReadWritePaths)
+// Development: <dataDir>/logs (supports custom per-node data dirs)
+func LogDirFor(dataDir string) string {
+	if isProductionLayout() {
+		return "/var/log/spinifex"
+	}
+	return filepath.Join(dataDir, "logs")
+}
+
 // realUserHomeDir returns the home directory of the real (non-sudo) user.
 // When running under sudo, SUDO_USER is set to the invoking user — resolve
 // their home directory so config/data land in the right place.
@@ -45,13 +55,12 @@ func DefaultConfigFile() string {
 }
 
 // isProductionLayout returns true when running in a production install.
-// Detected by: running as root AND /etc/spinifex directory already exists
-// (created by setup.sh during binary install).
+// Detected by: /etc/spinifex directory exists (created by setup.sh during
+// binary install). No root check — allows non-root users (e.g. tf-user) to
+// run CLI commands like `spx get nodes` without sudo or --config flags.
 func isProductionLayout() bool {
-	if os.Getuid() == 0 {
-		if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
-			return true
-		}
+	if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
+		return true
 	}
 	return false
 }

cmd/spinifex/cmd/paths_test.go

@@ -7,9 +7,9 @@ import (
 )
 
 func TestDefaultConfigDir_Dev(t *testing.T) {
-	// Non-root user without /etc/spinifex should get dev paths
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	// Without /etc/spinifex should get dev paths
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	homeDir, _ := os.UserHomeDir()
@@ -21,8 +21,8 @@ func TestDefaultConfigDir_Dev(t *testing.T) {
 }
 
 func TestDefaultDataDir_Dev(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	homeDir, _ := os.UserHomeDir()
@@ -34,8 +34,8 @@ func TestDefaultDataDir_Dev(t *testing.T) {
 }
 
 func TestDefaultConfigFile(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	got := DefaultConfigFile()
@@ -46,12 +46,12 @@ func TestDefaultConfigFile(t *testing.T) {
 	}
 }
 
-func TestIsProductionLayout_NonRoot(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+func TestIsProductionLayout_NoEtcSpinifex(t *testing.T) {
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	if isProductionLayout() {
-		t.Error("isProductionLayout() = true for non-root user, want false")
+		t.Error("isProductionLayout() = true without /etc/spinifex, want false")
 	}
 }

cmd/spinifex/cmd/service.go

@@ -25,6 +25,7 @@ import (
 	"github.com/mulgadc/predastore/s3"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/service"
+	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
@@ -244,13 +245,13 @@ var viperblockStartCmd = &cobra.Command{
 
 		accessKey := viper.GetString("access-key")
 		if accessKey != "" {
-			fmt.Println("Overwriting access-key:", accessKey)
+			fmt.Println("Overwriting access-key: ****")
 			nodeConfig.Predastore.AccessKey = accessKey
 		}
 
 		secretKey := viper.GetString("secret-key")
 		if secretKey != "" {
-			fmt.Println("Overwriting secret-key:", secretKey)
+			fmt.Println("Overwriting secret-key: ****")
 			nodeConfig.Predastore.SecretKey = secretKey
 		}
 
@@ -286,6 +287,7 @@ var viperblockStartCmd = &cobra.Command{
 
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
 			PluginPath: pluginPath,
 			S3Host:     nodeConfig.Predastore.Host,
 			Bucket:     nodeConfig.Predastore.Bucket,
@@ -557,6 +559,7 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
 
+		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)
 
 		if err != nil {
@@ -711,7 +714,9 @@ var vpcdStartCmd = &cobra.Command{
 			})
 		}
 
-		// Derive OVN chassis names from cluster node names (format: "chassis-{hostname}")
+		// Fallback chassis names derived from config node names. vpcd prefers
+		// discovering chassis from the OVN Southbound DB at startup; these are
+		// only used if that discovery fails.
 		var chassisNames []string
 		for nodeName := range clusterConfig.Nodes {
 			chassisNames = append(chassisNames, "chassis-"+nodeName)
@@ -729,6 +734,12 @@ var vpcdStartCmd = &cobra.Command{
 			}
 		}
 
+		baseDir := viper.GetString("base-dir")
+		if baseDir != "" {
+			fmt.Println("Overwriting vpcd base-dir to:", baseDir)
+			nodeConfig.BaseDir = baseDir
+		}
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -856,7 +867,7 @@ func init() {
 
 	serviceCmd.AddCommand(viperblockCmd)
 
-	viperblockCmd.PersistentFlags().String("s3-host", "0.0.0.0:8443", "Predastore (S3) host URI")
+	viperblockCmd.PersistentFlags().String("s3-host", "", "Predastore (S3) host URI")
 	viper.BindEnv("s3-host", "SPINIFEX_VIPERBLOCK_S3_HOST")
 	viper.BindPFlag("s3-host", predastoreCmd.PersistentFlags().Lookup("s3-host"))
 

cmd/spinifex/cmd/templates/nats.conf

@@ -1,3 +1,4 @@
+# spinifex-config-version: 1
 # NATS Server Configuration
 # Generated by Spinifex admin init
 
@@ -16,7 +17,7 @@ cluster {
 
   routes = [
 {{- range .ClusterRoutes }}
-    "nats://{{ . }}",
+    "nats-route://{{$.NatsToken}}@{{ . }}",
 {{- end }}
   ]
 }
@@ -26,7 +27,7 @@ server_name: spinifex-nats-{{.Node}}
 
 # Authorization
 authorization {
-#  token: "{{.NatsToken}}"
+  token: "{{.NatsToken}}"
 }
 
 # JetStream for persistent messaging and KV store
@@ -40,7 +41,7 @@ jetstream {
 debug: false
 trace: false
 logtime: true
-log_file: "{{.DataDir}}/logs/nats.log"
+log_file: "{{.LogDir}}/nats.log"
 
 # Limits
 max_connections: 1000

cmd/spinifex/cmd/top.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/types"
@@ -99,7 +100,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 				resp.Node,
 				fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
 				fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
-				fmt.Sprintf("%d", resp.VMCount),
+				strconv.Itoa(resp.VMCount),
 			})
 
 			for _, cap := range resp.InstanceTypes {
@@ -146,8 +147,8 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 		agg := capacityMap[name]
 		capTable = append(capTable, []string{
 			name,
-			fmt.Sprintf("%d", agg.Available),
-			fmt.Sprintf("%d", agg.VCPU),
+			strconv.Itoa(agg.Available),
+			strconv.Itoa(agg.VCPU),
 			formatMemGB(agg.MemoryGB),
 		})
 	}

docs/COMMANDS.md

@@ -54,11 +54,29 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured | **DONE** |
 | `spx admin join` | `--host` (required), `--node` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error) | **DONE** |
 
+### Version
+
+| Command | Flags | Prerequisites | Basic Logic | Status |
+|---------|-------|---------------|-------------|--------|
+| `spx version` | — | None | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) | **DONE** |
+
 ### Cluster Operations
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------|---------------|-------------|------------|--------|
-| `spx admin cluster shutdown` | — | Cluster must be running | Gracefully shuts down the entire cluster | 1. Shutdown running cluster<br>2. All nodes stop cleanly | **DONE** |
+| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Cluster must be running | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. | 1. Shutdown running cluster<br>2. All nodes stop cleanly<br>3. Force shutdown with unresponsive nodes<br>4. Dry-run prints plan | **DONE** |
+
+### Certificate Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
+
+### Upgrade Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Existing installation (`spinifex.toml` must exist in config dir) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | 1. No installation present (error, suggests `spx admin init`)<br>2. No pending migrations (prints "No pending config migrations")<br>3. Pending migrations with interactive confirmation (y/yes applies, anything else aborts)<br>4. Pending migrations with `--yes` (applies without prompt)<br>5. Migration failure surfaces error and exits non-zero | **DONE** |
 
 ### Account Management
 
@@ -81,7 +99,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `run-instances` | `--image-id`, `--instance-type`, `--count` (Min/MaxCount), `--key-name`, `--user-data`, `--subnet-id` (auto-creates ENI, assigns private IP), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName only — routes via spread or cluster strategy) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--iam-instance-profile`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | `describe-images` (AMI must exist), `create-key-pair` (optional), VPC/SG (optional) | Gateway parses AWS query → if Placement.GroupName set, looks up strategy: spread → `distributeInstancesSpread()` (1 instance per node, atomic CAS reservation), cluster → `distributeInstancesCluster()` (pin all to single node); otherwise NATS `ec2.runinstances` → daemon creates QEMU/KVM VM with viperblock-backed root volume via NBD → if SubnetId provided, auto-creates ENI with private IP → cloud-init injects user-data/keys → on termination, removes instance from placement group → returns reservation with instance ID | 1. Launch with valid AMI and key pair<br>2. Launch with invalid AMI ID (error)<br>3. Launch with block device mappings (custom volume size)<br>4. Launch multiple instances (MinCount/MaxCount)<br>5. Launch with subnet-id (auto-creates ENI)<br>6. Invalid instance type returns error<br>7. Launch with spread placement group (1 per node)<br>8. Launch with cluster placement group (all on one node)<br>9. Insufficient capacity for placement group (error) | **DONE** |
-| `describe-instances` | `--instance-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes | **DONE** |
+| `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:\*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway fans out NATS `ec2.DescribeInstances` to all nodes (no queue group) → each daemon returns local instances → gateway aggregates and returns combined list. Filters applied per-node before aggregation (reduces payload). Also applies to stopped/terminated instances via `describeInstancesFromKV()`. | 1. Describe all instances (no filter)<br>2. Describe by instance ID<br>3. Describe with filters (e.g. instance-state-name)<br>4. Instance not found returns empty set<br>5. Multi-node aggregation returns instances from all nodes<br>6. Filter by tag<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | `run-instances` (instance must exist in stopped state) | Gateway sends NATS `ec2.cmd.{instance-id}` → daemon restarts stopped QEMU process with same config → state transitions stopped→pending→running | 1. Start a stopped instance<br>2. Start already-running instance (error: IncorrectInstanceState)<br>3. Start with invalid instance ID<br>4. Verify volumes re-mount on start | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | `run-instances` (instance must be running) | Gateway sends NATS to target node → daemon issues QMP `system_powerdown` for graceful shutdown → monitors heartbeat until QEMU exits → state transitions running→stopping→stopped | 1. Graceful stop of running instance<br>2. Force stop (kills QEMU process)<br>3. Stop already-stopped instance (error)<br>4. Verify ~30s heartbeat detection | **DONE** |
 | `terminate-instances` | `--instance-ids`, `DeleteOnTermination` (per-volume flag, default true) | `--dry-run` | `run-instances` (instance must exist) | Gateway sends NATS to target node → daemon kills QEMU process → cleans up NBD mounts → deletes volumes with `DeleteOnTermination=true` via `volumeService.DeleteVolume()` (S3 cleanup of vol/, vol-efi/, vol-cloudinit/) → internal volumes (EFI, cloud-init) always cleaned up via `ebs.delete` NATS → volumes with `DeleteOnTermination=false` left in available state → state→terminated | 1. Terminate running instance<br>2. Terminate stopped instance<br>3. Terminate with DeleteOnTermination=true deletes volumes<br>4. Terminate with DeleteOnTermination=false preserves volumes<br>5. Terminate already-terminated (idempotent)<br>6. Internal volumes (EFI, cloud-init) always cleaned up<br>7. Invalid instance ID | **DONE** |
@@ -89,6 +107,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `describe-instance-types` | `--filters` (capacity filter only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, all other filters | None | Gateway fans out NATS `ec2.DescribeInstanceTypes` to all nodes → each daemon reports supported types (t3.micro/small/medium/large) with vCPU/memory specs → gateway deduplicates and returns | 1. List all instance types<br>2. Filter by specific type<br>3. Filter with `capacity=true` shows available slots<br>4. Verify vCPU/memory specs match hardware | **DONE** |
 | `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data` | `--disable-api-termination`, `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | Instance must be stopped (in NATS KV) | Gateway validates input (exactly one attribute per call, instance ID format) → NATS `ec2.ModifyInstanceAttribute` with `spinifex-workers` queue group → daemon loads stopped instance from JetStream KV → applies attribute change → writes back to KV → returns `{}` on success. **InstanceType**: updates vm.InstanceType, Config, and Instance fields; clears StateReason (enables recovery from instance-type-missing bug). **UserData**: stores decoded content in vm.UserData and re-encodes to base64 for RunInstancesInput (cloud-init on next start). No instance type pre-validation (matches AWS — invalid types accepted, fail at StartInstances time). | 1. Change instance type while stopped<br>2. Change user data while stopped<br>3. Modify running instance (error: NotFound — running instances not in KV)<br>4. Instance not found (error: InvalidInstanceID.NotFound)<br>5. Instance not stopped (error: IncorrectInstanceState)<br>6. Invalid instance type accepted (fails on start with InsufficientInstanceCapacity)<br>7. StateReason cleared on type change (recovery from capacity-unavailable)<br>8. Missing/malformed instance ID (error: InvalidInstanceID.Malformed)<br>9. No attribute set (error: InvalidParameterValue)<br>10. Multiple attributes in one call (error: InvalidParameterValue) | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest` (always returns latest), `--dry-run` | Instance must be running on a node | Gateway sends NATS `ec2.{instanceId}.GetConsoleOutput` (per-instance topic, routed to owning node) → daemon reads console log file from disk → returns last 64KB base64-encoded with timestamp. Always available regardless of serial console access setting (matches AWS behavior). | 1. Get output from running instance<br>2. Empty log file returns empty output<br>3. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, instanceInitiatedShutdownBehavior, disableApiTermination, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.DescribeInstanceAttribute` with `spinifex-workers` queue group → daemon checks running instances first (`d.Instances.VMS`), then stopped instances in JetStream KV → returns single attribute per call (matches AWS behavior). Stored attributes (`instanceType`, `userData`) return real values; unstored attributes return AWS defaults (`instanceInitiatedShutdownBehavior`=stop, `disableApiTermination`=false, etc.) | 1. Get instanceType from running instance<br>2. Get userData from stopped instance<br>3. Get default disableApiTermination<br>4. Invalid attribute name (error)<br>5. Instance not found (error: InvalidInstanceID.NotFound) | **DONE** |
+| `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | None | Gateway-only stub — returns `CpuCredits: "standard"` for each requested instance ID. No daemon round-trip. T-series credit mode is not persisted. | 1. Get credit spec for T-series instance<br>2. Multiple instance IDs | **DONE** |
 | `monitor-instances` | — | `--instance-ids` | Instance must exist | Enable basic monitoring (CPU, disk, network) → store metrics in NATS KV → return monitoring state | 1. Enable monitoring<br>2. Verify monitoring state in describe-instances | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | Instance must exist | Disable detailed monitoring → revert to basic monitoring → return monitoring state | 1. Disable monitoring<br>2. Verify monitoring state reverts in describe-instances | **NOT STARTED** |
 
@@ -97,7 +117,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format` (pem/ppk), `--tag-specifications`, `--dry-run` | None | NATS `ec2.CreateKeyPair` → daemon generates SSH keypair → stores public key in Predastore S3 (`/bucket/ec2/{name}.pub`) → returns private key material (one-time) | 1. Create RSA key pair<br>2. Create ED25519 key pair<br>3. Duplicate key name (error: InvalidKeyPair.Duplicate)<br>4. Verify key material returned only on creation | **DONE** |
-| `describe-key-pairs` | `--key-names`, `--key-pair-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty | **DONE** |
+| `describe-key-pairs` | `--key-names`, `--key-pair-ids`, `--filters` (key-pair-id, key-name, fingerprint, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeKeyPairs` → daemon lists public keys from Predastore S3 → applies filters → returns key names and fingerprints | 1. List all key pairs<br>2. Filter by key name<br>3. Filter by key pair ID<br>4. Non-existent key returns empty<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 | `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | Key must exist | NATS `ec2.DeleteKeyPair` → daemon deletes public key from Predastore S3 → returns success | 1. Delete existing key pair<br>2. Delete non-existent key (idempotent, no error)<br>3. Verify key no longer in describe-key-pairs | **DONE** |
 | `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | None | NATS `ec2.ImportKeyPair` → daemon stores provided public key in Predastore S3 → returns key name and fingerprint | 1. Import valid SSH public key<br>2. Import invalid key material (error)<br>3. Import duplicate name (error)<br>4. Verify imported key usable with run-instances | **DONE** |
 
@@ -105,7 +125,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias) | `--executable-users`, `--filters`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds and Owners → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName) | **DONE** |
+| `describe-images` | `--image-ids` (format validation only), `--owners` (self, account ID, alias), `--filters` (name, state, architecture, image-id, is-public, owner-id, description, image-type, tag:\*) | `--executable-users`, `--include-deprecated`, `--include-disabled`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeImages` → daemon reads AMI metadata from Predastore S3 buckets (ami-*) → filters by ImageIds, Owners, and Filters → returns image list with state, architecture, block device mappings | 1. List all images<br>2. Filter by image ID<br>3. Filter by owner (self/amazon)<br>4. Non-existent AMI returns empty<br>5. Verify metadata fields (architecture, state, rootDeviceName)<br>6. Filter by name wildcard<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `create-image` | `--instance-id`, `--name`, `--description`, `--tag-specifications` | `--no-reboot`, `--block-device-mappings`, `--dry-run` | Instance must exist (running or stopped) | Gateway validates input → NATS `ec2.{instanceId}.CreateImage` (per-instance topic) → daemon extracts root volume → snapshots via `ebs.snapshot` NATS (running) or offline S3 copy (stopped) → creates AMI metadata in S3 (`{amiId}/config.json`) → stores tags → returns ami-ID. Duplicate AMI name validation enforced. | 1. Create image from running instance<br>2. Create image from stopped instance<br>3. Invalid instance ID (error)<br>4. Duplicate AMI name (error)<br>5. Verify new AMI appears in describe-images<br>6. Launch new instance from created AMI | **DONE** |
 | `register-image` | — | `--name`, `--description`, `--architecture`, `--root-device-name`, `--virtualization-type`, `--block-device-mappings`, `--boot-mode`, `--ena-support`, `--sriov-net-support`, `--tpm-support`, `--imds-support` | Image data must exist in S3 | Create AMI metadata entry in Predastore → assign ami-ID → set state to available → return ami-ID | 1. Register with valid metadata<br>2. Missing required name (error)<br>3. Verify registered image in describe-images | **NOT STARTED** |
 | `deregister-image` | — | `--image-id`, `--dry-run` | AMI must exist, no instances using it | Mark AMI metadata as deregistered in S3 → AMI no longer available for new launches → existing instances unaffected | 1. Deregister existing AMI<br>2. Deregister non-existent AMI (error)<br>3. Verify deregistered AMI not in describe-images<br>4. Existing instances from AMI still run | **NOT STARTED** |
@@ -119,13 +139,13 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata) | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value | **DONE** |
+| `describe-volumes` | `--volume-ids` (fast-path lookup), `DeleteOnTermination` (from persisted VolumeMetadata), `--filters` (volume-id, status, size, volume-type, attachment.instance-id, attachment.status, attachment.device, availability-zone, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeVolumes` → daemon queries viperblock for volume metadata → applies filters → returns volume list with state, size, attachments, type, DeleteOnTermination flag | 1. List all volumes<br>2. Filter by volume ID<br>3. Filter by attachment state<br>4. Non-existent volume returns empty<br>5. DeleteOnTermination reflects persisted value<br>6. Filter by status, size, volume-type<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | Volume must exist | NATS `ec2.ModifyVolume` → daemon sends resize request to viperblock → NBD does not support live resize, instance must be stopped → returns modification state | 1. Increase volume size<br>2. Modify volume type<br>3. Decrease size (error - not supported)<br>4. Modify attached volume (requires stop/start) | **DONE** |
 | `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3 only), `--snapshot-id` (creates volume from snapshot) | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput`, `--tag-specifications` | Valid AZ configured via `spinifex init` | Gateway validates input → NATS `ec2.CreateVolume` → daemon generates vol-ID via viperblock → creates volume (empty or from snapshot) of specified size → persists config.json to Predastore S3 → returns vol-ID with state=available | 1. Create 80GB gp3 volume<br>2. Boundary sizes (1 GiB min, 16384 GiB max)<br>3. Invalid AZ (error)<br>4. Verify volume in describe-volumes<br>5. Unsupported volume type (error - only gp3)<br>6. Size out of range (error)<br>7. Create from snapshot | **DONE** |
 | `delete-volume` | `--volume-id` | `--dry-run` | Volume must exist and be detached (state=available) | Gateway validates vol- prefix → NATS `ec2.DeleteVolume` → daemon confirms state=available and no AttachedInstance → NATS `ebs.delete` to viperblockd (stops nbdkit/WAL) → deletes S3 objects under vol-id/, vol-id-efi/, vol-id-cloudinit/ → returns success | 1. Delete detached volume<br>2. Delete attached volume (error: VolumeInUse)<br>3. Delete non-existent volume (error: InvalidVolume.NotFound)<br>4. Verify volume gone from describe-volumes<br>5. Malformed volume ID (error: InvalidVolumeID.Malformed)<br>6. Double delete (idempotent NotFound) | **DONE** |
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (optional, auto-assigns `/dev/sd[f-p]`) | `--dry-run` | Volume must exist (available), instance must exist (running) | Gateway sends to `ec2.cmd.{instanceId}` → daemon validates volume (Predastore) → `ebs.mount` via NATS (viperblock starts NBD server) → QMP `blockdev-add` (nbd-{volId}) → QMP `device_add` (virtio-blk-pci, vdisk-{volId}) → three-phase rollback on failure → update EBSRequests + BlockDeviceMappings → persist to JetStream + Predastore → respond with VolumeAttachment | 1. Attach volume to running instance<br>2. Auto-assign device name<br>3. Attach already-attached volume (VolumeInUse)<br>4. Attach to non-existent instance (InvalidInstanceID.NotFound)<br>5. Attach to stopped instance (IncorrectInstanceState)<br>6. Volume not found (InvalidVolume.NotFound)<br>7. All device slots full (AttachmentLimitExceeded)<br>8. Volume persists across stop/start | **DONE** |
 | `detach-volume` | `--volume-id`, `--instance-id` (optional, resolved via DescribeVolumes), `--device` (optional cross-check), `--force` | `--dry-run` | Volume must be attached, instance must be running | Gateway resolves InstanceId if omitted (via DescribeVolumes) → sends to `ec2.cmd.{instanceId}` → daemon validates (running, attached, not boot/EFI/CloudInit, device match) → three-phase hot-unplug: QMP `device_del` (force continues on failure) → QMP `blockdev-del` (abort if fails, preserves state to prevent double-attach) → `ebs.unmount` via NATS (best-effort) → remove from EBSRequests + BlockDeviceMappings → update volume metadata to available → persist state → respond with VolumeAttachment (state=detaching) | 1. Detach with explicit InstanceId<br>2. Detach without InstanceId (gateway resolution)<br>3. Detach with correct --device cross-check<br>4. Missing VolumeId (InvalidParameterValue)<br>5. Volume not attached (IncorrectState)<br>6. Nonexistent volume (InvalidVolume.NotFound)<br>7. Nonexistent instance (InvalidInstanceID.NotFound)<br>8. Instance not running (IncorrectInstanceState)<br>9. Device mismatch (InvalidParameterValue)<br>10. Boot volume protection (OperationNotPermitted)<br>11. Force flag (continues past device_del failure)<br>12. Volume reusability (re-attach after detach) | **DONE** |
-| `describe-volume-status` | `--volume-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes | **DONE** |
+| `describe-volume-status` | `--volume-ids`, `--filters` (volume-id, volume-status.status, availability-zone) | `--max-results`, `--next-token`, `--dry-run` | None | Gateway validates vol- prefix → NATS `ec2.DescribeVolumeStatus` → daemon fetches VolumeConfig from Predastore S3 (parallel for specific IDs, sequential list-all for no IDs) → applies filters → builds VolumeStatusItem per volume (status=ok, io-enabled=passed, io-performance=not-applicable) → returns InvalidVolume.NotFound for missing explicit IDs → skips internal sub-volumes (-efi, -cloudinit) | 1. List all volume statuses<br>2. Filter by specific volume IDs (fast path)<br>3. Non-existent volume ID returns InvalidVolume.NotFound<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Internal sub-volumes excluded from listing<br>6. Nil/empty input defaults to all volumes<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
 | `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | None | Query pending/completed volume modifications → return modification state, progress, original/target size | 1. Check in-progress modification<br>2. Check completed modification<br>3. No modifications returns empty | **NOT STARTED** |
 
 ### EC2 - Snapshot Management
@@ -135,7 +155,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `create-snapshot` | `--volume-id`, `--description`, `--tag-specifications` | `--dry-run` | Volume must exist in Predastore | Gateway validates vol- prefix → NATS `ec2.CreateSnapshot` → daemon reads VolumeConfig from Predastore → generates snap-ID via viperblock → stores SnapshotConfig (metadata-only, points to source volume) as completed → returns Snapshot with state=completed, progress=100% | 1. Snapshot from valid volume<br>2. Missing volume ID (InvalidParameterValue)<br>3. Volume not found (InvalidVolume.NotFound)<br>4. Invalid volume ID format (InvalidVolume.Malformed)<br>5. Snapshot with description<br>6. Snapshot with tag specifications | **DONE** |
 | `create-snapshots` | — | `--instance-specification`, `--description`, `--tag-specifications` | Instance must exist, instance-volume attachment tracking | Create snapshots of all volumes attached to instance → return list of snapshot IDs. Blocked: requires instance-volume attachment tracking to discover which volumes to snapshot. | 1. Snapshot all volumes on instance<br>2. Instance with no volumes | **NOT STARTED** |
 | `delete-snapshot` | `--snapshot-id` | `--dry-run` | Snapshot must exist | Gateway validates snap- prefix → NATS `ec2.DeleteSnapshot` → daemon verifies snapshot exists in Predastore → lists and deletes all objects under snapshot prefix → returns success | 1. Delete existing snapshot<br>2. Delete non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing snapshot ID (InvalidParameterValue)<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
-| `describe-snapshots` | `--snapshot-ids` | `--owner-ids`, `--filters`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → filters by snapshot ID if specified → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed) | **DONE** |
+| `describe-snapshots` | `--snapshot-ids`, `--filters` (snapshot-id, status, volume-id, volume-size, owner-id, tag:\*) | `--owner-ids`, `--max-results`, `--dry-run` | None | Gateway validates snap- prefix on IDs → NATS `ec2.DescribeSnapshots` → daemon lists snap- prefixed objects in Predastore → reads SnapshotConfig for each → applies filters → returns snapshot list | 1. List all snapshots<br>2. Filter by snapshot ID<br>3. Empty snapshot list<br>4. Invalid snapshot ID format (InvalidSnapshot.Malformed)<br>5. Filter by status, volume-id, volume-size<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
 | `copy-snapshot` | `--source-snapshot-id`, `--source-region`, `--description` | `--encrypted`, `--dry-run` | Source snapshot must exist | Gateway validates snap- prefix + source region → NATS `ec2.CopySnapshot` → daemon reads source SnapshotConfig → generates new snap-ID → copies metadata (preserves tags, description override) → stores as completed → returns new snapshot ID | 1. Copy within same region<br>2. Copy non-existent snapshot (InvalidSnapshot.NotFound)<br>3. Missing source ID (InvalidParameterValue)<br>4. Missing source region (MissingParameter)<br>5. Copy preserves tags<br>6. Copy with description override | **DONE** |
 
 ### EC2 - Tags
@@ -188,7 +208,7 @@ Gateway routing and KV CRUD are wired up but no OVN/OVS integration exists. EIGW
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-egress-only-internet-gateway` | `--vpc-id`, `--tag-specifications` | `--client-token`, `--dry-run` | VPC must exist | Gateway validates VpcId → NATS `ec2.CreateEgressOnlyInternetGateway` → daemon generates eigw-ID → creates EgressOnlyIGWRecord in `spinifex-eigw` KV bucket with state=attached (always attached at creation, unlike IGW) → returns EgressOnlyInternetGateway with attachment | 1. Create with valid VpcId<br>2. Missing VpcId (error: MissingParameter)<br>3. Create with inline tags<br>4. Verify ID format eigw-{16hex} | **STARTED** (KV only, no OVN) |
 | `delete-egress-only-internet-gateway` | `--egress-only-internet-gateway-id` | `--dry-run` | EIGW must exist | NATS `ec2.DeleteEgressOnlyInternetGateway` → daemon verifies EIGW exists in KV → deletes from `spinifex-eigw` KV → returns ReturnCode=true. No detach required (unlike IGW). | 1. Delete existing EIGW<br>2. Delete non-existent EIGW (error)<br>3. Missing ID (error: MissingParameter) | **STARTED** (KV only, no OVN) |
-| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids` | `--filters`, `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list | **STARTED** (KV only, no OVN) |
+| `describe-egress-only-internet-gateways` | `--egress-only-internet-gateway-ids`, `--filters` (egress-only-internet-gateway-id, tag:\*) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeEgressOnlyInternetGateways` → daemon lists all keys from `spinifex-eigw` KV bucket → filters by ID if specified → applies filters → always includes Attachments array → returns list | 1. Describe all EIGWs<br>2. Filter by specific ID<br>3. Non-existent ID returns empty list<br>4. Unknown filter returns InvalidParameterValue | **STARTED** (KV only, no OVN) |
 
 ### EC2 - Placement Groups
 
@@ -236,67 +256,76 @@ Placement groups are stored in `spinifex-placement-groups` JetStream KV bucket,
 
 ### EC2 - Elastic IP
 
+EIP operations are conditionally available — only registered when external IPAM is configured (pool mode). EIPs are stored in `spinifex-eip-allocations` KV bucket. Association publishes `vpc.add-nat`/`vpc.delete-nat` events to vpcd for OVN NAT configuration.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `allocate-address` | — | `--domain` (vpc), `--tag-specifications`, `--dry-run` | None | NATS `ec2.AllocateAddress` → daemon allocates IP from pool → stores in JetStream KV with eipalloc-ID → return AllocationId, PublicIp, Domain | 1. Allocate EIP<br>2. Verify in describe-addresses<br>3. Allocate without --domain uses default | **NOT STARTED** |
-| `release-address` | — | `--allocation-id`, `--dry-run` | EIP must exist and be disassociated | NATS `ec2.ReleaseAddress` → daemon verifies not associated → release IP back to pool → delete from KV → return success | 1. Release disassociated EIP<br>2. Release associated EIP (error)<br>3. Release non-existent (error) | **NOT STARTED** |
-| `associate-address` | — | `--allocation-id`, `--instance-id` or `--network-interface-id`, `--dry-run` | EIP and target must exist | NATS `ec2.AssociateAddress` → daemon links EIP to instance/ENI → configure OVS DNAT/SNAT rules → return eipassoc-ID | 1. Associate with instance<br>2. Re-associate (moves EIP)<br>3. Associate non-existent EIP (error) | **NOT STARTED** |
-| `disassociate-address` | — | `--association-id`, `--dry-run` | Association must exist | NATS `ec2.DisassociateAddress` → daemon unlinks EIP from instance/ENI → remove OVS DNAT/SNAT rules → return success | 1. Disassociate existing<br>2. Disassociate non-existent (error) | **NOT STARTED** |
-| `describe-addresses` | — | `--allocation-ids`, `--filters`, `--public-ips` | None | NATS `ec2.DescribeAddresses` → daemon lists EIPs from KV → return list with association info | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP | **NOT STARTED** |
+| `allocate-address` | `--public-ipv4-pool` (optional, specific IPAM pool), `--tag-specifications` | `--domain` (hardcoded vpc), `--dry-run` | EIP service must be initialized (external IPAM configured) | Gateway validates input → NATS `ec2.AllocateAddress` with `spinifex-workers` queue group → daemon allocates IP from external IPAM pool (named or best-matching) → creates EIPRecord with state=allocated in `spinifex-eip-allocations` KV → returns AllocationId, PublicIp, Domain=vpc, PublicIpv4Pool | 1. Allocate EIP from default pool<br>2. Allocate from specific pool<br>3. Pool exhausted (InsufficientAddressCapacity)<br>4. With tag specifications<br>5. Verify in describe-addresses | **DONE** |
+| `release-address` | `--allocation-id` | `--dry-run` | EIP must exist and be disassociated | Gateway validates AllocationId non-empty → NATS `ec2.ReleaseAddress` → daemon retrieves EIP record from KV → checks state != associated (InvalidAddressLocked if still associated) → releases IP back to IPAM pool → deletes KV record → returns success | 1. Release disassociated EIP<br>2. Release associated EIP (error: InvalidAddressLocked)<br>3. Release non-existent (error: InvalidAllocationID.NotFound)<br>4. Missing AllocationId (error: MissingParameter) | **DONE** |
+| `associate-address` | `--allocation-id`, `--network-interface-id` OR `--instance-id`, `--private-ip-address` (optional override) | `--dry-run`, `--allow-reassociation` | EIP and target ENI/instance must exist | Gateway validates AllocationId + one of NetworkInterfaceId/InstanceId required → NATS `ec2.AssociateAddress` → daemon retrieves EIP → resolves target ENI (by ID or instance lookup) → generates AssociationId → updates record state=associated with ENI/instance/VPC/private IP → publishes `vpc.add-nat` event to vpcd (OVN NAT configuration) → returns AssociationId | 1. Associate with instance<br>2. Associate with specific ENI<br>3. Associate with private IP override<br>4. Missing AllocationId (MissingParameter)<br>5. Missing target (MissingParameter)<br>6. Non-existent EIP (InvalidAllocationID.NotFound)<br>7. Non-existent ENI (InvalidNetworkInterfaceID.NotFound) | **DONE** |
+| `disassociate-address` | `--association-id` | `--dry-run` | Association must exist | Gateway validates AssociationId non-empty → NATS `ec2.DisassociateAddress` → daemon scans KV to find EIP by AssociationId → publishes `vpc.delete-nat` event to vpcd → clears association fields → reverts state to allocated → updates KV → returns success | 1. Disassociate existing<br>2. Non-existent association (InvalidAssociationID.NotFound)<br>3. Missing AssociationId (MissingParameter) | **DONE** |
+| `describe-addresses` | `--allocation-ids`, `--public-ips`, `--filters` (allocation-id, public-ip, instance-id, association-id, domain, tag:\*) | `--dry-run` | None | NATS `ec2.DescribeAddresses` → daemon scans all EIP records in account → filters by AllocationIds and/or PublicIps if specified → applies filters → returns Address list with allocation, association, PublicIp, PrivateIpAddress, NetworkInterfaceId, InstanceId, Domain | 1. List all EIPs<br>2. Filter by allocation ID<br>3. Filter by public IP<br>4. Non-existent AllocationId returns error<br>5. Empty result when no EIPs<br>6. Filter by instance-id, domain<br>7. Unknown filter returns InvalidParameterValue | **DONE** |
+| `describe-addresses-attribute` | `--allocation-ids` (optional filter) | `--attribute`, `--dry-run`, `--max-results`, `--next-token` | None | Gateway validates input → NATS `ec2.DescribeAddressesAttribute` with `spinifex-workers` queue group → daemon looks up EIP records by AllocationIds (or scans all account EIPs if none specified) → returns AddressAttribute list with AllocationId and PublicIp per entry. Unlike DescribeAddresses, returns empty list (not error) for non-existent AllocationIds. PtrRecord always nil (reverse-DNS not supported). | 1. List all EIP attributes<br>2. Filter by allocation IDs<br>3. Non-existent AllocationId returns empty (not error)<br>4. Nil input returns error | **DONE** |
 
 ### EC2 - NAT Gateway
 
+NAT gateways are stored in `spinifex-vpc-nat-gateways` KV bucket. Deleted gateways are moved to `spinifex-vpc-deleted-nat-gateways` with a 1-hour TTL for Terraform polling. SNAT rule coordination uses `vpc.add-nat-gateway` and `vpc.delete-nat-gateway` NATS events (published via route table service on CreateRoute and during DeleteNatGateway respectively).
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-nat-gateway` | — | `--subnet-id`, `--allocation-id`, `--connectivity-type`, `--tag-specifications` | Subnet must exist, EIP must be allocated (for public type) | NATS `ec2.CreateNatGateway` → daemon creates NAT gateway in JetStream KV with nat-ID → configure OVS SNAT rules → state=pending→available → return NatGateway | 1. Create public NAT gateway<br>2. Create private NAT gateway (no allocation-id)<br>3. Invalid subnet (error) | **NOT STARTED** |
-| `delete-nat-gateway` | — | `--nat-gateway-id`, `--dry-run` | NAT gateway must exist | NATS `ec2.DeleteNatGateway` → daemon marks as deleting → remove OVS SNAT rules → delete from KV → return NatGatewayId | 1. Delete existing NAT gateway<br>2. Delete non-existent (error) | **NOT STARTED** |
-| `describe-nat-gateways` | — | `--nat-gateway-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeNatGateways` → daemon lists NAT gateways from KV → return list with state, subnet, addresses | 1. List all NAT gateways<br>2. Filter by ID<br>3. Filter by VPC ID | **NOT STARTED** |
+| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type` (public only), `--tag-specifications`, `--dry-run` | Subnet must exist, EIP must be allocated and unassociated | Gateway validates SubnetId + AllocationId (required) → NATS `ec2.CreateNatGateway` with `spinifex-workers` queue group → daemon verifies subnet exists in `spinifex-vpc-subnets` KV → verifies EIP exists in `spinifex-vpc-eips` KV and has no AssociationId (Resource.AlreadyAssociated if in use) → generates nat-ID → stores NatGatewayRecord in `spinifex-vpc-nat-gateways` KV with state=available → returns NatGateway with addresses | 1. Create NAT gateway (happy path)<br>2. Subnet not found (InvalidSubnetID.NotFound)<br>3. EIP not found (InvalidAllocationID.NotFound)<br>4. EIP already associated (Resource.AlreadyAssociated)<br>5. Missing SubnetId or AllocationId (MissingParameter) | **DONE** |
+| `delete-nat-gateway` | `--nat-gateway-id` | `--dry-run` | NAT gateway must exist | Gateway validates NatGatewayId (required) → NATS `ec2.DeleteNatGateway` → daemon retrieves record from KV → scans all route tables in VPC for routes targeting this NAT gateway → publishes `vpc.delete-nat-gateway` event per affected subnet (SNAT rule cleanup) → sets state=deleted → stores in `spinifex-vpc-deleted-nat-gateways` with 1-hour TTL → deletes from active KV bucket | 1. Delete existing NAT gateway<br>2. Delete non-existent (InvalidNatGatewayID.NotFound)<br>3. Deleted gateway visible when queried by ID (state=deleted)<br>4. Deleted gateway excluded from general list queries (TTL auto-expire) | **DONE** |
+| `describe-nat-gateways` | `--nat-gateway-ids`, `--filters` (vpc-id, state) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeNatGateways` → daemon scans `spinifex-vpc-nat-gateways` KV → applies NatGatewayIds filter and/or vpc-id/state filters → for specific IDs not found in active bucket, checks deleted bucket → returns InvalidNatGatewayID.NotFound if ID never existed → returns list with state, addresses, subnet, VPC | 1. List all NAT gateways<br>2. Filter by specific IDs<br>3. Filter by state<br>4. Filter by vpc-id<br>5. Non-existent ID (InvalidNatGatewayID.NotFound)<br>6. Deleted gateway visible by ID query | **DONE** |
 | `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | NAT gateway must exist (private type) | NATS `ec2.AssignPrivateNatGatewayAddress` → daemon assigns private IPs to NAT gateway → return assigned addresses | 1. Assign address<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 | `associate-nat-gateway-address` | — | `--nat-gateway-id`, `--allocation-ids` | NAT gateway and EIPs must exist | NATS `ec2.AssociateNatGatewayAddress` → daemon associates EIPs with NAT gateway → return association info | 1. Associate EIP<br>2. Missing NAT gateway ID (error) | **NOT STARTED** |
 
 ### EC2 - VPC Networking (Core)
 
-Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, and routers. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket).
+Requires OVN/OVS (`apt install openvswitch-switch ovn-host`). VPC/Subnet/ENI/SG CRUD stores metadata in NATS KV and publishes events to vpcd (the network topology daemon) which translates to OVN logical switches, ports, routers, and ACLs. Single AZ for Spinifex v1. IPAM uses CAS-based allocation from NATS KV (`spinifex-vpc-ipam` bucket). Security groups stored in `spinifex-vpc-security-groups` KV bucket with optimistic locking on rule updates.
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
 | `create-vpc` | `--cidr-block`, `--tag-specifications` | `--instance-tenancy`, `--dry-run` | None | Gateway validates CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateVpc` → daemon allocates unique VNI from `spinifex-vpc-vni-counter` KV → generates vpc-ID → stores VPCRecord in `spinifex-vpc-vpcs` KV → publishes `vpc.create` event to vpcd (creates OVN logical switch) → returns VPC with state=available | 1. Create VPC with /16 CIDR<br>2. Invalid CIDR format (error: InvalidVpcRange)<br>3. CIDR prefix outside /16–/28 (error)<br>4. Verify in describe-vpcs<br>5. Tags applied at creation | **DONE** |
 | `delete-vpc` | `--vpc-id` | `--dry-run` | VPC must be empty (no subnets) | Gateway validates VpcId → NATS `ec2.DeleteVpc` → daemon checks for dependent subnets in `spinifex-vpc-subnets` KV → returns DependencyViolation if subnets exist → deletes from `spinifex-vpc-vpcs` KV → publishes `vpc.delete` event to vpcd (removes OVN resources) | 1. Delete empty VPC<br>2. Delete VPC with subnets (DependencyViolation)<br>3. Delete non-existent VPC (error) | **DONE** |
-| `describe-vpcs` | `--vpc-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error | **DONE** |
-| `modify-vpc-attribute` | — | `--vpc-id`, `--enable-dns-support`, `--enable-dns-hostnames` | VPC must exist | NATS `ec2.ModifyVpcAttribute` → daemon updates VPC attribute in KV → return success | 1. Enable DNS support<br>2. Enable DNS hostnames<br>3. Missing VPC ID (error) | **NOT STARTED** |
+| `describe-vpcs` | `--vpc-ids`, `--filters` (vpc-id, state, cidr-block, is-default, owner-id, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeVpcs` → daemon lists all keys from `spinifex-vpc-vpcs` KV → filters by VPC IDs if specified → applies filters → returns error for non-existent requested IDs → returns VPC list with CIDR, state, CIDR associations | 1. List all VPCs<br>2. Filter by VPC ID<br>3. Non-existent VPC ID returns error<br>4. Filter by state, cidr-block, is-default<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-vpc-attribute` | `--vpc-id`, `--enable-dns-hostnames`, `--enable-dns-support`, `--enable-network-address-usage-metrics` | `--dry-run` | VPC must exist | Gateway validates VpcId non-empty → NATS `ec2.ModifyVpcAttribute` → daemon retrieves VPC record from `spinifex-vpc-vpcs` KV → updates boolean flags if provided → updates KV with optimistic locking → returns empty output | 1. Enable DNS hostnames<br>2. Enable DNS support<br>3. Enable network address usage metrics<br>4. Missing VPC ID (MissingParameter)<br>5. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
+| `describe-vpc-attribute` | `--vpc-id`, `--attribute` (enableDnsHostnames, enableDnsSupport, enableNetworkAddressUsageMetrics) | `--dry-run` | VPC must exist | Gateway validates VpcId + Attribute non-empty → NATS `ec2.DescribeVpcAttribute` → daemon retrieves VPC record → returns single attribute as AttributeBooleanValue (one attribute per call, matches AWS behavior) → validates attribute name | 1. Get enableDnsHostnames<br>2. Get enableDnsSupport<br>3. Get enableNetworkAddressUsageMetrics<br>4. Invalid attribute name (InvalidParameterValue)<br>5. Missing VPC ID (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound) | **DONE** |
 | `associate-vpc-cidr-block` | — | `--vpc-id`, `--cidr-block` | VPC must exist | NATS `ec2.AssociateVpcCidrBlock` → daemon adds secondary CIDR to VPC metadata → return association | 1. Add secondary CIDR<br>2. Overlapping CIDR (error)<br>3. Max CIDR blocks exceeded (error) | **NOT STARTED** |
 | `disassociate-vpc-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateVpcCidrBlock` → daemon removes secondary CIDR from VPC → return success | 1. Remove secondary CIDR<br>2. Remove primary CIDR (error) | **NOT STARTED** |
 | `create-subnet` | `--vpc-id`, `--cidr-block`, `--availability-zone`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates VpcId + CidrBlock (required, valid CIDR, /16–/28 prefix) → NATS `ec2.CreateSubnet` → daemon verifies parent VPC exists → validates subnet CIDR is within VPC CIDR range → checks for overlapping subnets in same VPC → calculates available IPs (total hosts - 5 AWS-reserved: .0/.1/.2/.3/.255) → generates subnet-ID → stores SubnetRecord in `spinifex-vpc-subnets` KV → publishes `vpc.create-subnet` event to vpcd → defaults AZ from config if omitted | 1. Create subnet within VPC CIDR<br>2. Subnet CIDR outside VPC range (error: InvalidSubnetRange)<br>3. Overlapping subnet CIDRs (error: InvalidSubnetConflict)<br>4. CIDR prefix outside /16–/28 (error)<br>5. Missing VPC or CIDR (error: MissingParameter) | **DONE** |
 | `delete-subnet` | `--subnet-id` | `--dry-run` | Subnet must exist | NATS `ec2.DeleteSubnet` → daemon verifies subnet exists → deletes from `spinifex-vpc-subnets` KV → publishes `vpc.delete-subnet` event to vpcd | 1. Delete existing subnet<br>2. Delete non-existent subnet (error) | **DONE** |
-| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies vpc-id filter if present → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error | **DONE** |
-| `modify-subnet-attribute` | — | `--subnet-id`, `--map-public-ip-on-launch`, `--assign-ipv6-address-on-creation` | Subnet must exist | NATS `ec2.ModifySubnetAttribute` → daemon updates subnet attribute in KV → return success | 1. Enable auto-assign public IP<br>2. Missing subnet ID (error) | **NOT STARTED** |
+| `describe-subnets` | `--subnet-ids`, `--filters` (vpc-id, subnet-id, availability-zone, cidr-block, state, default-for-az, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSubnets` → daemon lists all keys from `spinifex-vpc-subnets` KV → applies filters → filters by subnet IDs if specified → recalculates available IPs from CIDR → returns error for non-existent requested IDs | 1. List all subnets<br>2. Filter by VPC ID<br>3. Filter by subnet ID<br>4. Non-existent subnet returns error<br>5. Filter by availability-zone, cidr-block, state<br>6. Unknown filter returns InvalidParameterValue | **DONE** |
+| `modify-subnet-attribute` | `--subnet-id`, `--map-public-ip-on-launch` | `--assign-ipv6-address-on-creation`, `--dry-run` | Subnet must exist | Gateway validates SubnetId non-empty → NATS `ec2.ModifySubnetAttribute` → daemon retrieves Subnet record from `spinifex-vpc-subnets` KV → updates MapPublicIpOnLaunch flag if provided → updates KV with optimistic locking → returns empty output | 1. Enable auto-assign public IP<br>2. Disable auto-assign public IP<br>3. Missing SubnetId (MissingParameter)<br>4. Non-existent subnet (InvalidSubnetID.NotFound) | **DONE** |
 | `associate-subnet-cidr-block` | — | `--subnet-id`, `--ipv6-cidr-block` | Subnet must exist | NATS `ec2.AssociateSubnetCidrBlock` → daemon adds IPv6 CIDR to subnet → return association | 1. Add IPv6 CIDR<br>2. Missing subnet ID (error) | **NOT STARTED** |
 | `disassociate-subnet-cidr-block` | — | `--association-id` | Association must exist | NATS `ec2.DisassociateSubnetCidrBlock` → daemon removes IPv6 CIDR from subnet → return success | 1. Remove IPv6 CIDR<br>2. Invalid association (error) | **NOT STARTED** |
-| `create-security-group` | — | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | VPC must exist | Create security group metadata → create default OVS flow rules (deny all inbound, allow all outbound) → assign sg-ID | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error)<br>3. Verify default rules | **NOT STARTED** |
-| `delete-security-group` | — | `--group-id` | SG must not be in use by instances | Verify no instances reference SG → remove OVS flow rules → delete metadata | 1. Delete unused SG<br>2. Delete SG in use (error) | **NOT STARTED** |
-| `describe-security-groups` | — | `--group-ids`, `--group-names`, `--filters`, `--max-results` | None | Query NATS KV for SG metadata → return SG list with rules | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name | **NOT STARTED** |
-| `authorize-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr`, `--source-group` | SG must exist | Add inbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Duplicate rule (idempotent) | **NOT STARTED** |
-| `authorize-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist | Add outbound rule → create OVS OpenFlow rule → persist to metadata | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR | **NOT STARTED** |
-| `revoke-security-group-ingress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove inbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
-| `revoke-security-group-egress` | — | `--group-id`, `--protocol`, `--port`, `--cidr` | SG must exist, rule must exist | Remove outbound rule → delete OVS OpenFlow rule → update metadata | 1. Revoke existing rule<br>2. Revoke non-existent rule (error) | **NOT STARTED** |
+| `create-security-group` | `--group-name`, `--description`, `--vpc-id`, `--tag-specifications` | `--dry-run` | VPC must exist | Gateway validates GroupName + VpcId required → NATS `ec2.CreateSecurityGroup` → daemon verifies VPC exists → checks duplicate name in same VPC → generates sg-ID → creates SecurityGroupRecord in `spinifex-vpc-security-groups` KV with default egress rule (allow all outbound 0.0.0.0/0), no ingress rules → publishes `vpc.create-sg` event to vpcd (OVN ACL configuration) → returns GroupId | 1. Create SG in VPC<br>2. Duplicate name in same VPC (error: InvalidGroup.Duplicate)<br>3. Verify default egress rule (allow all)<br>4. Verify no default ingress rules<br>5. Missing GroupName (MissingParameter)<br>6. Non-existent VPC (InvalidVpcID.NotFound)<br>7. With tag specifications | **DONE** |
+| `delete-security-group` | `--group-id` | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.DeleteSecurityGroup` → daemon retrieves SG from KV → deletes from `spinifex-vpc-security-groups` KV → publishes `vpc.delete-sg` event to vpcd → returns success | 1. Delete existing SG<br>2. Delete non-existent SG (error: InvalidGroup.NotFound)<br>3. Missing GroupId (MissingParameter) | **DONE** |
+| `describe-security-groups` | `--group-ids`, `--filters` (vpc-id, group-name, group-id, description, ip-permission.cidr, tag:\*) | `--group-names`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeSecurityGroups` → daemon scans all SG records in account → applies filters (AND logic across filter names, OR across values) → returns SecurityGroup list with ingress/egress rules → returns error for non-existent requested GroupIds | 1. List all SGs<br>2. Filter by VPC ID<br>3. Filter by group name<br>4. Filter by GroupId<br>5. Non-existent GroupId (InvalidGroup.NotFound)<br>6. Empty result when no SGs<br>7. Filter by description, ip-permission.cidr<br>8. Unknown filter returns InvalidParameterValue | **DONE** |
+| `authorize-security-group-ingress` | `--group-id`, `--ip-permissions` (IpProtocol, FromPort, ToPort, IpRanges/CidrIp, UserIdGroupPairs/GroupId) | `--dry-run` | SG must exist | Gateway validates GroupId non-empty → NATS `ec2.AuthorizeSecurityGroupIngress` → daemon retrieves SG → converts IpPermissions to SGRule objects → appends to IngressRules → updates KV with optimistic locking (revision check) → publishes `vpc.update-sg` event to vpcd → returns true | 1. Allow SSH (port 22) from 0.0.0.0/0<br>2. Allow from specific CIDR<br>3. Allow from source security group<br>4. Missing GroupId (MissingParameter)<br>5. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `authorize-security-group-egress` | `--group-id`, `--ip-permissions` | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.AuthorizeSecurityGroupEgress` → daemon retrieves SG → appends rules to EgressRules → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Allow HTTPS outbound<br>2. Restrict to specific CIDR<br>3. Missing GroupId (MissingParameter)<br>4. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-ingress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupIngress` → daemon retrieves SG → removes matching rules from IngressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
+| `revoke-security-group-egress` | `--group-id`, `--ip-permissions` (matching rules to remove) | `--dry-run` | SG must exist | Gateway validates GroupId → NATS `ec2.RevokeSecurityGroupEgress` → daemon retrieves SG → removes matching rules from EgressRules (set difference) → updates KV with optimistic locking → publishes `vpc.update-sg` event → returns true | 1. Revoke existing rule<br>2. Missing GroupId (MissingParameter)<br>3. Non-existent SG (InvalidGroup.NotFound) | **DONE** |
 | `create-internet-gateway` | `--tag-specifications` | `--dry-run` | None | NATS `ec2.CreateInternetGateway` → daemon generates igw-ID → creates IGWRecord in `spinifex-igw` KV bucket with state=available (not attached) → returns InternetGateway with empty Attachments | 1. Create IGW<br>2. Verify in describe-internet-gateways<br>3. Tags applied at creation | **DONE** |
 | `attach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW and VPC must exist | NATS `ec2.AttachInternetGateway` → daemon validates IGW exists and is not already attached (ResourceAlreadyAssociated) → sets VpcId on record → state=attached → publishes `vpc.igw-attach` event to vpcd (creates OVN external switch, gateway port, SNAT rules) | 1. Attach IGW to VPC<br>2. Attach already-attached IGW (ResourceAlreadyAssociated)<br>3. Non-existent IGW (error)<br>4. Missing params (MissingParameter) | **DONE** |
 | `detach-internet-gateway` | `--internet-gateway-id`, `--vpc-id` | `--dry-run` | IGW must be attached to specified VPC | NATS `ec2.DetachInternetGateway` → daemon validates IGW exists and is attached to specified VPC (GatewayNotAttached if wrong VPC) → clears VpcId → state=available → publishes `vpc.igw-detach` event to vpcd (removes OVN resources) | 1. Detach attached IGW<br>2. Detach from wrong VPC (GatewayNotAttached)<br>3. Detach unattached IGW (error) | **DONE** |
 | `delete-internet-gateway` | `--internet-gateway-id` | `--dry-run` | IGW must be detached | NATS `ec2.DeleteInternetGateway` → daemon validates IGW exists and VpcId is empty (DependencyViolation if attached) → deletes from `spinifex-igw` KV | 1. Delete detached IGW<br>2. Delete attached IGW (DependencyViolation)<br>3. Delete non-existent (error) | **DONE** |
-| `describe-internet-gateways` | `--internet-gateway-ids` | `--filters`, `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info | **DONE** |
+| `describe-internet-gateways` | `--internet-gateway-ids`, `--filters` (internet-gateway-id, attachment.vpc-id, attachment.state, tag:\*) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeInternetGateways` → daemon lists all keys from `spinifex-igw` KV → filters by IGW IDs if specified → applies filters → returns list with Attachments array (if attached) | 1. List all IGWs<br>2. Filter by IGW ID<br>3. Verify attachment info<br>4. Filter by attachment.vpc-id, attachment.state<br>5. Unknown filter returns InvalidParameterValue | **DONE** |
 
 ### EC2 - Route Tables
 
+Route tables are stored in `spinifex-vpc-route-tables` JetStream KV bucket, keyed by `{accountID}.{routeTableId}`. Each route table has a Routes array (with local route pre-populated) and an Associations array. All operations validate VPC consistency. When a route targeting a NAT gateway is created, `vpc.add-nat-gateway` events are published per associated subnet for vpcd SNAT rule creation.
+
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-route-table` | — | `--vpc-id`, `--tag-specifications` | VPC must exist | NATS `ec2.CreateRouteTable` → daemon creates route table in KV with rtb-ID → add default local route for VPC CIDR → return route table | 1. Create route table<br>2. Verify default local route<br>3. Missing VPC ID (error) | **NOT STARTED** |
-| `delete-route-table` | — | `--route-table-id`, `--dry-run` | Route table must exist and not be main route table | NATS `ec2.DeleteRouteTable` → daemon verifies not main table → delete from KV → return success | 1. Delete non-main route table<br>2. Delete main route table (error)<br>3. Delete non-existent (error) | **NOT STARTED** |
-| `describe-route-tables` | — | `--route-table-ids`, `--filters`, `--max-results` | None | NATS `ec2.DescribeRouteTables` → daemon lists route tables from KV → return tables with routes | 1. List all route tables<br>2. Filter by VPC | **NOT STARTED** |
-| `create-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` or `--egress-only-internet-gateway-id` | Route table and target must exist | NATS `ec2.CreateRoute` → daemon adds route entry → configure OVS flow for destination CIDR → update metadata | 1. Add internet route via IGW<br>2. Add route to NAT gateway<br>3. Add route to EIGW<br>4. Conflicting route (error) | **NOT STARTED** |
-| `delete-route` | — | `--route-table-id`, `--destination-cidr-block` | Route must exist | NATS `ec2.DeleteRoute` → daemon removes route entry → remove OVS flow → update metadata | 1. Delete existing route<br>2. Delete non-existent route (error)<br>3. Delete local route (error) | **NOT STARTED** |
-| `replace-route` | — | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` or `--nat-gateway-id` | Route must exist | NATS `ec2.ReplaceRoute` → daemon replaces route target → update OVS flow → update metadata | 1. Replace IGW route with NAT<br>2. Replace non-existent route (error) | **NOT STARTED** |
-| `associate-route-table` | — | `--route-table-id`, `--subnet-id` or `--gateway-id` | Route table and target must exist | NATS `ec2.AssociateRouteTable` → daemon links route table to subnet/gateway → return rtbassoc-ID | 1. Associate with subnet<br>2. Missing route table ID (error) | **NOT STARTED** |
-| `disassociate-route-table` | — | `--association-id` | Association must exist, not main table association | NATS `ec2.DisassociateRouteTable` → daemon unlinks route table → return success | 1. Disassociate existing<br>2. Disassociate main table (error) | **NOT STARTED** |
+| `create-route-table` | `--vpc-id` | `--tag-specifications`, `--dry-run` | VPC must exist | Gateway validates VpcId (required) → NATS `ec2.CreateRouteTable` with `spinifex-workers` queue group → daemon verifies VPC exists in `spinifex-vpc-vpcs` KV → retrieves VPC CIDR → generates rtb-ID → creates RouteTableRecord with IsMain=false, local route pre-populated (destination=VPC CIDR, gateway="local"), empty associations → stores in `spinifex-vpc-route-tables` KV → returns RouteTable | 1. Create route table with local route (happy path)<br>2. VPC not found (InvalidVpcID.NotFound)<br>3. Missing VpcId (MissingParameter) | **DONE** |
+| `delete-route-table` | `--route-table-id` | `--dry-run` | Route table must exist, not main, no subnet associations | Gateway validates RouteTableId (required) → NATS `ec2.DeleteRouteTable` → daemon retrieves record → rejects if IsMain=true (DependencyViolation) → rejects if has non-main subnet associations (DependencyViolation) → deletes from KV → returns success | 1. Delete custom route table<br>2. Delete main route table (DependencyViolation)<br>3. Delete table with subnet associations (DependencyViolation)<br>4. Delete non-existent (InvalidRouteTableID.NotFound) | **DONE** |
+| `describe-route-tables` | `--route-table-ids`, `--filters` (vpc-id, route-table-id, association.main, association.route-table-association-id, association.subnet-id, route.destination-cidr-block, route.gateway-id) | `--max-results`, `--next-token`, `--dry-run` | None | NATS `ec2.DescribeRouteTables` → daemon scans all keys in `spinifex-vpc-route-tables` KV → filters by RouteTableIds if specified → applies all provided filters (AND logic) → returns error for non-existent requested IDs → converts records to EC2 RouteTable objects | 1. List all route tables<br>2. Filter by vpc-id<br>3. Filter by association.main (true/false)<br>4. Non-existent ID (InvalidRouteTableID.NotFound)<br>5. Empty result when no route tables | **DONE** |
+| `create-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id`, `--nat-gateway-id` | `--egress-only-internet-gateway-id`, `--vpc-peering-connection-id`, `--dry-run` | Route table and target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.CreateRoute` → daemon retrieves route table → checks for duplicate destination (RouteAlreadyExists) → validates exactly one of GatewayId/NatGatewayId provided → if GatewayId: verifies IGW exists in `spinifex-igw` KV and is attached to same VPC → if NatGatewayId: verifies NAT GW exists in `spinifex-vpc-nat-gateways` KV and is in same VPC → for NAT GW routes, publishes `vpc.add-nat-gateway` event per associated subnet → appends RouteRecord (state=active, origin=CreateRoute) → updates KV → returns Return=true | 1. Add IGW route (happy path)<br>2. Duplicate destination (RouteAlreadyExists)<br>3. IGW not found (InvalidInternetGatewayID.NotFound)<br>4. NAT GW not found (InvalidNatGatewayID.NotFound)<br>5. Gateway not in same VPC (InvalidParameterValue)<br>6. Missing RouteTableId or DestinationCidrBlock (MissingParameter) | **DONE** |
+| `delete-route` | `--route-table-id`, `--destination-cidr-block` | `--dry-run` | Route must exist | Gateway validates RouteTableId + DestinationCidrBlock (required) → NATS `ec2.DeleteRoute` → daemon retrieves route table → finds matching route → rejects if gateway is "local" (InvalidParameterValue — local routes cannot be deleted) → removes route from Routes array → updates KV → returns success | 1. Delete IGW route (happy path, local route remains)<br>2. Delete local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. Missing parameters (MissingParameter) | **DONE** |
+| `replace-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id` | `--nat-gateway-id`, `--dry-run` | Route must exist, new target must exist in same VPC | Gateway validates RouteTableId + DestinationCidrBlock + GatewayId (required) → NATS `ec2.ReplaceRoute` → daemon retrieves route table → finds matching route → rejects if local route → verifies IGW exists and is attached to same VPC → updates route GatewayId, clears NatGatewayId, sets state=active → updates KV → returns success | 1. Replace route target (IGW swap)<br>2. Replace local route (InvalidParameterValue)<br>3. Route not found (InvalidRoute.NotFound)<br>4. IGW not found (InvalidInternetGatewayID.NotFound) | **DONE** |
+| `associate-route-table` | `--route-table-id`, `--subnet-id` | `--gateway-id`, `--dry-run` | Route table and subnet must exist in same VPC | Gateway validates RouteTableId + SubnetId (required) → NATS `ec2.AssociateRouteTable` → daemon retrieves route table → verifies subnet exists and is in same VPC → scans ALL route tables in VPC to check subnet isn't already explicitly associated (Resource.AlreadyAssociated) → generates rtbassoc-ID → appends AssociationRecord (Main=false) → updates KV → returns AssociationId with state=associated | 1. Associate subnet (happy path)<br>2. Duplicate subnet association (Resource.AlreadyAssociated)<br>3. Subnet not found (InvalidSubnetID.NotFound)<br>4. Subnet in different VPC (InvalidParameterValue)<br>5. Missing parameters (MissingParameter) | **DONE** |
+| `disassociate-route-table` | `--association-id` | `--dry-run` | Association must exist, not main table association | Gateway validates AssociationId (required) → NATS `ec2.DisassociateRouteTable` → daemon scans ALL route tables in account to find matching AssociationId → rejects if Main=true (InvalidParameterValue — main associations are immutable) → removes association from Associations array → updates KV → returns success | 1. Disassociate subnet (happy path)<br>2. Disassociate main table (InvalidParameterValue)<br>3. Association not found (InvalidAssociationID.NotFound) | **DONE** |
+| `replace-route-table-association` | `--association-id`, `--route-table-id` | `--dry-run` | Association and new route table must exist | Gateway validates AssociationId + RouteTableId (required) → NATS `ec2.ReplaceRouteTableAssociation` → daemon verifies new route table exists → scans all route tables to find matching AssociationId → if Main=true: atomically swaps IsMain flag between old and new table → two-phase commit: removes association from old table, creates new AssociationId on new table → compensating write on failure (logs CRITICAL if compensation fails) → returns new AssociationId with state=associated | 1. Move subnet between tables (happy path)<br>2. Replace main table association (swaps IsMain flags)<br>3. Association not found (InvalidAssociationID.NotFound)<br>4. Route table not found (InvalidRouteTableID.NotFound) | **DONE** |
 
 ### EC2 - VPC Peering
 
@@ -376,7 +405,7 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `describe-network-interfaces` | `--network-interface-ids`, `--filters` (subnet-id, vpc-id, attachment.instance-id) | `--max-results`, `--dry-run` | None | NATS `ec2.DescribeNetworkInterfaces` → daemon lists all keys from `spinifex-vpc-enis` KV → applies filters (subnet-id, vpc-id, attachment.instance-id) → filters by ENI IDs if specified → returns error for non-existent requested IDs | 1. List all ENIs<br>2. Filter by subnet-id<br>3. Filter by vpc-id<br>4. Filter by attachment.instance-id<br>5. Non-existent ENI returns error | **DONE** |
 | `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | ENI and instance must exist | Internal operation used by `run-instances --subnet-id`. Marks ENI as "in-use", sets instance attachment, generates attachment ID. No standalone API endpoint yet. | 1. Attach ENI to instance<br>2. Already attached (error)<br>3. Missing device index (error) | **NOT STARTED** (internal only) |
 | `detach-network-interface` | — | `--attachment-id`, `--force` | ENI must be attached | Internal operation used by terminate-instances. Marks ENI as "available", clears attachment info. No standalone API endpoint yet. | 1. Detach ENI<br>2. Force detach<br>3. Not attached (error) | **NOT STARTED** (internal only) |
-| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--source-dest-check`, `--groups` | ENI must exist | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon updates ENI attributes in KV → return success | 1. Modify description<br>2. Toggle source/dest check<br>3. Update security groups | **NOT STARTED** |
+| `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--groups` | ENI must exist; at least one attribute required | NATS `ec2.ModifyNetworkInterfaceAttribute` → daemon validates ENI exists → updates description and/or security groups in `spinifex-vpc-enis` KV (optimistic locking) → return success | 1. Modify description<br>2. Update security groups<br>3. No attributes (InvalidParameterValue)<br>4. Non-existent ENI (error) | **DONE** |
 | `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` or `--secondary-private-ip-address-count` | ENI must exist | NATS `ec2.AssignPrivateIpAddresses` → daemon allocates IPs from subnet pool → assign to ENI → return assigned IPs | 1. Assign specific IPs<br>2. Assign by count<br>3. Cannot specify both (error) | **NOT STARTED** |
 | `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | IPs must be assigned to ENI | NATS `ec2.UnassignPrivateIpAddresses` → daemon releases IPs → return success | 1. Unassign secondary IP<br>2. Unassign primary IP (error) | **NOT STARTED** |
 
@@ -409,19 +438,6 @@ ENIs are managed via `spinifex-vpc-enis` KV bucket with CAS-based IPAM allocatio
 | `list-snapshot-blocks` | — | `--snapshot-id`, `--max-results`, `--next-token` | Snapshot must exist | List all non-empty blocks in snapshot with tokens and sizes | 1. List blocks of populated snapshot<br>2. List blocks of empty snapshot | **NOT STARTED** |
 | `list-changed-blocks` | — | `--second-snapshot-id`, `--first-snapshot-id`, `--max-results` | Snapshots must exist | Compare two snapshots → return list of differing block indexes | 1. Compare parent and child snapshots<br>2. Compare unrelated snapshots | **NOT STARTED** |
 
-### S3 (via Predastore)
-
-Currently S3 operations are handled directly by Predastore. Consider moving control/data plane to Spinifex format for AWS gateway integration.
-
-| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
-|---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `s3 mb` (CreateBucket) | — | `s3://bucket-name` | None | NATS `s3.createbucket` → Predastore creates bucket → return success | 1. Create valid bucket<br>2. Duplicate bucket name (error)<br>3. Invalid bucket name (error) | **NOT STARTED** (Predastore handles directly) |
-| `s3 rb` (DeleteBucket) | — | `s3://bucket-name`, `--force` | Bucket must exist | NATS `s3.deletebucket` → verify bucket empty (or force) → Predastore deletes bucket | 1. Delete empty bucket<br>2. Delete non-empty bucket (error)<br>3. Force delete non-empty bucket | **NOT STARTED** |
-| `s3 ls` (ListBuckets/Objects) | — | `s3://bucket/prefix`, `--recursive` | None | NATS `s3.listobjects` → Predastore lists objects with prefix → return list | 1. List all buckets<br>2. List objects in bucket<br>3. List with prefix filter | **NOT STARTED** |
-| `s3 cp` (PutObject/GetObject) | — | `source dest`, `--recursive`, `--acl` | Bucket must exist | NATS `s3.putobject`/`s3.getobject` → Predastore stores/retrieves with Reed-Solomon encoding | 1. Upload single file<br>2. Download single file<br>3. Recursive upload/download<br>4. Large file (multipart) | **NOT STARTED** |
-| `s3 rm` (DeleteObject) | — | `s3://bucket/key`, `--recursive` | Object must exist | NATS `s3.deleteobject` → Predastore deletes object shards | 1. Delete single object<br>2. Recursive delete<br>3. Delete non-existent (idempotent) | **NOT STARTED** |
-| `s3 sync` | — | `source dest`, `--delete`, `--exclude`, `--include` | Bucket must exist | Compare source and dest → upload changed/new files → optionally delete removed files | 1. Sync local dir to S3<br>2. Sync with --delete<br>3. Sync with exclude filter | **NOT STARTED** |
-
 ### IAM - User Management
 
 All IAM operations are account-scoped. Users, policies, and access keys are namespaced per account. The gateway extracts the account ID from the authenticated access key via `c.Locals("sigv4.accountId")` and passes it to all IAM service methods. Root user (account `000000000000`) bypasses policy evaluation entirely.
@@ -477,5 +493,146 @@ The gateway evaluates IAM policies on every request (except root user bypass). P
 | Explicit deny | Deny statements override Allow (deny wins) | **DONE** |
 | Wildcard matching | `*` matches all, `ec2:*` matches all EC2, `ec2:Describe*` matches prefix | **DONE** |
 | Account-scoped evaluation | Policies resolved from caller's account context | **DONE** |
-| EC2 action mapping | 54 EC2 actions mapped to IAM policy strings | **DONE** |
+| EC2 action mapping | All registered EC2 actions mapped to IAM policy strings | **DONE** |
 | IAM action mapping | 16 IAM actions mapped to IAM policy strings | **DONE** |
+
+### IAM - Roles
+
+IAM roles are required for instance profiles, cross-account access, and service-linked permissions. Without roles, workloads must use static access keys. Roles should be stored in a `spinifex-iam-roles` KV bucket keyed by `{accountID}.{roleName}`. Trust policies (AssumeRolePolicyDocument) define who can assume the role.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-role` | — | `--role-name`, `--assume-role-policy-document`, `--path`, `--description`, `--max-session-duration`, `--permissions-boundary`, `--tags` | None | IAMService.CreateRole(accountID, input) → validate RoleName + AssumeRolePolicyDocument (required) → validate trust policy JSON → generate AROA-prefixed RoleId → store RoleRecord in `spinifex-iam-roles` KV at `{accountID}.{roleName}` → ARN: `arn:aws:iam::{accountID}:role/{path}{roleName}` → return Role | 1. Create with valid trust policy<br>2. Malformed trust policy (MalformedPolicyDocument)<br>3. Duplicate name (EntityAlreadyExists)<br>4. Missing RoleName (InvalidInput)<br>5. With path and description | **NOT STARTED** |
+| `get-role` | — | `--role-name` | Role must exist | IAMService.GetRole(accountID, input) → reads from KV at `{accountID}.{roleName}` → returns Role with ARN, Path, AssumeRolePolicyDocument, CreateDate | 1. Get existing role<br>2. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-roles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListRoles(accountID, input) → prefix-scans KV for `{accountID}.*` → filters by path prefix → returns role list scoped to caller's account | 1. List all roles<br>2. List with path prefix<br>3. Empty list<br>4. Account isolation | **NOT STARTED** |
+| `delete-role` | — | `--role-name` | Role must exist, no attached policies, no instance profiles | IAMService.DeleteRole(accountID, input) → verify no attached policies (DeleteConflict) → verify not in any instance profile (DeleteConflict) → delete from KV | 1. Delete role with no attachments<br>2. Delete role with policies (DeleteConflict)<br>3. Delete role in instance profile (DeleteConflict)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-role` | — | `--role-name`, `--description`, `--max-session-duration` | Role must exist | IAMService.UpdateRole(accountID, input) → update description and/or session duration in KV → return success | 1. Update description<br>2. Update max session duration<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `update-assume-role-policy` | — | `--role-name`, `--policy-document` | Role must exist | IAMService.UpdateAssumeRolePolicy(accountID, input) → validate trust policy JSON → update AssumeRolePolicyDocument in KV → return success | 1. Update trust policy<br>2. Malformed policy (MalformedPolicyDocument)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `attach-role-policy` | — | `--role-name`, `--policy-arn` | Role and policy must exist | IAMService.AttachRolePolicy(accountID, input) → verify policy exists → add ARN to role's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent role (NoSuchEntity)<br>4. Non-existent policy (NoSuchEntity) | **NOT STARTED** |
+| `detach-role-policy` | — | `--role-name`, `--policy-arn` | Policy must be attached | IAMService.DetachRolePolicy(accountID, input) → remove ARN from role's AttachedPolicies | 1. Detach attached policy<br>2. Not attached (NoSuchEntity)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-role-policies` | — | `--role-name`, `--path-prefix`, `--max-items`, `--marker` | Role must exist | IAMService.ListAttachedRolePolicies(accountID, input) → reads role's AttachedPolicies list → returns ARNs and names | 1. List attached policies<br>2. No policies (empty list)<br>3. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Instance Profiles
+
+Instance profiles are containers for IAM roles that allow EC2 instances to assume a role and obtain temporary credentials. Required for `run-instances --iam-instance-profile`. Stored in `spinifex-iam-instance-profiles` KV bucket keyed by `{accountID}.{profileName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-instance-profile` | — | `--instance-profile-name`, `--path`, `--tags` | None | IAMService.CreateInstanceProfile(accountID, input) → validate name (required) → generate AIPA-prefixed InstanceProfileId → store in `spinifex-iam-instance-profiles` KV with empty Roles array → ARN: `arn:aws:iam::{accountID}:instance-profile/{path}{name}` → return InstanceProfile | 1. Create profile<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-instance-profile` | — | `--instance-profile-name` | Profile must exist | IAMService.GetInstanceProfile(accountID, input) → reads from KV → returns InstanceProfile with Roles array | 1. Get existing profile<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-instance-profiles` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListInstanceProfiles(accountID, input) → prefix-scans KV → returns list | 1. List all profiles<br>2. Empty list | **NOT STARTED** |
+| `list-instance-profiles-for-role` | — | `--role-name`, `--max-items`, `--marker` | Role must exist | IAMService.ListInstanceProfilesForRole(accountID, input) → scan all profiles → filter where Roles contains role name → return list | 1. List profiles for role<br>2. Role not in any profile (empty) | **NOT STARTED** |
+| `delete-instance-profile` | — | `--instance-profile-name` | Profile must exist, Roles must be empty | IAMService.DeleteInstanceProfile(accountID, input) → verify Roles empty (DeleteConflict if not) → delete from KV | 1. Delete empty profile<br>2. Delete profile with role (DeleteConflict)<br>3. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-role-to-instance-profile` | — | `--instance-profile-name`, `--role-name` | Both must exist, max 1 role per profile | IAMService.AddRoleToInstanceProfile(accountID, input) → verify profile exists → verify role exists → check Roles array is empty (LimitExceeded if role already present) → append role → update KV | 1. Add role to empty profile<br>2. Profile already has role (LimitExceeded)<br>3. Non-existent profile (NoSuchEntity)<br>4. Non-existent role (NoSuchEntity) | **NOT STARTED** |
+| `remove-role-from-instance-profile` | — | `--instance-profile-name`, `--role-name` | Role must be in profile | IAMService.RemoveRoleFromInstanceProfile(accountID, input) → remove role from Roles array → update KV | 1. Remove role<br>2. Role not in profile (NoSuchEntity) | **NOT STARTED** |
+
+### IAM - Groups
+
+Groups provide a way to attach policies to multiple users. Stored in `spinifex-iam-groups` KV bucket keyed by `{accountID}.{groupName}`.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-group` | — | `--group-name`, `--path` | None | IAMService.CreateGroup(accountID, input) → validate GroupName (required) → generate AGPA-prefixed GroupId → store in KV → return Group | 1. Create group<br>2. Duplicate name (EntityAlreadyExists)<br>3. Missing name (InvalidInput) | **NOT STARTED** |
+| `get-group` | — | `--group-name` | Group must exist | IAMService.GetGroup(accountID, input) → reads group from KV → returns Group with member Users list | 1. Get existing group<br>2. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `list-groups` | — | `--path-prefix`, `--max-items`, `--marker` | None | IAMService.ListGroups(accountID, input) → prefix-scans KV → returns list | 1. List all groups<br>2. Empty list | **NOT STARTED** |
+| `delete-group` | — | `--group-name` | Group must exist, no members, no attached policies | IAMService.DeleteGroup(accountID, input) → verify no members → verify no attached policies → delete from KV | 1. Delete empty group<br>2. Delete group with members (DeleteConflict)<br>3. Delete group with policies (DeleteConflict)<br>4. Non-existent (NoSuchEntity) | **NOT STARTED** |
+| `add-user-to-group` | — | `--group-name`, `--user-name` | Both must exist | IAMService.AddUserToGroup(accountID, input) → verify group + user exist → add user to group's Members list → idempotent | 1. Add user<br>2. Already a member (idempotent)<br>3. Non-existent group (NoSuchEntity)<br>4. Non-existent user (NoSuchEntity) | **NOT STARTED** |
+| `remove-user-from-group` | — | `--group-name`, `--user-name` | User must be in group | IAMService.RemoveUserFromGroup(accountID, input) → remove user from Members list → update KV | 1. Remove member<br>2. Not a member (NoSuchEntity) | **NOT STARTED** |
+| `list-groups-for-user` | — | `--user-name`, `--max-items`, `--marker` | User must exist | IAMService.ListGroupsForUser(accountID, input) → scan all groups → filter where Members contains user → return list | 1. List groups for user<br>2. User in no groups (empty) | **NOT STARTED** |
+| `attach-group-policy` | — | `--group-name`, `--policy-arn` | Group and policy must exist | IAMService.AttachGroupPolicy(accountID, input) → verify policy exists → add ARN to group's AttachedPolicies → idempotent | 1. Attach policy<br>2. Already attached (idempotent)<br>3. Non-existent group (NoSuchEntity) | **NOT STARTED** |
+| `detach-group-policy` | — | `--group-name`, `--policy-arn` | Policy must be attached | IAMService.DetachGroupPolicy(accountID, input) → remove ARN from AttachedPolicies | 1. Detach policy<br>2. Not attached (NoSuchEntity) | **NOT STARTED** |
+| `list-attached-group-policies` | — | `--group-name`, `--path-prefix`, `--max-items`, `--marker` | Group must exist | IAMService.ListAttachedGroupPolicies(accountID, input) → reads group's AttachedPolicies → returns ARNs and names | 1. List attached policies<br>2. No policies (empty) | **NOT STARTED** |
+
+### STS (Security Token Service)
+
+STS provides temporary credentials and identity verification. `GetCallerIdentity` is called by every AWS SDK/CLI invocation. `AssumeRole` is required once IAM roles are implemented. STS needs its own gateway service endpoint (`sts` service in the `supportedServices` map) with a dedicated action map.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `get-caller-identity` | — | *(no flags)* | Valid SigV4 credentials | Gateway extracts accountID and username from authenticated request → returns Account, Arn (`arn:aws:iam::{accountID}:user/{userName}` or `arn:aws:sts::{accountID}:assumed-role/{roleName}/{sessionName}`), UserId. Note: a Spinifex-custom version exists at `spinifex.GetCallerIdentity` but needs a proper STS endpoint. | 1. Get identity for IAM user<br>2. Get identity for assumed role<br>3. Get identity for root user<br>4. Invalid credentials (AuthFailure) | **NOT STARTED** |
+| `assume-role` | — | `--role-arn`, `--role-session-name`, `--duration-seconds`, `--policy`, `--external-id`, `--serial-number`, `--token-code`, `--tags`, `--transitive-tag-keys` | Role must exist, caller must be allowed by trust policy | Gateway validates RoleArn + RoleSessionName (required) → looks up role → evaluates trust policy (AssumeRolePolicyDocument) against caller → generates temporary credentials (AccessKeyId with ASIA prefix, SecretAccessKey, SessionToken) → credentials expire after DurationSeconds (default 3600, max 43200) → return Credentials + AssumedRoleUser (ARN, AssumedRoleId) | 1. Assume role with valid trust policy<br>2. Trust policy denies caller (AccessDenied)<br>3. Non-existent role (NoSuchEntity)<br>4. Expired session token rejected at SigV4<br>5. Custom duration<br>6. Session tags | **NOT STARTED** |
+| `get-session-token` | — | `--duration-seconds`, `--serial-number`, `--token-code` | Valid IAM user credentials (not role) | Gateway validates caller is IAM user (not assumed role) → generates temporary credentials (ASIA prefix) → session inherits user's permissions → return Credentials | 1. Get token for IAM user<br>2. Duration within range<br>3. Reject if caller is assumed role | **NOT STARTED** |
+
+### ELBv2 (Elastic Load Balancing v2 — Application & Network Load Balancer)
+
+All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key format: `lb.{lbId}`, `tg.{tgId}`, `listener.{listenerId}`. The data plane uses a system-managed LB VM running HAProxy, launched automatically during CreateLoadBalancer. HAProxy configuration is pushed via NATS (`elbv2.lb.{lbId}.config` topic) whenever listeners or targets change. Agent readiness is polled via `elbv2.lb.{lbId}.ping`.
+
+#### ELBv2 - Load Balancers
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-load-balancer` | `--name`, `--subnets`, `--security-groups`, `--scheme` (internet-facing/internal), `--tags`, `--ip-address-type` (ipv4 only) | `--type` (hardcoded application), `--customer-owned-ipv4-pool`, `--dry-run` | None (subnets optional) | Gateway validates Name (required) → NATS `elbv2.CreateLoadBalancer` with `spinifex-workers` queue group → daemon validates scheme (internet-facing or internal, InvalidScheme if other) → checks duplicate name in account (DuplicateLoadBalancerName) → generates LB ID → builds DNS name `{name}-{lbId}.{region}.elb.spinifex.local` → if subnets provided, creates one ENI per subnet with `spinifex:managed-by=elbv2` tag (rolls back on failure) → if system AMI available, launches sys.micro LB VM with cloud-init running `/usr/local/bin/lb-agent` → state=provisioning → background goroutine polls agent ping for ~5min → on agent ready: state=active, on timeout: state=failed → if no VM launcher: state=active immediately → persists to KV → returns LoadBalancer with ARN, DNSName, State | 1. Create with name (happy path)<br>2. Internal scheme<br>3. Invalid scheme (InvalidScheme)<br>4. Duplicate name (DuplicateLoadBalancerName)<br>5. Missing name (MissingParameter)<br>6. With tags<br>7. With subnets (ENI creation) | **DONE** |
+| `delete-load-balancer` | `--load-balancer-arn` | `--dry-run` | LB must exist | Gateway validates LoadBalancerArn (required) → NATS `elbv2.DeleteLoadBalancer` → daemon verifies LB exists and belongs to account → terminates LB VM (async background) → deletes all listeners for this LB → deletes system-managed ENIs → deletes LB record from KV → returns success | 1. Delete existing LB<br>2. Not found (LoadBalancerNotFound)<br>3. Cascades listener cleanup<br>4. ENI cleanup | **DONE** |
+| `describe-load-balancers` | `--load-balancer-arns`, `--names` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeLoadBalancers` → daemon lists all LB records from KV → filters by account ID → applies ARN filter and/or Name filter → returns LoadBalancer list with State, DNSName, Scheme, VpcId, AvailabilityZones | 1. List all LBs<br>2. Filter by name<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Target Groups
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-target-group` | `--name`, `--protocol` (HTTP), `--port` (default 80), `--vpc-id`, `--target-type` (instance only), `--health-check-protocol`, `--health-check-port`, `--health-check-path` (default /), `--health-check-interval-seconds` (default 30), `--health-check-timeout-seconds` (default 5), `--healthy-threshold-count` (default 5), `--unhealthy-threshold-count` (default 2), `--matcher` (HttpCode, default 200), `--tags` | `--health-check-enabled`, `--protocol-version`, `--ip-address-type`, `--dry-run` | None | Gateway validates Name (required) → NATS `elbv2.CreateTargetGroup` → daemon checks duplicate name in same VPC (DuplicateTargetGroupName) → generates TG ID → creates HealthCheckConfig with provided values or defaults → stores record with empty Targets array → returns TargetGroup with ARN | 1. Create with defaults<br>2. Custom health check<br>3. Duplicate name (DuplicateTargetGroupName)<br>4. Missing name (MissingParameter) | **DONE** |
+| `delete-target-group` | `--target-group-arn` | `--dry-run` | TG must exist, not referenced by listener | Gateway validates TargetGroupArn (required) → NATS `elbv2.DeleteTargetGroup` → daemon verifies TG exists → scans all listeners to check if any DefaultActions reference this TG (TargetGroupInUse if referenced) → deletes from KV | 1. Delete TG<br>2. TG in use by listener (TargetGroupInUse)<br>3. Not found (TargetGroupNotFound) | **DONE** |
+| `describe-target-groups` | `--target-group-arns`, `--names`, `--load-balancer-arn` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeTargetGroups` → daemon lists all TG records → filters by account → applies ARN/Name filters → if LoadBalancerArn specified, lists listeners for that LB and includes only TGs referenced by those listeners → returns TargetGroup list | 1. List all TGs<br>2. Filter by LB ARN (via listener lookup)<br>3. Account isolation | **DONE** |
+
+#### ELBv2 - Targets
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `register-targets` | `--target-group-arn`, `--targets` (Id=instance-id, Port=optional override) | `--dry-run` | TG must exist, instances should exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.RegisterTargets` → daemon fetches TG → deduplicates against existing targets (key: `{id}:{port}`, idempotent) → resolves instance ID → private IP via VPC ENI lookup → adds target with state=initial → persists to KV → calls `pushConfigForTargetGroup()` to regenerate HAProxy config on all LBs using this TG → returns success | 1. Register targets<br>2. Idempotent re-registration<br>3. TG not found (TargetGroupNotFound)<br>4. Missing ARN or targets (MissingParameter) | **DONE** |
+| `deregister-targets` | `--target-group-arn`, `--targets` (Id, Port) | `--dry-run` | TG must exist | Gateway validates TargetGroupArn + Targets (required) → NATS `elbv2.DeregisterTargets` → daemon fetches TG → builds removal set (key: `{id}:{port}`) → filters Targets array to keep only non-matching → persists to KV → calls `pushConfigForTargetGroup()` to update HAProxy config → returns success | 1. Deregister targets<br>2. TG not found (TargetGroupNotFound)<br>3. Missing parameters (MissingParameter) | **DONE** |
+| `describe-target-health` | `--target-group-arn`, `--targets` (optional filter to specific targets) | `--include` | TG must exist | Gateway validates TargetGroupArn (required) → NATS `elbv2.DescribeTargetHealth` → daemon fetches TG → builds filter set from Targets param (if specified) → for each target, returns TargetHealthDescription with Target (Id, Port) and TargetHealth (State, Description). Note: health state is stored in records, not actively polled — HAProxy runs its own independent health checks. | 1. Describe all targets<br>2. Filter to specific targets<br>3. TG not found (TargetGroupNotFound) | **DONE** |
+
+#### ELBv2 - Listeners
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-listener` | `--load-balancer-arn`, `--default-actions` (Type=forward, TargetGroupArn), `--protocol` (HTTP, default), `--port` (default 80) | `--ssl-policy`, `--certificates`, `--alpn-policy`, `--mutual-authentication`, `--dry-run` | LB must exist | Gateway validates LoadBalancerArn + DefaultActions (required) → NATS `elbv2.CreateListener` → daemon verifies LB exists → checks for duplicate port on same LB (DuplicateListener) → generates listener ID → builds ARN → stores ListenerRecord in KV → calls `pushConfig(lb)` to regenerate HAProxy config (creates frontend for listener port, backend for target group with health checks) → returns Listener with ARN, Protocol, Port, DefaultActions | 1. Create listener<br>2. Duplicate port (DuplicateListener)<br>3. LB not found (LoadBalancerNotFound)<br>4. Graceful handling when NATS unavailable | **DONE** |
+| `delete-listener` | `--listener-arn` | `--dry-run` | Listener must exist | Gateway validates ListenerArn (required) → NATS `elbv2.DeleteListener` → daemon verifies listener exists and belongs to account → deletes from KV → fetches parent LB → calls `pushConfig(lb)` to regenerate HAProxy config without this listener → returns success | 1. Delete listener<br>2. Not found (ListenerNotFound) | **DONE** |
+| `describe-listeners` | `--load-balancer-arn`, `--listener-arns` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeListeners` → daemon lists listeners → if LoadBalancerArn specified, filters to that LB → applies ARN filter → account isolation → returns Listener list | 1. Filter by LB ARN<br>2. Account isolation | **DONE** |
+
+#### ELBv2 - Tags
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-tags` | `--resource-arns` (loadbalancer, targetgroup, listener) | — | Resource(s) must exist | Gateway validates non-empty ResourceArns → NATS `elbv2.DescribeTags` → daemon parses each ARN's resource segment to dispatch by type → looks up record (LB/TG/listener) → enforces account isolation (cross-account → not-found) → returns TagDescriptions in input order with sorted-by-key Tag list. Listener records don't store tags yet, so they always return an empty Tags slice (matches AWS behaviour for untagged resources). Required by Terraform AWS provider post-create refresh on `aws_lb`, `aws_lb_target_group`, and `aws_lb_listener`. | 1. LB tags round-trip<br>2. TG tags round-trip<br>3. Listener returns empty Tags (no error)<br>4. Multiple ARNs in one call<br>5. Unknown LB/TG/listener (per-type not-found)<br>6. Cross-account ARN (not-found, no leak)<br>7. Invalid/non-ELBv2 ARN (InvalidParameterValue)<br>8. Empty/nil ResourceArns (MissingParameter)<br>9. Untagged LB returns empty Tags | **DONE** |
+
+#### ELBv2 - Not Implemented
+
+| Feature | Description | Priority | Status |
+|---------|-------------|----------|--------|
+| Listener rules | Path-based and hostname-based routing (CreateRule, DeleteRule, DescribeRules, ModifyRule) | High | **NOT STARTED** |
+| HTTPS/TLS termination | SSL certificates, SSL policies, ALPN | High | **NOT STARTED** |
+| Modify operations | ModifyLoadBalancerAttributes, ModifyTargetGroup, ModifyTargetGroupAttributes, ModifyListener | Medium | **NOT STARTED** |
+| Connection draining | Deregistration delay for graceful target removal | Medium | **NOT STARTED** |
+| Stickiness | Session affinity / sticky sessions on target groups | Medium | **NOT STARTED** |
+| Active health checking | API-driven health state updates (currently HAProxy-only) | Medium | **NOT STARTED** |
+| Target types | IP and Lambda target types (only instance supported) | Low | **NOT STARTED** |
+| Access logs | S3 access log delivery | Low | **NOT STARTED** |
+| WAF integration | AWS WAF association for web application firewall | Low | **NOT STARTED** |
+
+### CloudWatch (Basic Monitoring)
+
+Basic CloudWatch metrics support is required for `monitor-instances`/`unmonitor-instances` to be meaningful. Daemon nodes should publish CPU, disk, and network metrics for running instances. Metrics stored in a time-series-friendly structure in JetStream KV or dedicated NATS subjects.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `put-metric-data` | — | `--namespace`, `--metric-data` (MetricName, Value, Unit, Timestamp, Dimensions) | None | Gateway validates Namespace + MetricData (required) → NATS `cloudwatch.PutMetricData` → daemon stores metric data points in time-series KV (key: `{namespace}.{metricName}.{dimensionHash}`) → return success | 1. Put single metric<br>2. Put with dimensions (InstanceId)<br>3. Missing namespace (MissingParameter) | **NOT STARTED** |
+| `get-metric-statistics` | — | `--namespace`, `--metric-name`, `--start-time`, `--end-time`, `--period`, `--statistics` (Average, Sum, Min, Max, SampleCount), `--dimensions` | None | Gateway validates required fields → NATS `cloudwatch.GetMetricStatistics` → daemon queries time-series data → aggregates by period → returns Datapoints with requested statistics | 1. Get CPU utilization over 1 hour<br>2. Get with dimensions (InstanceId)<br>3. No data returns empty | **NOT STARTED** |
+| `list-metrics` | — | `--namespace`, `--metric-name`, `--dimensions`, `--recently-active` | None | NATS `cloudwatch.ListMetrics` → daemon scans metric namespaces → returns Metrics list with Namespace, MetricName, Dimensions | 1. List all metrics<br>2. Filter by namespace<br>3. Filter by metric name | **NOT STARTED** |
+| `describe-alarms` | — | `--alarm-names`, `--alarm-name-prefix`, `--state-value`, `--action-prefix` | None | NATS `cloudwatch.DescribeAlarms` → daemon lists alarms from KV → return MetricAlarms with State, Threshold, ComparisonOperator | 1. List all alarms<br>2. Filter by state (OK, ALARM, INSUFFICIENT_DATA) | **NOT STARTED** |
+| `put-metric-alarm` | — | `--alarm-name`, `--namespace`, `--metric-name`, `--statistic`, `--period`, `--evaluation-periods`, `--threshold`, `--comparison-operator`, `--alarm-actions`, `--dimensions` | None | NATS `cloudwatch.PutMetricAlarm` → daemon stores alarm config in KV → alarm evaluation runs on metric publish → state transitions trigger alarm actions | 1. Create alarm<br>2. Update existing alarm<br>3. Missing required fields | **NOT STARTED** |
+| `delete-alarms` | — | `--alarm-names` | Alarms must exist | NATS `cloudwatch.DeleteAlarms` → daemon deletes alarm records from KV → return success | 1. Delete existing alarm<br>2. Non-existent alarm (ResourceNotFound) | **NOT STARTED** |
+
+### Auto Scaling
+
+Auto Scaling manages instance fleets based on scaling policies. Depends on Launch Templates (EC2) and optionally ELBv2 target groups for health-based scaling. Stored in `spinifex-autoscaling` KV bucket.
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `create-auto-scaling-group` | — | `--auto-scaling-group-name`, `--launch-template` (Id/Name, Version), `--min-size`, `--max-size`, `--desired-capacity`, `--vpc-zone-identifier` (subnet IDs), `--target-group-arns`, `--health-check-type` (EC2/ELB), `--health-check-grace-period`, `--tags` | Launch template must exist | NATS `autoscaling.CreateAutoScalingGroup` → daemon validates launch template → stores ASG record in KV → launches DesiredCapacity instances using template → registers with target groups if specified → starts health check loop → return success | 1. Create ASG with launch template<br>2. Scale to desired capacity<br>3. Missing launch template (ValidationError)<br>4. Min > Max (ValidationError) | **NOT STARTED** |
+| `update-auto-scaling-group` | — | `--auto-scaling-group-name`, `--min-size`, `--max-size`, `--desired-capacity`, `--launch-template`, `--health-check-type`, `--health-check-grace-period` | ASG must exist | NATS `autoscaling.UpdateAutoScalingGroup` → daemon updates ASG config in KV → if DesiredCapacity changed, scale up/down accordingly → return success | 1. Update desired capacity<br>2. Update launch template<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-auto-scaling-group` | — | `--auto-scaling-group-name`, `--force-delete` | ASG must exist | NATS `autoscaling.DeleteAutoScalingGroup` → if force: terminate all instances → deregister from target groups → delete from KV → return success. Without force: reject if instances > 0 | 1. Delete empty ASG<br>2. Force delete with instances<br>3. Delete with instances (ResourceInUse) | **NOT STARTED** |
+| `describe-auto-scaling-groups` | — | `--auto-scaling-group-names`, `--filters`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingGroups` → daemon lists ASG records → return list with Instances, DesiredCapacity, MinSize, MaxSize | 1. List all ASGs<br>2. Filter by name | **NOT STARTED** |
+| `set-desired-capacity` | — | `--auto-scaling-group-name`, `--desired-capacity`, `--honor-cooldown` | ASG must exist | NATS `autoscaling.SetDesiredCapacity` → daemon updates desired count → scale up (launch) or scale down (terminate oldest) → return success | 1. Scale up<br>2. Scale down<br>3. Exceeds max (ValidationError) | **NOT STARTED** |
+| `describe-auto-scaling-instances` | — | `--instance-ids`, `--max-records` | None | NATS `autoscaling.DescribeAutoScalingInstances` → daemon lists all ASG-managed instances → return list with ASG name, lifecycle state, health status | 1. List all ASG instances<br>2. Filter by instance ID | **NOT STARTED** |
+| `put-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name`, `--policy-type` (TargetTrackingScaling, StepScaling, SimpleScaling), `--target-tracking-configuration`, `--scaling-adjustment`, `--cooldown` | ASG must exist | NATS `autoscaling.PutScalingPolicy` → daemon stores policy in KV → links to CloudWatch alarms for metric-based triggers → return PolicyARN | 1. Create target tracking policy<br>2. Create step scaling policy<br>3. Non-existent ASG (ValidationError) | **NOT STARTED** |
+| `delete-scaling-policy` | — | `--auto-scaling-group-name`, `--policy-name` | Policy must exist | NATS `autoscaling.DeleteScalingPolicy` → daemon deletes policy → remove CloudWatch alarm links → return success | 1. Delete existing policy<br>2. Non-existent policy | **NOT STARTED** |

docs/DESIGN.md

@@ -4,55 +4,9 @@ Spinifex is an AWS-compatible infrastructure platform for bare-metal, edge, and
 
 ## High-Level Architecture
 
-```
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                              AWS SDK / CLI                                  │
-│                    (aws ec2 run-instances --endpoint-url ...)               │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                                      │ HTTPS (SigV4 Auth)
-                                      ▼
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                           AWS Gateway (Port 9999)                           │
-│                                                                             │
-│   • TLS termination                    • AWS SigV4 authentication           │
-│   • EC2 Query Protocol parsing         • Routes to service handlers         │
-│   • XML response generation            • NATS client connection             │
-│                                                                             │
-│   File: spinifex/services/awsgw/awsgw.go                                        │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                                      │ JSON over NATS
-                                      ▼
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                            NATS Message Broker                              │
-│                                                                             │
-│   Topics:                              Queue Groups:                        │
-│   • ec2.RunInstances                   • spinifex-workers (load balanced)       │
-│   • ec2.DescribeInstances              • No queue (fan-out to all nodes)    │
-│   • ec2.cmd.{instanceID}              (per-instance start/stop/terminate)   │
-│   • ebs.mount / ebs.unmount                                                 │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                    ┌─────────────────┼─────────────────┐
-                    ▼                 ▼                 ▼
-┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
-│   Spinifex Daemon        │ │   Spinifex Daemon        │ │   Spinifex Daemon        │
-│   (Node 1)           │ │   (Node 2)           │ │   (Node N)           │
-│                      │ │                      │ │                      │
-│ • NATS subscriber    │ │ • NATS subscriber    │ │ • NATS subscriber    │
-│ • Resource manager   │ │ • Resource manager   │ │ • Resource manager   │
-│ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │
-│ • QMP monitoring     │ │ • QMP monitoring     │ │ • QMP monitoring     │
-│                      │ │                      │ │                      │
-│ File: daemon.go      │ │ File: daemon.go      │ │ File: daemon.go      │
-└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
-          │                       │                       │
-          ▼                       ▼                       ▼
-┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
-│   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │
-└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
-```
+<p align="center">
+  <img src="../.github/assets/design-architecture.svg" alt="Spinifex high-level architecture — AWS SDK, gateway, NATS broker, per-node daemons, QEMU/KVM VMs" width="900">
+</p>
 
 ## Request Flow
 

docs/INSTALL_ORIGINAL.md

@@ -48,7 +48,7 @@ sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu
 
 **Note:** OVN and Open vSwitch are required for VPC networking (virtual switches, routers, DHCP, Geneve overlay). The packages are installed above, but the setup and configuration step (`setup-ovn.sh`) is covered later — see the [Setup OVN](#setup-ovn) section below.
 
-Ensure the Go toolkit is installed for version 1.26.1 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
+Ensure the Go toolkit is installed for version 1.26.2 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
 
 Confirm Go is correctly installed, and set in your $PATH.
 
@@ -940,9 +940,9 @@ All nodes should show `Ready` status. Use `./bin/spx get vms` to see running ins
 Check individual daemon health endpoints:
 
 ```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
-curl -s http://$SPINIFEX_NODE2:4432/health
-curl -s http://$SPINIFEX_NODE3:4432/health
+curl -sk https://$SPINIFEX_NODE1:4432/health
+curl -sk https://$SPINIFEX_NODE2:4432/health
+curl -sk https://$SPINIFEX_NODE3:4432/health
 ```
 
 Check NATS cluster routing (from any node):

docs/TAG-CONVENTIONS.md

@@ -0,0 +1,66 @@
+# Spinifex Tag Conventions
+
+Spinifex uses a small set of reserved tag keys to mark resources that are
+owned and managed by the platform itself (as opposed to resources created
+directly by customer AWS API calls). The UI uses these tags to hide
+system-owned resources from customer-facing listings so operators do not
+mistake them for user workloads and accidentally terminate or modify
+them.
+
+## `spinifex:managed-by`
+
+Identifies the Spinifex platform component that owns a resource.
+
+| Value   | Component                       | Tagged resource types                 |
+|---------|---------------------------------|---------------------------------------|
+| `elbv2` | ALB / HAProxy load balancers    | LB VM (`instance`), LB ENI, LB AMI    |
+
+The tag is applied by the backend at create/launch time:
+
+- **Instance** — set in
+  `spinifex/spinifex/daemon/daemon_system_instance.go` when
+  `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto
+  the stored `vm.VM.ManagedBy` field for fast lookup.
+- **ENI** — set in
+  `spinifex/spinifex/handlers/elbv2/service_impl.go` when
+  `CreateLoadBalancer` calls `CreateNetworkInterface`.
+- **AMI** — declared on the catalog entry in
+  `spinifex/spinifex/utils/images.go` and copied into
+  `AMIMetadata.Tags` by the image importer in
+  `spinifex/cmd/spinifex/cmd/admin.go`.
+
+Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`;
+new system components should register their value there and reuse the
+constants rather than hardcoding strings.
+
+## UI behaviour
+
+The admin **Nodes** page and the **Images** (AMIs) page filter out
+resources that carry `spinifex:managed-by`. Engineers who need to see
+system resources for debugging can append `?system=1` to the URL — the
+filter is bypassed and a hidden-count banner is replaced with the full
+listing. Direct navigation to a system AMI detail page without
+`?system=1` renders the "Image not found" fallback.
+
+The EC2 **Instances** page does not need UI-side filtering: backend
+SigV4 authentication scopes `DescribeInstances` by account ID, and
+system instances live under the global system account
+(`000000000000`), so customers never see them.
+
+## `spinifex:lb-arn`
+
+Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`.
+Stores the ARN of the parent load balancer so reconciliation can map an
+ENI back to its owner without a separate lookup. Customer-facing
+filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is
+informational.
+
+## Adding a new system component
+
+1. Add a new value constant next to `ManagedByELBv2` in
+   `spinifex/spinifex/tags/tags.go`.
+2. Tag every resource the component creates (instance, ENI, AMI,
+   volume, …) with `spinifex:managed-by=<value>`.
+3. Add the value to the table above so operators can recognise it.
+4. No frontend changes are required — the UI filters on the key, not
+   the value.

docs/creating-accounts/README.md

@@ -32,16 +32,10 @@ Spinifex supports multi-tenant account isolation. Each account gets its own IAM
 ## Create Account
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
-## Verify
-
-```bash
-aws sts get-caller-identity
-```
-
 ## Troubleshooting
 
 ## Credentials Not Working

docs/creating-an-environment/README.md

@@ -1,110 +0,0 @@
----
-title: "Creating an Environment"
-description: "End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances."
-category: "Environments"
-tags:
-  - environment
-  - vpc
-  - ec2
-  - ssh
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "Debian Cloud Images"
-    url: "https://cloud.debian.org/images/cloud/bookworm/latest/"
----
-
-# Creating an Environment
-
-> End-to-end guide: user accounts, SSH keys, VPCs, AMIs, and launching instances.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-This guide walks through the complete workflow for creating a production-ready Spinifex environment.
-
-## Instructions
-
-## 1. Import SSH Key
-
-```bash
-aws ec2 import-key-pair --key-name "spinifex-key" --public-key-material fileb://~/.ssh/id_rsa.pub
-```
-
-## 2. Create VPC
-
-```bash
-aws ec2 create-vpc --cidr-block 10.200.0.0/16
-export SPINIFEX_VPC="vpc-XXX"
-
-aws ec2 create-subnet --vpc-id $SPINIFEX_VPC --cidr-block 10.200.1.0/24
-export SPINIFEX_SUBNET="subnet-XXX"
-```
-
-## 3. Import AMI
-
-```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
-export SPINIFEX_AMI="ami-XXX"
-```
-
-## 4. Launch Instance
-
-```bash
-aws ec2 run-instances \
-  --image-id $SPINIFEX_AMI \
-  --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
-```
-
-## 5. Connect via SSH
-
-```bash
-ps auxw | grep $INSTANCE_ID
-# Look for: hostfwd=tcp:127.0.0.1:<port>-:22
-
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```
-
-## Troubleshooting
-
-## Instance Stuck in Pending
-
-Check the Spinifex daemon and QEMU logs for boot errors:
-
-```bash
-ls ~/spinifex/logs/
-cat ~/spinifex/logs/spinifex.log
-```
-
-Verify the AMI exists and architecture matches your host:
-
-```bash
-aws ec2 describe-images --image-ids $SPINIFEX_AMI
-```
-
-## SSH Connection Refused
-
-cloud-init takes 30-60 seconds to configure networking and SSH after boot. Wait and retry.
-
-Verify the SSH port forwarding is active:
-
-```bash
-ps auxw | grep hostfwd
-```
-
-Ensure you're using the correct key and port:
-
-```bash
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
-```

docs/hybrid-sync/README.md

@@ -29,6 +29,14 @@ Spinifex's hybrid mode enables bidirectional data synchronization between local
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Push Local to Cloud
 
 ```bash
@@ -61,14 +69,3 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ \
 ```
 
 Only changed or missing files will be transferred on subsequent runs.
-
-## Permission Errors During Sync
-
-Verify your AWS credentials are configured for both the source and destination:
-
-```bash
-aws sts get-caller-identity --profile spinifex
-aws sts get-caller-identity --profile default
-```
-
-Ensure the IAM user has `s3:GetObject`, `s3:PutObject`, and `s3:ListBucket` permissions on both buckets.

docs/iam-users-and-policies/README.md

@@ -53,7 +53,6 @@ When you create an account with `spx admin account create`, Spinifex bootstraps
 
 ```bash
 export AWS_PROFILE=spinifex-myteam
-aws sts get-caller-identity
 ```
 
 ## Users
@@ -68,13 +67,13 @@ The response includes the user's ARN, unique ID, and creation date:
 
 ```json
 {
-    "User": {
-        "UserName": "alice",
-        "UserId": "AIDA1A2B3C4D5E6F7890",
-        "Arn": "arn:aws:iam::000000000001:user/alice",
-        "Path": "/",
-        "CreateDate": "2026-03-24T10:00:00Z"
-    }
+  "User": {
+    "UserName": "alice",
+    "UserId": "AIDA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:user/alice",
+    "Path": "/",
+    "CreateDate": "2026-03-24T10:00:00Z"
+  }
 }
 ```
 
@@ -120,7 +119,6 @@ aws iam detach-user-policy --user-name alice \
 aws iam delete-user --user-name alice
 ```
 
-
 ## Access Keys
 
 Access keys are how users authenticate with the AWS CLI. Each user can have up to **2 access keys** at a time, allowing key rotation without downtime.
@@ -133,13 +131,13 @@ aws iam create-access-key --user-name alice
 
 ```json
 {
-    "AccessKey": {
-        "UserName": "alice",
-        "AccessKeyId": "AKIA1A2B3C4D5E6F7890ABCD",
-        "Status": "Active",
-        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
-        "CreateDate": "2026-03-24T10:05:00Z"
-    }
+  "AccessKey": {
+    "UserName": "alice",
+    "AccessKeyId": "AKIA1A2B3C4D5E6F7890ABCD",
+    "Status": "Active",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "CreateDate": "2026-03-24T10:05:00Z"
+  }
 }
 ```
 
@@ -153,7 +151,6 @@ Add the credentials to `~/.aws/credentials` under a new profile and set `AWS_PRO
 aws iam list-access-keys --user-name alice
 ```
 
-
 ### Rotate Access Keys
 
 Create a second key, update your configuration, then delete the old one:
@@ -189,7 +186,6 @@ aws iam update-access-key --user-name alice \
   --status Active
 ```
 
-
 ## Policies
 
 Policies are JSON documents that define what actions a user can perform on which resources.
@@ -198,14 +194,14 @@ Policies are JSON documents that define what actions a user can perform on which
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": ["ec2:RunInstances", "ec2:DescribeInstances"],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:RunInstances", "ec2:DescribeInstances"],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -249,32 +245,31 @@ aws iam create-policy \
 
 ```json
 {
-    "Policy": {
-        "PolicyName": "EC2ReadOnly",
-        "PolicyId": "ANPA1A2B3C4D5E6F7890",
-        "Arn": "arn:aws:iam::000000000001:policy/EC2ReadOnly",
-        "Path": "/",
-        "DefaultVersionId": "v1",
-        "CreateDate": "2026-03-24T10:10:00Z"
-    }
+  "Policy": {
+    "PolicyName": "EC2ReadOnly",
+    "PolicyId": "ANPA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:policy/EC2ReadOnly",
+    "Path": "/",
+    "DefaultVersionId": "v1",
+    "CreateDate": "2026-03-24T10:10:00Z"
+  }
 }
 ```
 
-
 ### Common Policy Examples
 
 **Full administrator access:**
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": "*",
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "*",
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -282,19 +277,19 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:GetObject",
-                "s3:PutObject",
-                "s3:ListBucket",
-                "s3:DeleteObject"
-            ],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:ListBucket",
+        "s3:DeleteObject"
+      ],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -302,21 +297,21 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-                "ec2:RunInstances",
-                "ec2:StartInstances",
-                "ec2:StopInstances",
-                "ec2:TerminateInstances",
-                "ec2:DescribeInstances",
-                "ec2:DescribeImages"
-            ],
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:RunInstances",
+        "ec2:StartInstances",
+        "ec2:StopInstances",
+        "ec2:TerminateInstances",
+        "ec2:DescribeInstances",
+        "ec2:DescribeImages"
+      ],
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -324,14 +319,14 @@ aws iam create-policy \
 
 ```json
 {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Deny",
-            "Action": "ec2:TerminateInstances",
-            "Resource": "*"
-        }
-    ]
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": "ec2:TerminateInstances",
+      "Resource": "*"
+    }
+  ]
 }
 ```
 
@@ -352,7 +347,6 @@ aws iam get-policy-version \
   --version-id v1
 ```
 
-
 ### List Policies
 
 ```bash
@@ -368,7 +362,6 @@ aws iam delete-policy \
   --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
 ```
 
-
 ## Attaching Policies
 
 Policies have no effect until attached to a user.
@@ -380,7 +373,6 @@ aws iam attach-user-policy --user-name alice \
   --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
 ```
 
-
 ### List a User's Policies
 
 ```bash
@@ -453,24 +445,24 @@ AWS_PROFILE=spinifex-carol aws ec2 terminate-instances --instance-ids i-123
 
 ## Command Reference
 
-| Command | Key Parameters | Notes |
-|---------|---------------|-------|
-| `create-user` | `--user-name`, `--path` | Path default: `/` |
-| `get-user` | `--user-name` | |
-| `list-users` | `--path-prefix` | |
-| `delete-user` | `--user-name` | Must have no keys or policies |
-| `create-access-key` | `--user-name` | Max 2 per user; secret shown once |
-| `list-access-keys` | `--user-name` | |
-| `delete-access-key` | `--user-name`, `--access-key-id` | |
-| `update-access-key` | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive` |
-| `create-policy` | `--policy-name`, `--policy-document` | Max 6144 bytes |
-| `get-policy` | `--policy-arn` | Metadata only |
-| `get-policy-version` | `--policy-arn`, `--version-id` | Use `v1`; includes document |
-| `list-policies` | | |
-| `delete-policy` | `--policy-arn` | Must not be attached |
-| `attach-user-policy` | `--user-name`, `--policy-arn` | Idempotent |
-| `detach-user-policy` | `--user-name`, `--policy-arn` | |
-| `list-attached-user-policies` | `--user-name` | |
+| Command                       | Key Parameters                               | Notes                             |
+| ----------------------------- | -------------------------------------------- | --------------------------------- |
+| `create-user`                 | `--user-name`, `--path`                      | Path default: `/`                 |
+| `get-user`                    | `--user-name`                                |                                   |
+| `list-users`                  | `--path-prefix`                              |                                   |
+| `delete-user`                 | `--user-name`                                | Must have no keys or policies     |
+| `create-access-key`           | `--user-name`                                | Max 2 per user; secret shown once |
+| `list-access-keys`            | `--user-name`                                |                                   |
+| `delete-access-key`           | `--user-name`, `--access-key-id`             |                                   |
+| `update-access-key`           | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive`            |
+| `create-policy`               | `--policy-name`, `--policy-document`         | Max 6144 bytes                    |
+| `get-policy`                  | `--policy-arn`                               | Metadata only                     |
+| `get-policy-version`          | `--policy-arn`, `--version-id`               | Use `v1`; includes document       |
+| `list-policies`               |                                              |                                   |
+| `delete-policy`               | `--policy-arn`                               | Must not be attached              |
+| `attach-user-policy`          | `--user-name`, `--policy-arn`                | Idempotent                        |
+| `detach-user-policy`          | `--user-name`, `--policy-arn`                |                                   |
+| `list-attached-user-policies` | `--user-name`                                |                                   |
 
 ## Troubleshooting
 

docs/install-airgapped/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Air-Gapped Installation"
+title: "Air-Gapped Install"
 description: "Deploy Spinifex in environments without internet connectivity. Covers offline package caching, USB deployment, and package verification."
 category: "Getting Started"
 tags:
@@ -85,7 +85,7 @@ cp -r /mnt/usb/spinifex-source ~/Development/mulga/spinifex
 cd ~/Development/mulga/spinifex && make build
 ```
 
-Then follow the [Source Install](/docs/source-install) guide from Step 3 (Setup OVN) onwards.
+Then follow the [Source Install](/docs/install-source) guide from Step 3 (Setup OVN) onwards.
 
 ## Troubleshooting
 

docs/install-multi-node/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Multi-Node Installation"
+title: "Multi-Node Install"
 description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
 category: "Getting Started"
 tags:
@@ -42,7 +42,7 @@ A Spinifex cluster distributes services across multiple servers for high availab
 ## Step 1. Install Spinifex on Each Server
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 ```
 
 ## Step 2. Set Node IP Variables
@@ -152,7 +152,7 @@ Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import a
 The init command must still be running when join executes. If init exited, re-run with `--force`.
 
 ```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
+curl -sk https://$SPINIFEX_NODE1:4432/health
 ```
 
 ### OVN Chassis Not Registering

docs/install-source/README.md

@@ -13,7 +13,7 @@ resources:
     url: "https://go.dev/dl/"
 ---
 
-# Source Install
+# Source Installation
 
 > Build Spinifex from source for development, custom builds, or contributing.
 
@@ -27,12 +27,12 @@ resources:
 
 ## Overview
 
-This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/installing-spinifex) is recommended.
+This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/install) is recommended.
 
 **Requirements:**
 
 - Ubuntu 22.04+ / Debian 12+
-- Go 1.26.1+
+- Go 1.26.2+
 - GCC, make, pkg-config
 - QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
 
@@ -57,15 +57,7 @@ cd spinifex
 
 Confirm `./bin/spx` exists.
 
-## Step 3. Initialize
-
-```bash
-sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
-```
-
-Save the admin credentials printed during init.
-
-## Step 4. Setup OVN
+## Step 3. Setup OVN
 
 If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:
 
@@ -83,13 +75,23 @@ If your WAN is a physical NIC, choose one:
 ./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
-## Step 5. Start Services
+## Step 4. Development initialisation
 
 ```bash
-./scripts/start-dev.sh
+./scripts/dev-install.sh
 ```
 
-## Step 6. Verify
+Note, this will bootstrap a single node development environment
+
+## Step 5. Start services
+
+```bash
+sudo systemctl status spinifex.target
+```
+
+Start all services
+
+## Step 6. Verify Installation
 
 ```bash
 export AWS_PROFILE=spinifex

docs/install/README.md

@@ -1,5 +1,5 @@
 ---
-title: "Installing Spinifex"
+title: "Single-Node Install"
 description: "Install Spinifex on a single server using the binary installer."
 category: "Getting Started"
 tags:
@@ -31,14 +31,14 @@ resources:
 
 Spinifex is an open-source infrastructure platform that brings core AWS services — EC2, VPC, EBS, and S3 — to bare-metal, edge, and on-prem environments.
 
-This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Installation](/docs/installing-spinifex-multi-node). To build from source, see [Source Install](/docs/source-install).
+This guide installs Spinifex on a single server using the binary installer. For multi-server clusters, see [Multi-Node Install](/docs/install-multi-node). To build from source, see [Source Install](/docs/install-source).
 
 **Supported Operating Systems:**
 
 - Ubuntu 22.04 / 24.04 / 25.10
 - Debian 12 / 13
 
-**What gets installed:**
+**What Gets Installed:**
 
 - Spinifex daemon and CLI
 - QEMU/KVM for virtual machine management
@@ -52,7 +52,7 @@ This guide installs Spinifex on a single server using the binary installer. For
 ## Step 1. Install Spinifex
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 ```
 
 The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
@@ -70,15 +70,20 @@ If your WAN is a physical NIC, choose one:
 ```bash
 # Dedicated WAN NIC (not your SSH connection):
 sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1
-
-# Single-NIC host (SSH-safe macvlan):
-sudo /usr/local/share/spinifex/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
+**Separating VPC tunnel traffic from WAN:** If you want Geneve tunnel traffic (inter-VM east-west for VPC traffic) to use a dedicated interface instead of the WAN IP, add `--encap-ip=<IP>` to specify the tunnel endpoint address:
+
+```bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --encap-ip=10.0.0.1
+```
+
+This is recommended for production and required in multi-node deployments. See [Multi-Node Install](/docs/install-multi-node) for details.
+
 ## Step 3. Initialize
 
 ```bash
-sudo spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+sudo spx admin init --node node1 --nodes 1
 ```
 
 This auto-detects your network topology, generates configuration and TLS certificates, installs the CA into the system trust store, and configures AWS CLI credentials. Save the admin credentials printed during init — they will not be shown again.

docs/launching-instances/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Launching Instances"
 description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex."
-category: "Environments"
+category: "Compute"
 tags:
   - ec2
   - instances
@@ -20,8 +20,16 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
-- [Instructions](#instructions)
+- [Launch](#launch)
+- [Manage](#manage)
+- [Reboot](#reboot)
+- [Modify Instance Attributes](#modify-instance-attributes)
+- [Console Output](#console-output)
+- [Instance Types](#instance-types)
+- [SSH (Development)](#ssh-development)
 - [Troubleshooting](#troubleshooting)
+  - [Instance Fails to Boot](#instance-fails-to-boot)
+  - [Cannot SSH Into Instance](#cannot-ssh-into-instance)
 
 ---
 
@@ -34,19 +42,29 @@ Spinifex provides EC2-compatible VM management built on QEMU/KVM. Instances supp
 - `run-instances` — Launch new VMs
 - `describe-instances` — Query state
 - `stop-instances` / `start-instances` — Lifecycle
+- `reboot-instances` — In-place restart (QMP reset)
 - `terminate-instances` — Permanent removal
+- `modify-instance-attribute` — Change type, user data, or source/dest check
+- `get-console-output` — Retrieve serial console log
+- `describe-instance-types` — List available instance types
 
 ## Instructions
 
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
 ## Launch
 
 ```bash
 aws ec2 run-instances \
   --image-id $SPINIFEX_AMI \
   --instance-type t3.small \
-  --key-name spinifex-key \
-  --subnet-id $SPINIFEX_SUBNET \
-  --count 1
+  --key-name spinifex-key
 
 export INSTANCE_ID="i-XXX"
 ```
@@ -58,15 +76,72 @@ aws ec2 describe-instances --instance-ids $INSTANCE_ID
 aws ec2 stop-instances --instance-ids $INSTANCE_ID
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 reboot-instances --instance-ids $INSTANCE_ID
+```
+
+## Modify Instance Attributes
+
+Change instance type, user data, or source/dest check. Instance type and user data require the instance to be **stopped** first.
+
+### Change Instance Type
+
+```bash
+aws ec2 stop-instances --instance-ids $INSTANCE_ID
+
+aws ec2 modify-instance-attribute \
+  --instance-id $INSTANCE_ID \
+  --instance-type t3.medium
+
+aws ec2 start-instances --instance-ids $INSTANCE_ID
+```
+
+## Console Output
+
+Retrieve the serial console log for a running instance. Output is base64-encoded.
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID
+```
+
+Decode the output:
+
+```bash
+aws ec2 get-console-output --instance-id $INSTANCE_ID \
+  --query 'Output' --output text | base64 -d
+```
+
+## Instance Types
+
+List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.
+
+```bash
+aws ec2 describe-instance-types
+```
+
+Filter to a specific type:
+
+```bash
+aws ec2 describe-instance-types --instance-types t3.micro t3.small
+```
+
+Show capacity (how many of each type can still be launched):
+
+```bash
+aws ec2 describe-instance-types \
+  --filters Name=capacity,Values=true
 ```
 
 ## SSH (Development)
 
+In development mode, find the QEMU port forward and connect via localhost:
+
 ```bash
 ps auxw | grep $INSTANCE_ID
 ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
 ```
 
+For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
+
 ## Troubleshooting
 
 ### Instance Fails to Boot
@@ -81,8 +156,8 @@ aws ec2 describe-images --image-ids $SPINIFEX_AMI
 If the AMI is for a different architecture (e.g. arm64 on an x86_64 host), import the correct image:
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-x86_64
+spx admin images list
+spx admin images import --name debian-12-x86_64
 ```
 
 ### Cannot SSH Into Instance

docs/moving-aws-workload/README.md

@@ -37,10 +37,17 @@ Spinifex provides drop-in compatibility with AWS APIs, making it possible to mig
 
 ## Instructions
 
-## Configure AWS CLI
+## Prerequisites
+
+Ensure the AWS profile is set:
 
 ```bash
 export AWS_PROFILE=spinifex
+```
+
+## Configure AWS CLI
+
+```bash
 aws ec2 describe-instances
 aws s3 ls
 ```

docs/placement-groups/README.md

@@ -0,0 +1,121 @@
+---
+title: "Placement Groups"
+description: "Create and manage spread and cluster placement groups for hardware-level instance placement control."
+category: "Compute"
+tags:
+  - ec2
+  - placement
+  - spread
+  - cluster
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS Placement Groups"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html"
+---
+
+# Placement Groups
+
+> Create and manage spread and cluster placement groups for hardware-level instance placement control.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Create](#create)
+- [Launch Into a Placement Group](#launch-into-a-placement-group)
+- [Describe](#describe)
+- [Delete](#delete)
+- [Troubleshooting](#troubleshooting)
+  - [InsufficientInstanceCapacity Error](#insufficientinstancecapacity-error)
+  - [Cannot Delete Placement Group](#cannot-delete-placement-group)
+
+---
+
+## Overview
+
+Placement groups control how Spinifex distributes instances across physical hosts. Two strategies are supported:
+
+- **Spread** — One instance per physical host, maximising fault isolation
+- **Cluster** — All instances on the same host, minimising latency
+
+The `partition` strategy is not supported.
+
+**Supported operations:**
+
+- `create-placement-group` — Create a new group
+- `describe-placement-groups` — Query groups with optional filters
+- `delete-placement-group` — Remove an empty group
+
+## Instructions
+
+## Prerequisites
+
+Ensure the AWS profile is set:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Create
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-spread-group \
+  --strategy spread
+```
+
+```bash
+aws ec2 create-placement-group \
+  --group-name my-cluster-group \
+  --strategy cluster
+```
+
+## Launch Into a Placement Group
+
+```bash
+aws ec2 run-instances \
+  --image-id $SPINIFEX_AMI \
+  --instance-type t3.small \
+  --key-name spinifex-key \
+  --placement GroupName=my-spread-group \
+  --count 3
+```
+
+## Describe
+
+```bash
+aws ec2 describe-placement-groups
+aws ec2 describe-placement-groups --group-names my-spread-group
+aws ec2 describe-placement-groups --group-ids pg-abc123
+aws ec2 describe-placement-groups \
+  --filters Name=strategy,Values=spread
+```
+
+## Delete
+
+The group must have no running instances before it can be deleted:
+
+```bash
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+aws ec2 delete-placement-group --group-name my-spread-group
+```
+
+## Troubleshooting
+
+### InsufficientInstanceCapacity Error
+
+Not enough distinct physical hosts for a spread launch. Reduce `--count` or terminate existing instances to free host slots:
+
+```bash
+spx admin nodes list
+```
+
+### Cannot Delete Placement Group
+
+The group still has running instances. Terminate them first:
+
+```bash
+aws ec2 describe-instances \
+  --filters Name=placement-group-name,Values=my-spread-group
+aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+```

docs/setting-up-your-cluster/README.md

@@ -26,6 +26,7 @@ resources:
 - [4. Launch an Instance](#4-launch-an-instance)
 - [5. Connect via SSH](#5-connect-via-ssh)
 - [6. Managing Instances](#6-managing-instances)
+- [7. Launching the Web UI](#7-launching-the-web-ui)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -33,9 +34,9 @@ resources:
 ## Overview
 
 This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
-- [Binary Install (Single Node)](/docs/installing-spinifex)
-- [Multi-Node Installation](/docs/installing-spinifex-multi-node)
-- [Source Install](/docs/source-install)
+- [Binary Install (Single Node)](/docs/install)
+- [Multi-Node Install](/docs/install-multi-node)
+- [Source Install](/docs/install-source)
 
 ## Instructions
 
@@ -166,14 +167,25 @@ Query available instance types for your hardware:
 aws ec2 describe-instance-types
 ```
 
-Launch an instance in the public subnet:
+Launch an instance in the public subnet, note this will select an instance type with 2 VCPU and 1024MB of memory.
+
+> **Note:** Replace `AMI_NAME` with the previously imported image above with the `ami-` prefix.
 
 ```bash
-AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
+AMI_NAME="ami-ubuntu-24.04-x86_64"
+
+AMI_ID=$(aws ec2 describe-images \
+  --filters "Name=name,Values=$AMI_NAME" \
+  --query 'Images | sort_by(@, &CreationDate) | [-1].ImageId' \
+  --output text)
+
+INSTANCE_TYPE=$(aws ec2 describe-instance-types \
+  --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
+  --output text)
 
 INSTANCE_ID=$(aws ec2 run-instances \
   --image-id $AMI_ID \
-  --instance-type t3.small \
+  --instance-type $INSTANCE_TYPE \
   --key-name spinifex-key \
   --subnet-id $SUBNET_ID \
   --count 1 \
@@ -256,6 +268,51 @@ On a multi-node cluster, instances are distributed across nodes:
 spx get vms
 ```
 
+## 7. Launching the Web UI
+
+Spinifex ships with a built-in web console — an alternative to the AWS CLI, SDKs, and Terraform, analogous to the AWS Management Console. Every action in the UI is a standard AWS SigV4 API call, so the same IAM policies and audit behaviour apply.
+
+![Spinifex Web UI](../spinifex-ui.jpg)
+
+### Open the Console
+
+The UI is served by each node on port `3000` over TLS. Replace `YOUR_NODE_IP` with the address of the node you installed Spinifex on (or `localhost` if you're on the node itself):
+
+```
+https://YOUR_NODE_IP:3000
+```
+
+### Trust the Self-Signed Certificate (required)
+
+On first load, your browser will show a TLS warning — Spinifex generates a self-signed certificate at install time. This is expected.
+
+1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
+2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
+3. Install the certificate as a **trusted root** on your workstation:
+
+   - **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
+   - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
+   - **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
+   - **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
+
+4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
+
+> **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
+
+### Log In with AWS Credentials
+
+The console authenticates against the AWS credentials in `~/.aws/credentials` on the node where Spinifex is installed:
+
+```ini
+[spinifex]
+aws_access_key_id     = AKIA...
+aws_secret_access_key = ...
+```
+
+At the Spinifex login screen, paste the **Access Key ID** and **Secret Access Key** from the `[spinifex]` profile (or whichever profile maps to the IAM user or role you want to use). Additional users and policies can be managed through the UI or via `aws iam` commands — see [IAM Users and Policies](/docs/iam-users-and-policies).
+
+Once logged in, you have browser-based access to every Spinifex feature: launch and manage instances, attach EBS volumes, browse S3 buckets, configure VPCs and security groups, and manage IAM users and keys — all backed by the same AWS-compatible control plane the CLI uses.
+
 ## Additional Options
 
 ### Private Subnets (No Public IP)
@@ -280,10 +337,6 @@ spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
-### Web UI
-
-The Spinifex web interface is available at `https://localhost:3000` when services are running.
-
 ## Troubleshooting
 
 ### Instance Stuck in Pending

docs/spinifex-admin-cli/README.md

@@ -27,7 +27,7 @@ resources:
 
 The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
 
-**Binary location:** `~/Development/mulga/spinifex/bin/spx`
+**Binary location:** `/usr/local/bin/spx` (binary install) or `~/Development/mulga/spinifex/bin/spx` (source install)
 
 All services in the Spinifex platform are managed through this single binary.
 
@@ -36,14 +36,14 @@ All services in the Spinifex platform are managed through this single binary.
 ## Account Management
 
 ```bash
-./bin/spx admin account create --name myteam
+spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
 ## Node Management
 
 ```bash
-./bin/spx get nodes
+spx get nodes
 ```
 
 ```
@@ -56,20 +56,20 @@ node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
 ## Monitor Resources
 
 ```bash
-./bin/spx top nodes
+spx top nodes
 ```
 
 ## Image Management
 
 ```bash
-./bin/spx admin images list
-./bin/spx admin images import --name debian-12-arm64
+spx admin images list
+spx admin images import --name debian-12-arm64
 ```
 
 ## Cluster Shutdown
 
 ```bash
-./bin/spx admin cluster shutdown
+spx admin cluster shutdown
 ```
 
 ## Troubleshooting
@@ -79,7 +79,7 @@ node3   Ready     127.0.0.3       ap-southeast-2   ap-southeast-2a  2m       0
 The binary may not be executable. Fix permissions:
 
 ```bash
-chmod +x ./bin/spx
+chmod +x /usr/local/bin/spx
 ```
 
 If you get permission errors during operations, ensure you're running with appropriate privileges. Some OVN and networking commands require `sudo`.

docs/terraform/bastion-private-subnet/README.md

@@ -0,0 +1,155 @@
+---
+title: "Bastion with Private Subnet"
+description: "Deploy a bastion host to access an isolated private compute instance with no internet connectivity."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - bastion
+  - vpc
+  - security
+  - private subnet
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Bastion with Private Subnet
+
+> Deploy a bastion host to access an isolated private compute instance with no internet connectivity.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a VPC with public and private subnets where the private subnet has no internet access. A bastion host in the public subnet is the only way to reach instances in the private subnet. This pattern is used for sensitive workloads that must remain isolated from the internet — the private instance cannot make or receive any connections outside the VPC.
+
+**Architecture:**
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-bastion.svg" alt="Bastion architecture — WAN reaches bastion in public subnet, SSH hop to app server in private subnet" width="900">
+</p>
+
+**What you'll learn:**
+
+- Creating public and private subnets in a VPC
+- Isolating compute instances from the internet using route tables and security groups
+- Using a bastion host as the sole access point to private resources
+- SSH hopping through a bastion to reach private instances
+
+**Use cases:**
+
+- Sensitive data processing that must not have internet egress
+- Internal services that should only be reachable within the VPC
+- Compliance workloads requiring network isolation
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/bastion-private-subnet
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Connect
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry.
+
+SSH into the bastion:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_public_ip>
+```
+
+From the bastion, SSH to the private instance. The key is pre-installed at `~/.ssh/bastion-demo.pem` via cloud-init:
+
+```bash
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+### Step 4. Verify Isolation
+
+From the private instance, confirm there is no internet connectivity:
+
+```bash
+# This should time out — the private instance has no route to the internet
+curl --connect-timeout 5 https://deb.debian.org || echo "No internet access (expected)"
+```
+
+The private instance can only communicate within the VPC (`10.20.0.0/16`). Its security group restricts egress to VPC-internal traffic only.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Cannot SSH to Private Instance
+
+The private instance has no public IP — it is only reachable from the bastion. SSH to the bastion first, then use the pre-installed key to hop:
+
+```bash
+ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+```
+
+If the key is missing on the bastion, check that cloud-init completed successfully:
+
+```bash
+sudo journalctl -u cloud-init --no-pager
+ls -la ~/.ssh/bastion-demo.pem
+```
+
+### Private Instance Can Reach the Internet
+
+If the private instance unexpectedly has internet access, check that its route table has no default route:
+
+```bash
+aws ec2 describe-route-tables --profile spinifex
+```
+
+The private route table should have no `0.0.0.0/0` route. Also verify the private security group egress is restricted to the VPC CIDR (`10.20.0.0/16`).
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/bastion-private-subnet/main.tf

@@ -0,0 +1,346 @@
+# Example 2: Bastion Host with Private Subnet
+#
+# Deploys a VPC with both public and private subnets. A bastion host in the
+# public subnet provides SSH access to an isolated instance in the private
+# subnet. The private instance has no internet connectivity — ideal for
+# sensitive workloads that must remain air-gapped from the internet.
+#
+# Architecture:
+#
+#   WAN ──SSH──▶ Bastion (public subnet)
+#                    │
+#                    ▼ SSH (private IP)
+#                 App Server (private subnet, no internet)
+#
+# Usage:
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   # SSH to the bastion
+#   ssh -i bastion-demo.pem ec2-user@<bastion_ip>
+#
+#   # From the bastion, SSH to the private instance
+#   # (the key is pre-installed at ~/.ssh/bastion-demo.pem via cloud-init)
+#   ssh -i ~/.ssh/bastion-demo.pem ec2-user@<private_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair (shared by bastion and private instances)
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "bastion" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "bastion" {
+  key_name   = "bastion-demo"
+  public_key = tls_private_key.bastion.public_key_openssh
+}
+
+resource "local_file" "bastion_pem" {
+  filename        = "${path.module}/bastion-demo.pem"
+  content         = tls_private_key.bastion.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "bastion-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — only the public subnet routes through this
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet — bastion host lives here
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "bastion-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "bastion-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Private Subnet — isolated instances live here (no public IPs, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "private" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.20.2.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = false
+
+  tags = {
+    Name = "bastion-demo-private"
+  }
+}
+
+# Private route table — no default route, no internet access
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "bastion-demo-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private" {
+  subnet_id      = aws_subnet.private.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Groups
+# ---------------------------------------------------------------------------
+
+# Bastion: SSH from anywhere
+resource "aws_security_group" "bastion" {
+  name        = "bastion-demo-bastion-sg"
+  description = "Bastion: SSH from WAN"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "bastion-demo-bastion-sg"
+  }
+}
+
+# Private instances: SSH only from the bastion security group
+resource "aws_security_group" "private" {
+  name        = "bastion-demo-private-sg"
+  description = "Private: SSH from bastion only"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description     = "SSH from bastion"
+    from_port       = 22
+    to_port         = 22
+    protocol        = "tcp"
+    security_groups = [aws_security_group.bastion.id]
+  }
+
+  egress {
+    description = "VPC internal only"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["10.20.0.0/16"]
+  }
+
+  tags = {
+    Name = "bastion-demo-private-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Bastion Host (public subnet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "bastion" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.bastion.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  associate_public_ip_address = true
+
+  # Copy the SSH private key onto the bastion so you can hop to private instances
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+    mkdir -p /home/ec2-user/.ssh
+    cat > /home/ec2-user/.ssh/bastion-demo.pem <<'KEY'
+    ${tls_private_key.bastion.private_key_openssh}
+    KEY
+    chmod 600 /home/ec2-user/.ssh/bastion-demo.pem
+    chown -R ec2-user:ec2-user /home/ec2-user/.ssh
+  USERDATA
+  )
+
+  tags = {
+    Name = "bastion-demo-bastion"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Private Instance (private subnet — no public IP, no internet)
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "private" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.private.id
+  vpc_security_group_ids = [aws_security_group.private.id]
+  key_name               = aws_key_pair.bastion.key_name
+
+  tags = {
+    Name = "bastion-demo-private-app"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH is unreachable, wait and retry."
+}
+
+output "bastion_public_ip" {
+  value = aws_instance.bastion.public_ip
+}
+
+output "private_instance_ip" {
+  value = aws_instance.private.private_ip
+}
+
+output "ssh_to_bastion" {
+  description = "SSH to the bastion host"
+  value       = "ssh -i bastion-demo.pem ec2-user@${aws_instance.bastion.public_ip}"
+}
+
+output "ssh_to_private_from_bastion" {
+  description = "From the bastion, SSH to the private instance (key is pre-installed via cloud-init)"
+  value       = "ssh -i ~/.ssh/bastion-demo.pem ec2-user@${aws_instance.private.private_ip}"
+}

docs/terraform/nginx-alb/README.md

@@ -0,0 +1,231 @@
+---
+title: "Nginx Web Server (Load Balanced)"
+description: "Deploy a VPC with two private EC2 instances running Nginx, fronted by an internet-facing ALB using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - nginx
+  - ec2
+  - elbv2
+  - alb
+  - vpc
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Nginx Web Servers with ALB
+
+> Deploys two Nginx web server instances behind an Application Load Balancer on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy two Nginx web servers behind an internet-facing Application Load Balancer on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC with public and private subnets, an internet gateway and NAT Gateway, route tables, security group, SSH key pair, an application load balancer (ALB) and two EC2 instances with cloud-init user-data that installs and starts Nginx. Only the ALB is reachable from outside the VPC — the Nginx instances live in the **private subnets** and reach the internet only for cloud-init bootstrapping via the NAT Gateway.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-nginx-alb.svg" alt="Nginx + ALB VPC — IGW, two public subnets with ALB ENIs and NAT GW, two private subnets hosting nginx" width="900">
+</p>
+
+**What you'll learn:**
+
+- Configuring the AWS Terraform provider to target Spinifex
+- Creating a VPC with public + private subnets, an IGW and a NAT Gateway
+- Provisioning a multi-AZ internet-facing ALB fronting private workers
+- Provisioning an EC2 instance with cloud-init user-data
+- Generating SSH key pairs with the TLS provider
+
+**What gets created**
+
+| Resource | Name | Purpose |
+|---|---|---|
+| VPC | `nginx-alb-vpc` | Isolated network (10.20.0.0/16) |
+| Public Subnets | `nginx-alb-public-a`, `nginx-alb-public-b` | Two AZs hosting the ALB and NAT Gateway |
+| Private Subnets | `nginx-alb-private-a`, `nginx-alb-private-b` | Two AZs hosting the Nginx workers |
+| Internet Gateway | `nginx-alb-igw` | Routes internet traffic for the public subnets |
+| Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
+| NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
+| Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
+| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Debian 12 with Nginx via cloud-init (private subnets) |
+| ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
+| Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
+| Listener | HTTP :80 | Forwards traffic to the target group |
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/nginx-alb
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Install Load Balancer AMI
+
+Next install the load balancer AMI images, which is used as the AMI disk image for launching the load-balancer and a requirement.
+
+```bash
+spx admin images import --name lb-alpine-3.21.6-x86_64
+```
+
+### Step 3. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+```
+
+
+### Step 4. Specify instance and apply
+
+Next, depending on your architecture and CPU/memory requirements you must specify an instance type to launch.
+
+Either specify an instance type directly (e.g Intel)
+
+```bash
+# AMD instance
+export TF_VAR_instance_type="t3a.small"
+
+# Or, Intel
+export TF_VAR_instance_type="t3.small"
+```
+
+Or alternatively, using the AWS CLI tool query your instance for available types (e.g Intel, AMD, ARM) that support 2 vCPUs and 1 GB RAM.
+
+```bash
+TF_VAR_instance_type=$(aws ec2 describe-instance-types \
+  --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
+  --output text)
+```
+
+Next, apply and launch the template:
+
+```bash
+tofu apply
+```
+
+### Step 5. Verify
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply, and the NAT Gateway must be `available` before cloud-init on the workers can reach the apt repository. If the ALB returns 5xx or HTTP is unreachable, wait and retry — the target group health checks need a moment to mark both instances healthy once Nginx has installed.
+
+The ALB is internet-facing, but the DNS name Spinifex returns (`*.elb.spinifex.local`) will not resolve from your host. Fetch the ALB's public IP with the AWS CLI:
+
+```bash
+ALB_IP=$(aws elbv2 describe-load-balancers --names nginx-alb \
+  --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+  --output text)
+echo "ALB public IP: $ALB_IP"
+```
+
+Then hit the ALB — successive requests should alternate between Server 1 and Server 2:
+
+```bash
+curl http://$ALB_IP
+curl http://$ALB_IP
+```
+
+Open `http://$ALB_IP` in your browser and refresh to see the page alternate content served from each instance.
+
+The Nginx instances themselves only have private IPs (see the `instance_1_private_ip` / `instance_2_private_ip` outputs) and are only reachable from inside the VPC — go through the ALB.
+
+Check target health via AWS CLI:
+
+```bash
+TG_ARN=$(aws elbv2 describe-target-groups \
+  --query 'TargetGroups[0].TargetGroupArn' \
+  --output text)
+
+echo "Target Group ARN: $TG_ARN"
+
+aws elbv2 describe-target-health --target-group-arn $TG_ARN
+```
+
+### Cleanup
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image. Check available AMIs:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```
+
+If missing import:
+
+```bash
+spx admin images import --name debian-12-x86_64
+```
+
+### Provider Connection Refused
+
+Verify Spinifex services are running:
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
+
+### ALB Returns 5xx / Targets Unhealthy
+
+Give the instances a moment to finish cloud-init (Nginx has to install before it can answer health checks). Check target health:
+
+```bash
+TG_ARN=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
+  --query 'TargetGroups[0].TargetGroupArn' --output text)
+aws elbv2 describe-target-health --target-group-arn "$TG_ARN"
+```
+
+If targets stay unhealthy, verify the instances are running:
+
+```bash
+aws ec2 describe-instances --profile spinifex
+```
+
+If cloud-init on the workers never finished, confirm the NAT Gateway is `available` (the private subnets rely on it for outbound apt access):
+
+```bash
+aws ec2 describe-nat-gateways --query 'NatGateways[].[NatGatewayId,State]'
+```
+
+### Nginx Not Responding
+
+The Nginx instances have no public IP, so you can't SSH in directly from your host. If you need to inspect cloud-init logs, launch a small jump host in the same VPC or run commands via the Spinifex console, then:
+
+```bash
+ssh -i nginx-alb-demo.pem ec2-user@<instance_private_ip>
+sudo journalctl -u cloud-init --no-pager
+sudo systemctl status nginx
+```

docs/terraform/nginx-alb/main.tf

@@ -0,0 +1,523 @@
+# Example: Nginx Web Servers with ALB on Spinifex
+#
+# Deploys a VPC with two public subnets (ALB + NAT Gateway) and two private
+# subnets hosting Nginx EC2 instances. An internet-facing Application Load
+# Balancer distributes HTTP traffic between the private workers, and the
+# NAT Gateway gives them outbound internet access so cloud-init can install
+# Nginx from the Debian apt repository.
+#
+# Demonstrates: VPC, public and private subnets, internet gateway, NAT
+# Gateway with Elastic IP, route tables, security group, key pair, cloud-init
+# user-data, EC2 instances, ALB, target group, and listener.
+#
+# Usage:
+#   cd spinifex/docs/terraform/nginx-alb
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply, fetch the ALB's public IP (the *.elb.spinifex.local DNS
+# name does not resolve from your host):
+#
+#   aws elbv2 describe-load-balancers --names nginx-alb \
+#     --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+#     --output text
+#
+# Then:
+#   curl http://<alb_public_ip>    # Load-balanced Nginx (alternates between instances)
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.65.0, < 5.66.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "instance_type" {
+  type = string
+  default = "t3.small"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — point the AWS provider at Spinifex
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2                    = var.spinifex_endpoint
+    iam                    = var.spinifex_endpoint
+    sts                    = var.spinifex_endpoint
+    elasticloadbalancingv2 = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"] # Spinifex system images
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "nginx" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "nginx" {
+  key_name   = "nginx-alb-demo"
+  public_key = tls_private_key.nginx.public_key_openssh
+}
+
+resource "local_file" "nginx_pem" {
+  filename        = "${path.module}/nginx-alb-demo.pem"
+  content         = tls_private_key.nginx.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "nginx-alb-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "nginx-alb-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnets (two AZs for the ALB and NAT Gateway)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public_a" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.1.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-public-a"
+  }
+}
+
+resource "aws_subnet" "public_b" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.2.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-public-b"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Private Subnets (two AZs for the Nginx workers)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "private_a" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.11.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-private-a"
+  }
+}
+
+resource "aws_subnet" "private_b" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.12.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-private-b"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# NAT Gateway — outbound internet for the private subnets
+#
+# Background plumbing: the private-subnet workers need outbound connectivity
+# during cloud-init so apt-get can install Nginx. A single NAT Gateway in
+# public_a provides SNAT for both private subnets via the private route table.
+# ---------------------------------------------------------------------------
+
+resource "aws_eip" "nat" {
+  domain = "vpc"
+
+  tags = {
+    Name = "nginx-alb-nat-eip"
+  }
+}
+
+resource "aws_nat_gateway" "main" {
+  allocation_id = aws_eip.nat.id
+  subnet_id     = aws_subnet.public_a.id
+
+  tags = {
+    Name = "nginx-alb-nat"
+  }
+
+  depends_on = [aws_internet_gateway.igw]
+}
+
+# ---------------------------------------------------------------------------
+# Route Tables — public subnets egress via IGW, private subnets via NAT GW
+# ---------------------------------------------------------------------------
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "nginx-alb-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public_a" {
+  subnet_id      = aws_subnet.public_a.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table_association" "public_b" {
+  subnet_id      = aws_subnet.public_b.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = aws_nat_gateway.main.id
+  }
+
+  tags = {
+    Name = "nginx-alb-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private_a" {
+  subnet_id      = aws_subnet.private_a.id
+  route_table_id = aws_route_table.private.id
+}
+
+resource "aws_route_table_association" "private_b" {
+  subnet_id      = aws_subnet.private_b.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "web" {
+  name        = "nginx-alb-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nginx-alb-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instances — two Nginx servers with distinct landing pages
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx_1" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = var.instance_type
+
+  subnet_id              = aws_subnet.private_a.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  # Workers pull nginx from apt via the NAT Gateway — creation must wait
+  # until the NAT Gateway is available so cloud-init can reach the repos.
+  depends_on = [aws_nat_gateway.main]
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    apt-get update -y
+    apt-get install -y nginx
+
+    INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id 2>/dev/null || hostname)
+    cat > /var/www/html/index.html <<HTML
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex ALB Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p><strong>Instance:</strong> $INSTANCE_ID (Server 1)</p>
+      <p>This Nginx server is behind an Application Load Balancer.</p>
+      <hr>
+      <p><small>Provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-alb-1"
+  }
+}
+
+resource "aws_instance" "nginx_2" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = var.instance_type
+
+  subnet_id              = aws_subnet.private_b.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  # Workers pull nginx from apt via the NAT Gateway — creation must wait
+  # until the NAT Gateway is available so cloud-init can reach the repos.
+  depends_on = [aws_nat_gateway.main]
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    apt-get update -y
+    apt-get install -y nginx
+
+    INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id 2>/dev/null || hostname)
+    cat > /var/www/html/index.html <<HTML
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex ALB Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p><strong>Instance:</strong> $INSTANCE_ID (Server 2)</p>
+      <p>This Nginx server is behind an Application Load Balancer.</p>
+      <hr>
+      <p><small>Provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-alb-2"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Application Load Balancer
+# ---------------------------------------------------------------------------
+
+resource "aws_lb" "web" {
+  name               = "nginx-alb"
+  internal           = false
+  load_balancer_type = "application"
+  security_groups    = [aws_security_group.web.id]
+  subnets            = [aws_subnet.public_a.id, aws_subnet.public_b.id]
+
+  tags = {
+    Name = "nginx-alb"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Target Group — HTTP health-checked on port 80
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_target_group" "nginx" {
+  name     = "nginx-alb-tg"
+  port     = 80
+  protocol = "HTTP"
+  vpc_id   = aws_vpc.main.id
+
+  health_check {
+    path                = "/"
+    protocol            = "HTTP"
+    healthy_threshold   = 2
+    unhealthy_threshold = 3
+    timeout             = 5
+    interval            = 10
+  }
+
+  tags = {
+    Name = "nginx-alb-tg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Register both instances as targets
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_target_group_attachment" "nginx_1" {
+  target_group_arn = aws_lb_target_group.nginx.arn
+  target_id        = aws_instance.nginx_1.id
+  port             = 80
+}
+
+resource "aws_lb_target_group_attachment" "nginx_2" {
+  target_group_arn = aws_lb_target_group.nginx.arn
+  target_id        = aws_instance.nginx_2.id
+  port             = 80
+}
+
+# ---------------------------------------------------------------------------
+# Listener — forward HTTP :80 to the target group
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_listener" "http" {
+  load_balancer_arn = aws_lb.web.arn
+  port              = 80
+  protocol          = "HTTP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.nginx.arn
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = <<-EOT
+    EC2 instances can take 30+ seconds to boot after apply — if HTTP is
+    unreachable, wait and retry.
+
+    The Nginx instances have private IPs only. The ALB DNS name ends in
+    .elb.spinifex.local and will not resolve from your host, so fetch the
+    ALB's public IP with:
+
+      aws elbv2 describe-load-balancers --names nginx-alb \
+        --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+        --output text
+
+    Then: curl http://<that-ip>
+  EOT
+}
+
+output "alb_name" {
+  value = aws_lb.web.name
+}
+
+output "alb_arn" {
+  value = aws_lb.web.arn
+}
+
+output "alb_dns_name" {
+  value = aws_lb.web.dns_name
+}
+
+output "instance_1_id" {
+  value = aws_instance.nginx_1.id
+}
+
+output "instance_1_private_ip" {
+  value = aws_instance.nginx_1.private_ip
+}
+
+output "instance_2_id" {
+  value = aws_instance.nginx_2.id
+}
+
+output "instance_2_private_ip" {
+  value = aws_instance.nginx_2.private_ip
+}

docs/terraform/nginx-webserver/README.md

@@ -0,0 +1,129 @@
+---
+title: "Nginx Web Server"
+description: "Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - nginx
+  - ec2
+  - vpc
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Nginx Web Server
+
+> Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy a complete Nginx web server on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC, public subnet, internet gateway, route table, security group, SSH key pair, and an EC2 instance with cloud-init user-data that installs and starts Nginx.
+
+**What you'll learn:**
+
+- Configuring the AWS Terraform provider to target Spinifex
+- Creating a VPC with public internet access
+- Provisioning an EC2 instance with cloud-init user-data
+- Generating SSH key pairs with the TLS provider
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/nginx-webserver
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 3. Verify
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+After apply completes, use the outputs to test:
+
+```bash
+curl http://<public_ip>
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+```
+
+Open the `web_url` output in your browser to see the Nginx welcome page.
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image. Check available AMIs:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```
+
+### Provider Connection Refused
+
+Verify Spinifex services are running:
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
+
+### SSH Connection Timeout
+
+Check that the security group allows inbound SSH (port 22) and that the instance has a public IP assigned. Verify the instance is running:
+
+```bash
+aws ec2 describe-instances --profile spinifex
+```
+
+### Nginx Not Responding
+
+SSH into the instance and check cloud-init logs:
+
+```bash
+ssh -i nginx-demo.pem ec2-user@<public_ip>
+sudo journalctl -u cloud-init --no-pager
+sudo systemctl status nginx
+```

docs/terraform/nginx-webserver/main.tf

@@ -0,0 +1,286 @@
+# Example 1: Nginx Web Server on Spinifex
+#
+# Deploys a VPC with a public subnet and an EC2 instance running Nginx.
+# Demonstrates: VPC, subnet, internet gateway, route table, security group,
+# key pair, cloud-init user-data, and EC2 instance provisioning.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/01-nginx-webserver
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>        # Nginx welcome page
+#   ssh -i nginx-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — point the AWS provider at Spinifex
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"] # Spinifex system images
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "nginx" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "nginx" {
+  key_name   = "nginx-demo"
+  public_key = tls_private_key.nginx.public_key_openssh
+}
+
+resource "local_file" "nginx_pem" {
+  filename        = "${path.module}/nginx-demo.pem"
+  content         = tls_private_key.nginx.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.10.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "nginx-demo-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway — gives the public subnet a route to the internet
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "nginx-demo-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.10.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "nginx-demo-public"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Route Table — send 0.0.0.0/0 through the internet gateway
+# ---------------------------------------------------------------------------
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "nginx-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "web" {
+  name        = "nginx-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nginx-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Debian 12 with Nginx installed via cloud-init
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install Nginx
+    apt-get update -y
+    apt-get install -y nginx
+
+    # Write a custom landing page
+    cat > /var/www/html/index.html <<'HTML'
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p>This Nginx server was deployed with Terraform on Spinifex infrastructure.</p>
+      <hr>
+      <p><small>Instance provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    # Ensure Nginx is running
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.nginx.id
+}
+
+output "public_ip" {
+  value = aws_instance.nginx.public_ip
+}
+
+output "ssh_command" {
+  value = "ssh -i nginx-demo.pem ec2-user@${aws_instance.nginx.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.nginx.public_ip}"
+}

docs/terraform/s3-webapp/README.md

@@ -0,0 +1,159 @@
+---
+title: "S3-Backed Web App"
+description: "Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - s3
+  - predastore
+  - flask
+  - webapp
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Predastore (S3)"
+    url: "https://github.com/mulgadc/predastore"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: S3-Backed Web App
+
+> Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy an EC2 instance running a Flask file-sharing web application backed by S3 (Predastore). Users can upload files through a web form and browse uploaded content — demonstrating Terraform managing both compute and object storage together.
+
+**Architecture:**
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-s3-webapp.svg" alt="S3 webapp — browser to Flask EC2 instance to Predastore via S3 API" width="900">
+</p>
+
+**What you'll learn:**
+
+- Configuring the AWS provider with both Spinifex and Predastore endpoints
+- Creating S3 buckets on Predastore via Terraform
+- Deploying a Python webapp with cloud-init that talks to S3
+- Passing credentials and configuration to instances via user-data
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- Predastore running (S3 API on port 8443)
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+- The EC2 instance must be able to reach Predastore — use the host's br-wan IP, not localhost
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/s3-webapp
+```
+
+Or create the files manually and paste the full configuration below.
+
+### Step 2. Create terraform.tfvars
+
+Before deploying, create a `terraform.tfvars` with your Predastore credentials. The `predastore_host` must be reachable from inside the VPC — use the host's br-wan or LAN IP, not localhost.
+
+<!-- INCLUDE: terraform.tfvars.example lang:hcl -->
+
+### Step 3. Create main.tf
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 4. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+### Step 5. Test the Application
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry.
+
+Open the `web_url` output in your browser. You should see the file browser UI. Upload a file and verify it appears in the list.
+
+```bash
+# Verify via CLI
+curl http://<public_ip>
+
+# Check the S3 bucket directly
+aws s3 ls s3://webapp-uploads/ --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Clean Up
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### Predastore Connection Refused from Instance
+
+The EC2 instance cannot reach `localhost` on the host. Set `predastore_host` to the host's br-wan or LAN IP address:
+
+```hcl
+predastore_host = "192.168.1.10:8443"
+```
+
+### S3 Bucket Creation Fails
+
+Verify Predastore is running and accessible:
+
+```bash
+curl -k https://localhost:8443/
+aws s3 ls --profile spinifex --endpoint-url https://localhost:8443
+```
+
+### Flask App Not Starting
+
+SSH into the instance and check the service:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+sudo systemctl status s3-webapp
+sudo journalctl -u s3-webapp --no-pager -n 50
+```
+
+### Upload Fails or Files Not Appearing
+
+Check that the S3 credentials in `/opt/webapp/.env` are correct and that the bucket exists:
+
+```bash
+ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+cat /opt/webapp/.env
+/opt/webapp/venv/bin/python -c "import boto3; print('boto3 OK')"
+```
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```

docs/terraform/s3-webapp/main.tf

@@ -0,0 +1,445 @@
+# Example 3: S3-Backed Web Application
+#
+# Deploys an EC2 instance running a simple file-sharing webapp backed by S3
+# (Predastore). Users can upload files through a web form and browse uploaded
+# content — demonstrating Terraform managing both compute (Spinifex) and
+# object storage (Predastore) resources together.
+#
+# Architecture:
+#
+#   Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
+#                         │
+#                         ▼ S3 API (boto3)
+#                     Predastore (port 8443)
+#
+# Prerequisites:
+#   - Spinifex services running (gateway on port 9999)
+#   - Predastore running (S3 API on port 8443)
+#   - The EC2 instance must be able to reach the Predastore endpoint.
+#     Set `predastore_host` to the IP reachable from inside the VPC
+#     (e.g. the host's br-wan IP, NOT localhost).
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/examples/03-s3-webapp
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply:
+#   curl http://<public_ip>          # File browser UI
+#   ssh -i s3-webapp-demo.pem ec2-user@<public_ip>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint (EC2/IAM)"
+}
+
+variable "predastore_endpoint" {
+  type        = string
+  default     = "https://localhost:8443"
+  description = "Predastore S3 endpoint (for Terraform to create buckets)"
+}
+
+variable "predastore_host" {
+  type        = string
+  description = "Predastore host:port reachable from inside the VPC (e.g. 192.168.1.10:8443)"
+}
+
+variable "s3_access_key" {
+  type        = string
+  description = "S3 access key for Predastore"
+}
+
+variable "s3_secret_key" {
+  type        = string
+  sensitive   = true
+  description = "S3 secret key for Predastore"
+}
+
+variable "bucket_name" {
+  type    = string
+  default = "webapp-uploads"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — EC2 via Spinifex gateway, S3 via Predastore
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region     = var.region
+  access_key = var.s3_access_key
+  secret_key = var.s3_secret_key
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    s3  = var.predastore_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+  }
+
+  s3_use_path_style = true
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# S3 Bucket (created on Predastore)
+# ---------------------------------------------------------------------------
+
+resource "aws_s3_bucket" "uploads" {
+  bucket = var.bucket_name
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "webapp" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "webapp" {
+  key_name   = "s3-webapp-demo"
+  public_key = tls_private_key.webapp.public_key_openssh
+}
+
+resource "local_file" "webapp_pem" {
+  filename        = "${path.module}/s3-webapp-demo.pem"
+  content         = tls_private_key.webapp.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC + Public Subnet
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.30.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "s3-webapp-demo-vpc"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "s3-webapp-demo-igw"
+  }
+}
+
+resource "aws_subnet" "public" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.30.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "s3-webapp-demo-public"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id      = aws_subnet.public.id
+  route_table_id = aws_route_table.public.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "webapp" {
+  name        = "s3-webapp-demo-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "s3-webapp-demo-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instance — Flask webapp that talks to Predastore S3
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "webapp" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.public.id
+  vpc_security_group_ids = [aws_security_group.webapp.id]
+  key_name               = aws_key_pair.webapp.key_name
+
+  associate_public_ip_address = true
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    # Install dependencies
+    apt-get update -y
+    apt-get install -y python3-pip python3-venv
+
+    # Create app directory and virtualenv
+    mkdir -p /opt/webapp
+    python3 -m venv /opt/webapp/venv
+    /opt/webapp/venv/bin/pip install flask boto3
+
+    # Write S3 credentials config
+    cat > /opt/webapp/.env <<'ENVFILE'
+    S3_ENDPOINT=https://${var.predastore_host}
+    S3_BUCKET=${var.bucket_name}
+    S3_ACCESS_KEY=${var.s3_access_key}
+    S3_SECRET_KEY=${var.s3_secret_key}
+    S3_REGION=${var.region}
+    ENVFILE
+
+    # Write the Flask application
+    cat > /opt/webapp/app.py <<'PYEOF'
+    import os, io, urllib3
+    from flask import Flask, request, redirect, url_for, Response
+
+    # Suppress TLS warnings for self-signed certs
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    # Load env
+    env = {}
+    with open("/opt/webapp/.env") as f:
+        for line in f:
+            line = line.strip()
+            if "=" in line and not line.startswith("#"):
+                k, v = line.split("=", 1)
+                env[k] = v
+
+    import boto3
+    from botocore.config import Config
+
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=env["S3_ENDPOINT"],
+        aws_access_key_id=env["S3_ACCESS_KEY"],
+        aws_secret_access_key=env["S3_SECRET_KEY"],
+        region_name=env["S3_REGION"],
+        verify=False,
+        config=Config(s3={"addressing_style": "path"}),
+    )
+
+    BUCKET = env["S3_BUCKET"]
+    app = Flask(__name__)
+
+    @app.route("/")
+    def index():
+        # List objects in the bucket
+        try:
+            resp = s3.list_objects_v2(Bucket=BUCKET)
+            objects = resp.get("Contents", [])
+        except Exception as e:
+            objects = []
+
+        rows = ""
+        for obj in objects:
+            key = obj["Key"]
+            size = obj["Size"]
+            rows += f'<tr><td><a href="/files/{key}">{key}</a></td><td>{size} bytes</td></tr>\n'
+
+        return f"""<!DOCTYPE html>
+    <html>
+    <head><title>Spinifex S3 File Browser</title></head>
+    <body style="font-family: sans-serif; max-width: 700px; margin: 40px auto;">
+      <h1>Spinifex S3 File Browser</h1>
+      <p>Bucket: <code>{BUCKET}</code></p>
+
+      <h2>Upload a File</h2>
+      <form method="POST" action="/upload" enctype="multipart/form-data">
+        <input type="file" name="file" required>
+        <button type="submit">Upload</button>
+      </form>
+
+      <h2>Files</h2>
+      <table border="1" cellpadding="6" cellspacing="0" style="border-collapse: collapse;">
+        <tr><th>Key</th><th>Size</th></tr>
+        {rows if rows else '<tr><td colspan="2">No files yet</td></tr>'}
+      </table>
+
+      <hr>
+      <p><small>Powered by Spinifex + Predastore</small></p>
+    </body>
+    </html>"""
+
+    @app.route("/upload", methods=["POST"])
+    def upload():
+        f = request.files.get("file")
+        if not f or not f.filename:
+            return redirect("/")
+        s3.put_object(Bucket=BUCKET, Key=f.filename, Body=f.read())
+        return redirect("/")
+
+    @app.route("/files/<path:key>")
+    def download(key):
+        try:
+            obj = s3.get_object(Bucket=BUCKET, Key=key)
+            return Response(
+                obj["Body"].read(),
+                headers={"Content-Disposition": f'inline; filename="{key}"'},
+            )
+        except Exception:
+            return "Not found", 404
+
+    if __name__ == "__main__":
+        app.run(host="0.0.0.0", port=80)
+    PYEOF
+
+    # Create a systemd service so the webapp starts on boot
+    cat > /etc/systemd/system/s3-webapp.service <<'SVCEOF'
+    [Unit]
+    Description=S3 File Browser Webapp
+    After=network.target
+
+    [Service]
+    Type=simple
+    ExecStart=/opt/webapp/venv/bin/python /opt/webapp/app.py
+    WorkingDirectory=/opt/webapp
+    Restart=always
+    RestartSec=3
+
+    [Install]
+    WantedBy=multi-user.target
+    SVCEOF
+
+    systemctl daemon-reload
+    systemctl enable s3-webapp
+    systemctl start s3-webapp
+  USERDATA
+  )
+
+  tags = {
+    Name = "s3-webapp-demo"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = "EC2 instances can take 30+ seconds to boot after apply. If SSH or HTTP is unreachable, wait and retry."
+}
+
+output "instance_id" {
+  value = aws_instance.webapp.id
+}
+
+output "public_ip" {
+  value = aws_instance.webapp.public_ip
+}
+
+output "bucket_name" {
+  value = aws_s3_bucket.uploads.id
+}
+
+output "ssh_command" {
+  value = "ssh -i s3-webapp-demo.pem ec2-user@${aws_instance.webapp.public_ip}"
+}
+
+output "web_url" {
+  value = "http://${aws_instance.webapp.public_ip}"
+}

docs/terraform/s3-webapp/terraform.tfvars.example

@@ -0,0 +1,14 @@
+# Copy this to terraform.tfvars and fill in your values.
+#
+# The predastore_host must be reachable from INSIDE the VPC guest — not
+# localhost. Use the host's br-wan or LAN IP, e.g. "192.168.1.10:8443".
+
+predastore_host = "192.168.1.10:8443"
+s3_access_key   = "AKIAIOSFODNN7EXAMPLE"
+s3_secret_key   = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+
+# Optional overrides:
+# spinifex_endpoint    = "https://localhost:9999"
+# predastore_endpoint  = "https://localhost:8443"
+# bucket_name          = "webapp-uploads"
+# region               = "ap-southeast-2"

docs/update/README.md

@@ -0,0 +1,124 @@
+---
+title: "Updating Spinifex"
+description: "Upgrade an existing Spinifex installation to the latest release."
+category: "Getting Started"
+tags:
+  - update
+  - upgrade
+  - migrate
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Single-Node Install"
+    url: "https://docs.mulgadc.com/docs/install"
+---
+
+# Updating Spinifex
+
+> Upgrade an existing Spinifex installation to the latest release.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Manual Upgrade](#manual-upgrade)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Updating Spinifex is the same command used to install it. The installer detects an existing installation, downloads the latest binary and runs any pending configuration migrations before restarting services.
+
+For operators who want to review migrations before they are applied, a manual upgrade path is also supported.
+
+## Instructions
+
+## Step 1. Re-run the Installer
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash
+```
+
+That's it. The installer will:
+
+1. Download and install the latest Spinifex binary.
+2. Reinstall systemd units so new services are picked up.
+3. Run any pending configuration migrations automatically (equivalent to `spx admin upgrade --yes`).
+4. Restart `spinifex.target` if the services were already running.
+
+## Step 2. Verify
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+If this returns a list of instance types, your upgrade is complete.
+
+## Manual Upgrade
+
+If you prefer to review pending migrations before they are applied, Spinifex supports running `spx admin init` to allow you to verify config file migrations.
+
+## Step 1. Install the New Binary Without Running Migrations
+
+```bash
+curl -fsSL https://install.mulgadc.com | INSTALL_SPINIFEX_SKIP_MIGRATE=1 bash
+```
+
+The installer will download the new binary and reinstall systemd units, but will **not** apply any configuration migrations.
+
+## Step 2. Review Pending Migrations
+
+```bash
+sudo spx admin upgrade
+```
+
+The command prints the current version of each config file, the migrations that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching your config.
+
+## Step 3. Apply Migrations
+
+When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply non-interactively:
+
+```bash
+sudo spx admin upgrade --yes
+```
+
+## Step 4. Restart Services
+
+Migrations modify config files on disk but do not restart running services. Apply the new config with:
+
+```bash
+sudo systemctl restart spinifex.target
+```
+
+## Troubleshooting
+
+### No Pending Config Migrations
+
+```
+No pending config migrations.
+```
+
+Your config is already at the latest version. Nothing to do.
+
+### No Spinifex Installation Found
+
+```
+No Spinifex installation found at /etc/spinifex
+Run 'spx admin init' first.
+```
+
+`spx admin upgrade` requires an initialized installation. If this is a fresh host, follow the [Single-Node Install](/docs/install) guide instead.
+
+### Migration Failure
+
+If a migration fails, the installer and `spx admin upgrade` exit non-zero and leave the config in its prior state where possible. Review the error output, then re-run `sudo spx admin upgrade` once the underlying issue is resolved.
+
+### Services Did Not Pick Up New Config
+
+Migrations edit config files on disk but the running daemons continue to use the config they loaded at start-up. Restart with:
+
+```bash
+sudo systemctl restart spinifex.target
+```

docs/vpc-networking/README.md

@@ -1,7 +1,7 @@
 ---
 title: "VPC Networking"
 description: "How Spinifex implements AWS-compatible VPC networking with public and private subnets, security groups, and Elastic IPs using OVN."
-category: "Environments"
+category: "Compute"
 tags:
   - vpc
   - networking
@@ -27,49 +27,20 @@ Instances can operate in two modes: **private** (overlay-only, no WAN access) or
 
 Spinifex maps AWS VPC concepts directly to OVN constructs:
 
-| AWS Concept | OVN Construct | What It Does |
-|-------------|---------------|--------------|
-| VPC | Logical Router | Isolates tenant networks, routes between subnets |
-| Subnet | Logical Switch + DHCP | L2 broadcast domain with automatic IP assignment |
-| ENI | Logical Switch Port | Per-instance network interface with MAC/IP binding |
-| Internet Gateway | External Switch + NAT | Connects VPC router to physical WAN |
-| Security Group | Port Group + ACLs | Stateful firewall rules enforced in OVS datapath |
-| Elastic IP | `dnat_and_snat` NAT rule | Static 1:1 NAT between public and private IP |
+| AWS Concept      | OVN Construct            | What It Does                                       |
+| ---------------- | ------------------------ | -------------------------------------------------- |
+| VPC              | Logical Router           | Isolates tenant networks, routes between subnets   |
+| Subnet           | Logical Switch + DHCP    | L2 broadcast domain with automatic IP assignment   |
+| ENI              | Logical Switch Port      | Per-instance network interface with MAC/IP binding |
+| Internet Gateway | External Switch + NAT    | Connects VPC router to physical WAN                |
+| Security Group   | Port Group + ACLs        | Stateful firewall rules enforced in OVS datapath   |
+| Elastic IP       | `dnat_and_snat` NAT rule | Static 1:1 NAT between public and private IP       |
 
 ## Network Path
 
-```
-                        Physical Network (WAN)
-                               │
-                        ┌──────┴──────┐
-                        │ br-wan │  OVS bridge (physical uplink)
-                        └──────┬──────┘
-                               │  ovn-bridge-mappings: external:br-wan
-                               │
-                    ┌──────────┴───────────────────────────┐
-                    │     VPC Logical Router               │
-                    │                                      │
-                    │  Gateway port ──── ext switch ─────── localnet port
-                    │    (public IP)     (OVN)              (maps to br-wan)
-                    │                                      │
-                    │  NAT rules:                          │
-                    │    SNAT: VPC CIDR → gateway IP       │
-                    │    dnat_and_snat: public ↔ private   │
-                    │                                      │
-                    │  Routes:                             │
-                    │    0.0.0.0/0 → WAN gateway           │
-                    │                                      │
-                    ├────────────┬──────────────┤          │
-                    │            │              │          │
-              ┌─────┴────┐ ┌─────┴────┐         │          │
-              │ subnet-a │ │ subnet-b │         │          │
-              │ (public) │ │ (private)│         │          │
-              │10.0.1/24 │ │10.0.2/24 │         │          │
-              │  eni-1   │ │  eni-3   │         │          │
-              │  eni-2   │ │  eni-4   │         │          │
-              └──────────┘ └──────────┘         │          │
-                                                └──────────┘
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-network-path.svg" alt="VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs" width="900">
+</p>
 
 Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay
 NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int`
@@ -86,9 +57,9 @@ Instances get a private IP only. They can communicate with other instances in th
 same VPC (even across subnets and hosts via the overlay). They cannot reach the
 internet or be reached from the WAN.
 
-```
-Instance (10.0.2.5) ──→ VPC Router ──→ [no default route] ──→ dropped
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
+</p>
 
 If the VPC has an IGW attached, private subnet instances CAN reach the internet
 via the VPC router's SNAT rule (outbound only — they share the gateway IP). They
@@ -99,10 +70,9 @@ still cannot be reached from the WAN because they have no public IP.
 Instances get both a private IP and a public IP. The public IP is a 1:1 NAT
 managed by OVN — the instance OS only sees its private IP.
 
-```
-Outbound: Instance (10.0.1.5) ──→ SNAT ──→ exits as 203.0.113.10
-Inbound:  203.0.113.10 ──→ DNAT ──→ arrives at 10.0.1.5
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-public-subnet-flow.svg" alt="Public subnet — outbound SNAT and inbound DNAT between private and public IPs" width="900">
+</p>
 
 **Requirements for a public subnet:**
 
@@ -112,14 +82,14 @@ Inbound:  203.0.113.10 ──→ DNAT ──→ arrives at 10.0.1.5
 
 ## Comparison
 
-| | Private Subnet | Public Subnet |
-|---|---|---|
-| Private IP | Yes | Yes |
-| Public IP | No | Auto-assigned from pool |
-| Outbound internet | Only if VPC has IGW (shared SNAT) | Yes (own public IP via SNAT) |
-| Inbound from WAN | No | Yes (via 1:1 NAT to public IP) |
-| Instance sees public IP? | N/A | No — only sees private IP |
-| Elastic IP support | Only if explicitly associated | Yes |
+|                          | Private Subnet                    | Public Subnet                  |
+| ------------------------ | --------------------------------- | ------------------------------ |
+| Private IP               | Yes                               | Yes                            |
+| Public IP                | No                                | Auto-assigned from pool        |
+| Outbound internet        | Only if VPC has IGW (shared SNAT) | Yes (own public IP via SNAT)   |
+| Inbound from WAN         | No                                | Yes (via 1:1 NAT to public IP) |
+| Instance sees public IP? | N/A                               | No — only sees private IP      |
+| Elastic IP support       | Only if explicitly associated     | Yes                            |
 
 ## External Connectivity Modes
 
@@ -191,36 +161,20 @@ dns_servers = ["192.168.1.1", "8.8.8.8"]
 Regardless of whether IPs come from a static range or DHCP, the OVN behavior
 is identical:
 
-```
-                          Two DHCP conversations (completely independent):
-
- ┌─────────────────────────────────────────────────────────────────────┐
- │ HOST ↔ ROUTER DHCP (on br-ext, via veth pair → br-wan)              │
- │   Spinifex asks router: "give me a public IP for this VM"           │
- │   Router responds: 192.168.1.75                                     │
- │   Spinifex stores IP, tells OVN: dnat_and_snat(192.168.1.75 ↔       │
- │                                                 172.31.0.4)         │
- │   (Only used in DHCP source mode. Static mode picks from range.)    │
- └─────────────────────────────────────────────────────────────────────┘
-
- ┌─────────────────────────────────────────────────────────────────────┐
- │ VM ↔ OVN DHCP (on tap → br-int → OVN logical switch)                │
- │   cloud-init asks: "what's my IP?"                                  │
- │   OVN DHCP responds: "you are 172.31.0.4"                           │
- │   VM never sees 192.168.1.x or the router                           │
- └─────────────────────────────────────────────────────────────────────┘
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-dhcp-conversations.svg" alt="Two independent DHCP conversations — host-to-router and VM-to-OVN" width="900">
+</p>
 
 ### Choosing Static vs DHCP
 
-| | Static Range | DHCP Source |
-|---|---|---|
-| **Public IPs from** | Admin-defined `range_start`..`range_end` | Router's DHCP server |
-| **IP predictability** | You know the exact range | Router assigns whatever is available |
-| **Setup effort** | Must reserve range, shrink router DHCP scope | Just set `source = "dhcp"` |
-| **Dependency** | None | Requires `dhclient` on host, working router DHCP |
-| **Best for** | Datacenters, ISP blocks, production | Homelabs, dev environments, shared networks |
-| **Capacity** | Exact: `range_end - range_start` IPs | Limited by router's DHCP pool size |
+|                       | Static Range                                 | DHCP Source                                      |
+| --------------------- | -------------------------------------------- | ------------------------------------------------ |
+| **Public IPs from**   | Admin-defined `range_start`..`range_end`     | Router's DHCP server                             |
+| **IP predictability** | You know the exact range                     | Router assigns whatever is available             |
+| **Setup effort**      | Must reserve range, shrink router DHCP scope | Just set `source = "dhcp"`                       |
+| **Dependency**        | None                                         | Requires `dhclient` on host, working router DHCP |
+| **Best for**          | Datacenters, ISP blocks, production          | Homelabs, dev environments, shared networks      |
+| **Capacity**          | Exact: `range_end - range_start` IPs         | Limited by router's DHCP pool size               |
 
 Both support the same AWS features: public subnets, Elastic IPs, security groups,
 DescribeInstances showing public IPs.
@@ -257,16 +211,16 @@ communicate within their VPC.
 
 ## Mode Comparison
 
-| Capability | `pool` (static) | `pool` (dhcp) | `nat` | Disabled |
-|------------|-----------------|---------------|-------|----------|
-| Outbound internet | Yes | Yes | Yes | No |
-| Inbound from WAN | Yes (1:1 NAT) | Yes (1:1 NAT) | No | No |
-| Public subnets | Yes | Yes | No | No |
-| Auto-assign public IPs | Yes | Yes | No | No |
-| Elastic IPs | Yes | Yes | No | No |
-| DescribeInstances shows public IP | Yes | Yes | No | No |
-| Admin must reserve IP range | Yes | No | No | No |
-| Needs router DHCP | No | Yes | Optional | No |
+| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`    | Disabled |
+| --------------------------------- | --------------- | ------------- | -------- | -------- |
+| Outbound internet                 | Yes             | Yes           | Yes      | No       |
+| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | No       | No       |
+| Public subnets                    | Yes             | Yes           | No       | No       |
+| Auto-assign public IPs            | Yes             | Yes           | No       | No       |
+| Elastic IPs                       | Yes             | Yes           | No       | No       |
+| DescribeInstances shows public IP | Yes             | Yes           | No       | No       |
+| Admin must reserve IP range       | Yes             | No            | No       | No       |
+| Needs router DHCP                 | No              | Yes           | Optional | No       |
 
 If you start with `nat` and later need public subnets, switch to `pool` and
 define a range (or use `source = "dhcp"`) — no data migration needed.
@@ -286,10 +240,10 @@ the best approach:
 
 Every Spinifex node has two OVS bridges:
 
-| Bridge | Purpose | Ports |
-|--------|---------|-------|
-| `br-int` | VM overlay traffic (Geneve tunnels) | VM TAP devices, tunnel ports |
-| `br-wan` | WAN uplink for public subnet traffic | macvlan on WAN NIC |
+| Bridge   | Purpose                              | Ports                        |
+| -------- | ------------------------------------ | ---------------------------- |
+| `br-int` | VM overlay traffic (Geneve tunnels)  | VM TAP devices, tunnel ports |
+| `br-wan` | WAN uplink for public subnet traffic | macvlan on WAN NIC           |
 
 `br-int` is always created by `setup-ovn.sh`. `br-wan` is created only when
 a WAN bridge is configured (auto-detected or via `--wan-bridge`, required for public subnets / external connectivity).
@@ -297,9 +251,9 @@ a WAN bridge is configured (auto-detected or via `--wan-bridge`, required for pu
 The connection between them is logical, not physical — OVN's `localnet` port type
 maps the logical external switch to `br-wan` via `ovn-bridge-mappings`.
 
-```
-VM TAP ──→ br-int ──→ OVN logical pipeline ──→ localnet ──→ br-wan ──→ WAN
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-datapath-localnet.svg" alt="Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN" width="900">
+</p>
 
 ## Bridge Modes
 
@@ -312,16 +266,16 @@ is the fallback for single-NIC hosts where the WAN NIC is also the SSH NIC.
 The WAN NIC is added directly to `br-wan` as an OVS port. OVS sees all
 traffic on the wire — any MAC, any protocol. No filtering, no workarounds.
 
-```
-Management NIC (host IP, SSH)
-WAN NIC ──→ br-wan (OVS port) ──→ OVN localnet
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-direct-bridge.svg" alt="Direct bridge mode — dedicated WAN NIC added directly to br-wan" width="900">
+</p>
 
 **Use when:** The host has a NIC dedicated to external/WAN traffic that is NOT
 used for SSH or management. Datacenter servers with separate management +
 public NICs. Homelab hosts with 2+ NICs.
 
 **Benefits over macvlan:**
+
 - **Distributed NAT**: DNAT processed on the VM's own chassis (no hairpin)
 - **No MAC workarounds**: OVS sees all frames, any MAC
 - **Host can reach VMs**: No macvlan parent↔child isolation
@@ -338,22 +292,15 @@ A macvlan sub-interface is created in bridge mode off the WAN NIC, and that
 macvlan is added to br-wan. The host keeps its IP on the original NIC —
 SSH stays up.
 
-```
-WAN NIC (host IP — unchanged)
-  │
-  ├── host traffic (SSH, management, overlay)
-  │
-  └── spx-ext-{nic} (macvlan, bridge mode, no IP)
-        │
-   br-wan
-        │
-   localnet → OVN ext switch
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-macvlan.svg" alt="Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan" width="900">
+</p>
 
 **Use when:** The WAN NIC is the same NIC used for SSH. Single-NIC homelabs,
 edge deployments. Adding the NIC directly to OVS would break host connectivity.
 
 **Limitations:**
+
 - Centralized NAT only (macvlan filters unicast not matching its MAC)
 - Host cannot reach VMs at their public IPs (macvlan isolation)
 - vpcd must align macvlan MAC with OVN router MAC on startup
@@ -365,23 +312,23 @@ sudo setup-ovn.sh --macvlan --wan-iface=eth0
 
 ### Mode Comparison
 
-| Capability | Direct bridge | Macvlan |
-|------------|--------------|---------|
-| NAT mode | Distributed (local DNAT) | Centralized (gateway hairpin) |
-| Multi-node external perf | Optimal | Hairpin through gateway chassis |
-| Host → VM public IP | Works | Blocked (kernel isolation) |
-| MAC alignment needed | No | Yes (fragile startup step) |
-| Gratuitous ARPs needed | No | Yes (`nat-addresses=router`) |
-| Works on single-NIC | No (breaks SSH) | Yes |
-| DHCP lease stability | Clean | Fragile (MAC changes post-lease) |
+| Capability               | Direct bridge            | Macvlan                          |
+| ------------------------ | ------------------------ | -------------------------------- |
+| NAT mode                 | Distributed (local DNAT) | Centralized (gateway hairpin)    |
+| Multi-node external perf | Optimal                  | Hairpin through gateway chassis  |
+| Host → VM public IP      | Works                    | Blocked (kernel isolation)       |
+| MAC alignment needed     | No                       | Yes (fragile startup step)       |
+| Gratuitous ARPs needed   | No                       | Yes (`nat-addresses=router`)     |
+| Works on single-NIC      | No (breaks SSH)          | Yes                              |
+| DHCP lease stability     | Clean                    | Fragile (MAC changes post-lease) |
 
 ### How setup-ovn.sh Decides
 
-| Flags | Result |
-|-------|--------|
-| `--wan-bridge=br-wan --wan-iface=eth1` | Direct bridge: NIC added to br-wan |
-| `--macvlan --wan-iface=eth0` | Macvlan: sub-interface created, added to br-wan |
-| (no WAN bridge) | Only br-int created, no WAN connectivity |
+| Flags                                  | Result                                          |
+| -------------------------------------- | ----------------------------------------------- |
+| `--wan-bridge=br-wan --wan-iface=eth1` | Direct bridge: NIC added to br-wan              |
+| `--macvlan --wan-iface=eth0`           | Macvlan: sub-interface created, added to br-wan |
+| (no WAN bridge)                        | Only br-int created, no WAN connectivity        |
 
 ### Setup (Macvlan Mode)
 
@@ -402,12 +349,12 @@ macvlan in `bridge` mode creates a virtual interface that shares the parent NIC'
 physical wire but has its own MAC address. The Linux kernel blocks direct L2
 frames between a parent interface and its macvlan children.
 
-| Path | Works? | Why |
-|------|--------|-----|
-| VM → internet | Yes | SNAT through OVN router → macvlan → WAN NIC → WAN |
-| LAN device → VM public IP | Yes | LAN → WAN NIC → macvlan → br-wan → OVN |
-| Host → VM public IP | No | macvlan isolation (kernel blocks parent↔child) |
-| Host → VM private IP | Yes | Overlay via br-int (unrelated to br-wan) |
+| Path                      | Works? | Why                                               |
+| ------------------------- | ------ | ------------------------------------------------- |
+| VM → internet             | Yes    | SNAT through OVN router → macvlan → WAN NIC → WAN |
+| LAN device → VM public IP | Yes    | LAN → WAN NIC → macvlan → br-wan → OVN            |
+| Host → VM public IP       | No     | macvlan isolation (kernel blocks parent↔child)    |
+| Host → VM private IP      | Yes    | Overlay via br-int (unrelated to br-wan)          |
 
 ## Per-Node Configuration
 
@@ -449,24 +396,9 @@ ip link show spx-ext-eth0
 
 ## Network Flow Diagram
 
-```
-┌───────────────────────────────────────────────────┐
-│ Bare-Metal Host                                   │
-│                                                   │
-│  ┌──────────┐         ┌──────────┐                │
-│  │ br-int   │         │br-wan.   │                │
-│  │ (overlay)│         │  (WAN)   │                │
-│  │          │         │          │                │
-│  │ tap-eni1 │  OVN    │spx-ext-  │                │
-│  │ tap-eni2 ├──NAT───→│{nic}     │                │
-│  │ tap-eni3 │pipeline │(macvlan) │                │
-│  └──────────┘         └────┬─────┘                │
-│                            │ macvlan bridge mode  │
-│                        WAN NIC (host IP unchanged)│
-└───────────────────────┼───────────────────────────┘
-                        │
-                   Physical WAN
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-host-flow.svg" alt="Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline" width="900">
+</p>
 
 ## Configuration Reference
 
@@ -475,19 +407,9 @@ levels: cluster-wide mode, IP pool definitions, and per-node NIC settings.
 
 ## Configuration Levels
 
-```
-spinifex.toml
-├── [network]                        # Cluster-wide: which mode?
-│   └── external_mode = "pool"
-│
-├── [[network.external_pools]]       # IP ranges: what IPs to use?
-│   ├── name, range_start, range_end
-│   ├── gateway, prefix_len
-│   └── region, az (optional)
-│
-└── [nodes.NAME.vpcd]                # Per-node: which NIC?
-    └── external_interface
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-config-layers.svg" alt="spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC" width="900">
+</p>
 
 ## Cluster-Wide: external_mode
 
@@ -496,11 +418,11 @@ spinifex.toml
 external_mode = "pool"    # "pool", "nat", or "" (disabled)
 ```
 
-| Value | Behavior |
-|-------|----------|
-| `"pool"` | Full public networking — public subnets, auto-assign, Elastic IPs |
-| `"nat"` | Outbound-only SNAT — all VMs share one external IP |
-| `""` / omitted | Overlay-only — no external connectivity |
+| Value          | Behavior                                                          |
+| -------------- | ----------------------------------------------------------------- |
+| `"pool"`       | Full public networking — public subnets, auto-assign, Elastic IPs |
+| `"nat"`        | Outbound-only SNAT — all VMs share one external IP                |
+| `""` / omitted | Overlay-only — no external connectivity                           |
 
 ## IP Pools: network.external_pools
 
@@ -523,18 +445,18 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 
 ### Field Details
 
-| Field | Required | Description |
-|-------|----------|-------------|
-| `name` | Yes | Unique pool name. Used as NATS KV key and in `AllocateAddress`. |
-| `source` | No | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch. |
-| `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides). |
-| `range_end` | Static only | Last IP in the range. |
-| `gateway` | Yes | Physical router/switch — the WAN default gateway. OVN sets `0.0.0.0/0 → gateway`. |
-| `gateway_ip` | NAT mode | Static IP for OVN router SNAT. In pool mode, defaults to `range_start` (static) or first DHCP lease (dhcp). In NAT mode, this is the single external IP all VMs share. |
-| `prefix_len` | Yes | Subnet mask for the external network (e.g., 24 = /24). |
-| `region` | No | Scopes pool to a region. Instances in this region prefer this pool. |
-| `az` | No | Scopes pool to an AZ. More specific than region. |
-| `dns_servers` | No | DNS servers propagated to VMs via OVN DHCP. |
+| Field         | Required    | Description                                                                                                                                                            |
+| ------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `name`        | Yes         | Unique pool name. Used as NATS KV key and in `AllocateAddress`.                                                                                                        |
+| `source`      | No          | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch.                                  |
+| `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides).                                                                      |
+| `range_end`   | Static only | Last IP in the range.                                                                                                                                                  |
+| `gateway`     | Yes         | Physical router/switch — the WAN default gateway. OVN sets `0.0.0.0/0 → gateway`.                                                                                      |
+| `gateway_ip`  | NAT mode    | Static IP for OVN router SNAT. In pool mode, defaults to `range_start` (static) or first DHCP lease (dhcp). In NAT mode, this is the single external IP all VMs share. |
+| `prefix_len`  | Yes         | Subnet mask for the external network (e.g., 24 = /24).                                                                                                                 |
+| `region`      | No          | Scopes pool to a region. Instances in this region prefer this pool.                                                                                                    |
+| `az`          | No          | Scopes pool to an AZ. More specific than region.                                                                                                                       |
+| `dns_servers` | No          | DNS servers propagated to VMs via OVN DHCP.                                                                                                                            |
 
 ### Why range_start/range_end Instead of CIDR?
 
@@ -561,8 +483,8 @@ ovn_sb_addr        = "tcp:10.1.3.181:6642"   # OVN Southbound DB
 external_interface = "eth1"                   # WAN NIC name
 ```
 
-| Field | Description |
-|-------|-------------|
+| Field                | Description                                                                                                                                                      |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `external_interface` | Physical NIC for WAN traffic. A macvlan sub-interface is created on this NIC for br-wan. Different servers may have different names (eth1, eno2, enp3s0, bond0). |
 
 ## Pool Selection Logic
@@ -588,8 +510,12 @@ name. Allocation uses CAS (Compare-And-Set) for lock-free concurrent access:
   "range_start": "192.168.1.150",
   "range_end": "192.168.1.250",
   "allocated": {
-    "192.168.1.150": {"type": "gateway"},
-    "192.168.1.151": {"type": "auto_assign", "eni_id": "eni-abc", "instance_id": "i-123"}
+    "192.168.1.150": { "type": "gateway" },
+    "192.168.1.151": {
+      "type": "auto_assign",
+      "eni_id": "eni-abc",
+      "instance_id": "i-123"
+    }
   }
 }
 ```
@@ -774,20 +700,21 @@ on the port group control traffic:
 ## Default Security Group
 
 Every VPC gets a default security group that:
+
 - Allows all inbound from instances in the same security group
 - Allows all outbound
 - Denies all other inbound
 
 ## AWS Rule → OVN ACL Translation
 
-| AWS Security Group Rule | OVN ACL Match |
-|------------------------|---------------|
-| Ingress TCP/22 from 0.0.0.0/0 | `outport == @sg && ip4 && tcp.dst == 22` |
+| AWS Security Group Rule         | OVN ACL Match                                                      |
+| ------------------------------- | ------------------------------------------------------------------ |
+| Ingress TCP/22 from 0.0.0.0/0   | `outport == @sg && ip4 && tcp.dst == 22`                           |
 | Ingress TCP/443 from 10.0.0.0/8 | `outport == @sg && ip4 && tcp.dst == 443 && ip4.src == 10.0.0.0/8` |
-| Ingress ALL from sg-other | `outport == @sg && ip4 && ip4.src == $sg_other_ip4` |
-| Ingress ICMP from anywhere | `outport == @sg && ip4 && icmp4` |
-| Egress ALL to 0.0.0.0/0 | `inport == @sg && ip4` |
-| Default deny inbound | `outport == @sg && ip4` (priority 900, action=drop) |
+| Ingress ALL from sg-other       | `outport == @sg && ip4 && ip4.src == $sg_other_ip4`                |
+| Ingress ICMP from anywhere      | `outport == @sg && ip4 && icmp4`                                   |
+| Egress ALL to 0.0.0.0/0         | `inport == @sg && ip4`                                             |
+| Default deny inbound            | `outport == @sg && ip4` (priority 900, action=drop)                |
 
 ## Example: Allow SSH + HTTP
 
@@ -1124,18 +1051,21 @@ ip route show
 ### Instance Has No Public IP
 
 1. Check subnet has `MapPublicIpOnLaunch`:
+
    ```bash
    aws ec2 describe-subnets --subnet-ids $SUBNET \
      --query 'Subnets[0].MapPublicIpOnLaunch'
    ```
 
 2. Check IGW is attached:
+
    ```bash
    aws ec2 describe-internet-gateways \
      --filters Name=attachment.vpc-id,Values=$VPC
    ```
 
 3. Check external IP pool:
+
    ```bash
    nats kv get spinifex-external-ipam wan
    ```
@@ -1272,6 +1202,7 @@ sudo systemctl start ovn-central ovn-controller
 `ovn-sbctl show`. Gateway traffic (SNAT/DNAT) doesn't work.
 
 **Cause:** Three names must align:
+
 1. OVS `external_ids:system-id` → what ovn-controller registers as
 2. OVN SB `Chassis.name` → what actually exists in the SB DB
 3. vpcd's chassis name → `chassis-{nodeName}` from config

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.1
+go 1.26.2
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -8,10 +8,10 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.1.3
-	github.com/mulgadc/viperblock v1.1.5
+	github.com/mulgadc/predastore v1.1.5
+	github.com/mulgadc/viperblock v1.1.6
 	github.com/nats-io/nats-server/v2 v2.12.6
-	github.com/nats-io/nats.go v1.49.0
+	github.com/nats-io/nats.go v1.50.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
 	github.com/pelletier/go-toml/v2 v2.3.0
 	github.com/pterm/pterm v0.12.83

go.sum

@@ -29,34 +29,34 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
 github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk=
 github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ=
@@ -65,8 +65,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCd
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -255,10 +255,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.1.3 h1:WddAOXmGJFQ7tRG+5B3yz/tCX46yW1CmbVL07y+tyBU=
-github.com/mulgadc/predastore v1.1.3/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
-github.com/mulgadc/viperblock v1.1.5 h1:ojmoJQrY7VkDGzdQj07w2F7ICRJX9RlGAKyn87Wa55o=
-github.com/mulgadc/viperblock v1.1.5/go.mod h1:utoIQLwCmPujOBaT0mL5NhW3m2fwjQPQiQRJH6+Wtk8=
+github.com/mulgadc/predastore v1.1.5 h1:BjTEP2xbzIh0FswuCvnVcq9uy6l235lcIOwkGG/Z+f4=
+github.com/mulgadc/predastore v1.1.5/go.mod h1:la4jzK8lSLfUyUR5wNdVrkXXq8JQMGiKFvp/eXT0wjM=
+github.com/mulgadc/viperblock v1.1.6 h1:SzvpZIFWZVpWWdUSPD9k7fgmVAOsyXj9Ra686MyFsRc=
+github.com/mulgadc/viperblock v1.1.6/go.mod h1:KoJH5WFaZdIEYQ/uxugFKhQRLcMa9gmNOtyZjRHf/wI=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -267,8 +267,8 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
 github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
 github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8=
 github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo=
-github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
-github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/nats.go v1.50.0 h1:5zAeQrTvyrKrWLJ0fu02W3br8ym57qf7csDzgLOpcds=
+github.com/nats-io/nats.go v1.50.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

scripts/build-system-image.sh

@@ -0,0 +1,336 @@
+#!/bin/bash
+set -euo pipefail
+
+# build-system-image.sh — Build a minimal Alpine AMI from a manifest
+#
+# Creates a pre-baked Alpine Linux image with custom packages, binaries,
+# and setup scripts installed, ready for import as a Spinifex AMI.
+#
+# Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
+# Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
+#   --import  Also import the image as an AMI via spx admin images import
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+usage() {
+    echo "Usage: $0 <manifest.conf> [--import]"
+    echo ""
+    echo "  manifest.conf  Path to image manifest (see scripts/images/ for examples)"
+    echo "  --import       Import the built image as an AMI"
+    exit 1
+}
+
+if [[ $# -lt 1 || "$1" == "-h" || "$1" == "--help" ]]; then
+    usage
+fi
+
+MANIFEST="$1"
+shift
+
+if [[ ! -f "$MANIFEST" ]]; then
+    echo "ERROR: Manifest not found: $MANIFEST"
+    exit 1
+fi
+
+DO_IMPORT=false
+QUIET=false
+for arg in "$@"; do
+    case "$arg" in
+        --import) DO_IMPORT=true ;;
+        --quiet)  QUIET=true ;;
+    esac
+done
+
+# Source the manifest
+# shellcheck source=/dev/null
+source "$MANIFEST"
+
+# Validate required manifest fields
+for field in IMAGE_NAME ALPINE_VERSION IMAGE_SIZE; do
+    if [[ -z "${!field:-}" ]]; then
+        echo "ERROR: Manifest missing required field: $field"
+        exit 1
+    fi
+done
+
+# Derived paths
+BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
+ALPINE_IMAGE="generic_alpine-${ALPINE_VERSION}-x86_64-bios-cloudinit-r0.qcow2"
+ALPINE_URL="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION%.*}/releases/cloud/${ALPINE_IMAGE}"
+OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
+OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
+NBD_DEV="/dev/nbd0"
+MOUNT_DIR="${BUILD_DIR}/mnt"
+
+cleanup() {
+    echo "Cleaning up..."
+    sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+    sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+    exec 9>&- 2>/dev/null || true  # release nbd lock if held
+    echo "Done."
+}
+trap cleanup EXIT
+
+# In quiet mode, redirect build output to /dev/null (import output still shown)
+if [[ "$QUIET" == true ]]; then
+    exec 3>&1         # save original stdout
+    exec 1>/dev/null  # suppress build output
+fi
+
+echo "=== System Image Builder ==="
+echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
+echo "Alpine:  ${ALPINE_VERSION}"
+echo "Size:    ${IMAGE_SIZE}"
+echo "Build:   ${BUILD_DIR}"
+echo ""
+
+# Step 0: Check prerequisites
+if ! command -v qemu-nbd &>/dev/null; then
+    echo "ERROR: qemu-nbd not found. Install qemu-utils."
+    exit 1
+fi
+
+if ! command -v qemu-img &>/dev/null; then
+    echo "ERROR: qemu-img not found. Install qemu-utils."
+    exit 1
+fi
+
+# Build binaries if BUILD_COMMANDS is set
+if [[ -n "${BUILD_COMMANDS:-}" ]]; then
+    echo "Building binaries: ${BUILD_COMMANDS}"
+    if ! (cd "$PROJECT_DIR" && eval "$BUILD_COMMANDS"); then
+        echo "ERROR: BUILD_COMMANDS failed: ${BUILD_COMMANDS}"
+        exit 1
+    fi
+fi
+
+# Verify binaries exist and are statically linked
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        src_path="${PROJECT_DIR}/${src}"
+        if [[ ! -f "$src_path" ]]; then
+            echo "ERROR: Binary not found: $src_path"
+            exit 1
+        fi
+        if ! file "$src_path" | grep -q "statically linked"; then
+            echo "ERROR: $src_path is not statically linked (Alpine uses musl — dynamic glibc binaries will fail)"
+            echo "  Rebuild with: CGO_ENABLED=0 go build ..."
+            exit 1
+        fi
+    done
+fi
+
+# Serialize the entire image build with flock — concurrent builds on the same
+# host (e.g. CI single-node + multi-node jobs) share /dev/nbd0 and BUILD_DIR.
+NBD_LOCK="/tmp/build-system-image.lock"
+echo "Acquiring build lock..."
+exec 9>"$NBD_LOCK"
+flock 9
+echo "Lock acquired"
+
+# If the raw image was built recently (< 10 min), skip the entire build.
+# This avoids duplicate work when concurrent CI jobs build the same image.
+if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") )) -lt 600 ]]; then
+    echo "=== Skipping build — $OUTPUT_RAW is fresh (< 10 min old) ==="
+
+    # Restore stdout if suppressed, then jump to import
+    if [[ "$QUIET" == true ]]; then
+        exec 1>&3 3>&-
+    fi
+
+    echo ""
+    echo "=== Build complete (cached) ==="
+    echo "  raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
+
+    if [[ "$DO_IMPORT" == true ]]; then
+        echo "Importing as AMI..."
+        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+        if [[ -n "${SYSTEM_TAG:-}" ]]; then
+            IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
+        fi
+        (cd "$PROJECT_DIR" && ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+    fi
+    exit 0
+fi
+
+mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
+
+# Step 1: Download Alpine cloud image
+if [[ -f "${BUILD_DIR}/${ALPINE_IMAGE}" ]]; then
+    echo "Alpine image already downloaded."
+else
+    echo "Downloading Alpine ${ALPINE_VERSION} cloud image..."
+    if ! curl --fail -L -o "${BUILD_DIR}/${ALPINE_IMAGE}" "$ALPINE_URL"; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Failed to download Alpine image from $ALPINE_URL"
+        exit 1
+    fi
+    # Verify the download is a valid qcow2 image
+    if ! qemu-img info "${BUILD_DIR}/${ALPINE_IMAGE}" &>/dev/null; then
+        rm -f "${BUILD_DIR}/${ALPINE_IMAGE}"
+        echo "ERROR: Downloaded file is not a valid qcow2 image"
+        exit 1
+    fi
+fi
+
+# Step 2: Copy image for customization
+rm -f "$OUTPUT_IMAGE"
+echo "Copying image for customization..."
+cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
+
+# Resize the image (Alpine cloud images are ~200MB, need room for packages)
+qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
+
+# Step 3: Connect via qemu-nbd
+echo "Connecting image via qemu-nbd..."
+sudo modprobe nbd max_part=4 2>/dev/null || true
+if [[ ! -e "${NBD_DEV}" ]]; then
+    echo "ERROR: ${NBD_DEV} does not exist. Is the nbd kernel module loaded? Try: sudo modprobe nbd"
+    exit 1
+fi
+sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
+
+# Wait for the nbd device to be ready (kernel needs time to scan the block device)
+for i in $(seq 1 10); do
+    if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
+        break
+    fi
+    sleep 1
+done
+sleep 1
+
+# Alpine cloud images have ext4 directly on the block device (no partition table).
+# Resize the filesystem to fill the resized image.
+echo "Checking filesystem..."
+sudo e2fsck -f -y "${NBD_DEV}" || {
+    ec=$?
+    if [[ $ec -gt 1 ]]; then
+        echo "ERROR: e2fsck failed with exit code $ec on ${NBD_DEV}"
+        exit 1
+    fi
+}
+
+echo "Resizing filesystem..."
+if ! sudo resize2fs "${NBD_DEV}"; then
+    echo "ERROR: resize2fs failed on ${NBD_DEV}"
+    exit 1
+fi
+
+# Step 4: Mount and customize
+echo "Mounting root filesystem..."
+sudo mount "${NBD_DEV}" "$MOUNT_DIR"
+
+# Set up resolv.conf for DNS inside chroot
+sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
+
+# Install packages
+if [[ -n "${APK_PACKAGES:-}" ]]; then
+    echo "Installing packages: ${APK_PACKAGES}..."
+    sudo chroot "$MOUNT_DIR" /bin/sh -c "
+set -e
+# Enable community repo
+sed -i 's|^#\(.*community\)|\1|' /etc/apk/repositories 2>/dev/null || true
+
+# Ensure community repo is present
+if ! grep -q community /etc/apk/repositories; then
+    MIRROR=\$(grep main /etc/apk/repositories | head -1 | sed 's|/main|/community|')
+    echo \"\$MIRROR\" >> /etc/apk/repositories
+fi
+
+apk update
+apk add ${APK_PACKAGES}
+"
+fi
+
+# Enable OpenRC services
+if [[ -n "${ENABLE_SERVICES:-}" ]]; then
+    echo "Enabling services: ${ENABLE_SERVICES}..."
+    IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
+    for svc in "${SERVICES[@]}"; do
+        if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
+            echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
+            exit 1
+        fi
+    done
+fi
+
+# Step 5: Copy binaries into the image (before setup script, which may reference them)
+if [[ -n "${INSTALL_BINARIES:-}" ]]; then
+    echo "Installing binaries..."
+    IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
+    for pair in "${BINARY_PAIRS[@]}"; do
+        src="${pair%%:*}"
+        dst="${pair#*:}"
+        src_path="${PROJECT_DIR}/${src}"
+        echo "  ${src} -> ${dst}"
+        sudo cp "$src_path" "${MOUNT_DIR}${dst}"
+        sudo chmod 755 "${MOUNT_DIR}${dst}"
+    done
+fi
+
+# Run custom setup script inside chroot (after binaries are installed)
+if [[ -n "${SETUP_SCRIPT:-}" ]]; then
+    setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
+    if [[ ! -f "$setup_path" ]]; then
+        echo "ERROR: Setup script not found: $setup_path"
+        exit 1
+    fi
+    echo "Running setup script: ${SETUP_SCRIPT}..."
+    sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
+    sudo chroot "$MOUNT_DIR" /tmp/setup.sh
+    sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
+fi
+
+# Step 6: Clean up and unmount
+echo "Cleaning up image..."
+sudo chroot "$MOUNT_DIR" /bin/sh -c '
+apk cache clean 2>/dev/null || true
+rm -rf /var/cache/apk/* /tmp/*
+'
+
+# Restore original resolv.conf (cloud-init will set it on boot)
+sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
+
+echo "Unmounting..."
+sudo umount "$MOUNT_DIR"
+sudo qemu-nbd --disconnect "${NBD_DEV}"
+
+# Step 7: Convert to raw for import
+echo "Converting to raw format..."
+qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
+
+# Restore stdout if suppressed
+if [[ "$QUIET" == true ]]; then
+    exec 1>&3 3>&-
+fi
+
+echo ""
+echo "=== Build complete ==="
+echo "  Image: ${IMAGE_NAME}"
+echo "  qcow2: $OUTPUT_IMAGE ($(du -h "$OUTPUT_IMAGE" | cut -f1))"
+echo "  raw:   $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
+echo ""
+
+if [[ "$DO_IMPORT" == true ]]; then
+    echo "Importing as AMI..."
+    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+    if [[ -n "${SYSTEM_TAG:-}" ]]; then
+        IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
+    fi
+    (cd "$PROJECT_DIR" && sudo -u spinifex-storage ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+else
+    echo "To import as AMI, run:"
+    echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
+    echo "    --file $OUTPUT_RAW \\"
+    if [[ -n "${SYSTEM_TAG:-}" ]]; then
+        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \\"
+        echo "    --tag ${SYSTEM_TAG}"
+    else
+        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64"
+    fi
+fi

scripts/check-coverage.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]
 
 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=60.0
+MIN=65.0
 QUIET="${2:-}"
 
 if [[ ! -s "$PROFILE" ]]; then

scripts/clone-deps.sh

@@ -3,7 +3,6 @@
 # Spinifex Development Dependencies Setup Script
 # This script clones Viperblock and Predastore repositories for cross-repo development
 
-# TODO: Improve, remove CLAUDE readme reference, simplify next-steps
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

scripts/coverage.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Run Go tests with coverage and report total code coverage.
-
-COVERPROFILE=$(mktemp /tmp/spinifex-coverage-XXXXXX.out)
-trap 'rm -f "$COVERPROFILE"' EXIT
-
-echo "Running tests with coverage..."
-echo ""
-LOG_IGNORE=1 go test -timeout 120s -coverprofile="$COVERPROFILE" -covermode=atomic ./spinifex/... 2>&1
-
-if [[ ! -s "$COVERPROFILE" ]]; then
-    echo "No coverage data collected."
-    exit 1
-fi
-
-echo ""
-echo "=== Total ==="
-go tool cover -func="$COVERPROFILE" | awk '/^total:/ { print $NF }'

scripts/create-multi-node.sh

@@ -92,7 +92,7 @@ sleep 5
 
 echo "Daemon health:"
 for IP in 127.0.0.1 127.0.0.2 127.0.0.3; do
-  echo "  $IP: $(curl -s http://$IP:4432/health | head -c 100)"
+  echo "  $IP: $(curl -sk https://$IP:4432/health | head -c 100)"
 done
 
 echo ""

scripts/dev-install.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# dev-install.sh — Full local development setup via production installer.
+# Builds from source, assembles a tarball, runs setup.sh for scaffolding
+# (user creation, directories, systemd units), initializes the cluster,
+# and starts services via systemd.
+#
+# Usage: ./scripts/dev-install.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
+
+echo "=== Building ==="
+cd "$PROJECT_ROOT" && make build
+cd "$MULGA_ROOT/viperblock" && make go_build_nbd
+
+echo "=== Assembling tarball for setup.sh ==="
+STAGING=$(mktemp -d)
+trap 'rm -rf "$STAGING"' EXIT
+cp "$PROJECT_ROOT/bin/spx" "$STAGING/spx"
+cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
+cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
+mkdir -p "$STAGING/systemd"
+cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
+mkdir -p "$STAGING/scripts"
+cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
+cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
+tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
+
+echo "=== Cleaning stale state from previous installs ==="
+# Stop any running services before modifying files
+sudo systemctl stop spinifex.target 2>/dev/null || true
+sudo systemctl reset-failed 2>/dev/null || true
+
+# Remove stale files owned by the dev user in production paths.
+# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave
+# files owned by tf-user that service users (spinifex-nats, etc.) can't read
+# under systemd's ProtectSystem=strict sandboxing.
+for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do
+    if [ -d "$dir" ]; then
+        # Remove PID files, stale logs, and the legacy ~/spinifex/config symlink
+        sudo find "$dir" -name '*.pid' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log.*' -delete 2>/dev/null || true
+    fi
+done
+# Remove legacy data dir contents that conflict with production layout
+if [ -d /var/lib/spinifex/config ]; then
+    sudo rm -rf /var/lib/spinifex/config
+fi
+
+echo "=== Running setup.sh (creates users, dirs, systemd units) ==="
+sudo INSTALL_SPINIFEX_TARBALL=/tmp/spinifex-local.tar.gz bash "$PROJECT_ROOT/scripts/setup.sh"
+rm -f /tmp/spinifex-local.tar.gz
+
+echo "=== Setting up OVN ==="
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+
+echo "=== Initializing ==="
+sudo spx admin init --force --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+
+echo "=== Installing CA certificate ==="
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+
+echo "=== Starting services ==="
+sudo systemctl start spinifex.target
+
+# Check if user needs to re-login for spinifex group membership
+if ! id -Gn 2>/dev/null | grep -qw spinifex; then
+    newgrp spinifex
+    # echo ""
+    # echo "NOTE: You were added to the 'spinifex' group but your current"
+    # echo "session doesn't reflect it yet. Log out and back in, or run:"
+    # echo "  newgrp spinifex"
+fi
+
+echo "=== Building and importing LB image ==="
+cd "$PROJECT_ROOT" && make build-lb-agent
+"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+
+echo "=== Done ==="
+echo "Services: sudo systemctl status spinifex.target"
+echo "Logs:     journalctl -u 'spinifex-*' -f"
+echo "Test:     AWS_PROFILE=spinifex aws ec2 describe-instances"
+echo "Iterate:  make deploy (rebuild + restart all)"

scripts/dev-setup.sh

@@ -165,24 +165,16 @@ else
 fi
 
 echo ""
-echo "🎉 Development environment setup complete!"
+echo "Development environment setup complete!"
 echo ""
-echo "📋 Next steps:"
+echo "Recommended: Use scripts/dev-install.sh for a production-parity setup"
+echo "  (systemd, privilege separation, production paths)."
 echo ""
-echo "   1. Setup OVN networking:"
-echo "      ./scripts/setup-ovn.sh --management"
+echo "  scripts/dev-install.sh  # One-time: build, install, init, start via systemd"
+echo "  make deploy             # Daily: rebuild + copy binary + restart all"
+echo "  make reinstall          # Changed a .service file? Re-run full setup"
 echo ""
-echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):"
-echo "      sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
-echo ""
-echo "   3. Start services:"
-echo "      ./scripts/start-dev.sh"
-echo ""
-echo "🔧 Development endpoints:"
-echo "   - Spinifex Gateway:  https://localhost:$SPINIFEX_GATEWAY_PORT"
-echo "   - Predastore S3: https://localhost:$PREDASTORE_PORT"
-echo "   - NATS:          nats://localhost:$NATS_PORT"
-echo ""
-echo "🧪 Test with AWS CLI:"
-echo "   export AWS_PROFILE=spinifex"
-echo "   aws ec2 describe-instances"
+echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):"
+echo "  ./scripts/setup-ovn.sh --management"
+echo "  sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
+echo "  ./scripts/start-dev.sh"

scripts/diff-coverage.sh

@@ -7,13 +7,12 @@ set -euo pipefail
 # Usage: scripts/diff-coverage.sh <coverprofile> [--base <ref>] [--threshold <pct>]
 #
 # Base ref auto-detection:
-#   dev branch    → origin/main
 #   main branch   → HEAD~1
-#   other branch  → origin/dev
+#   other branch  → origin/main
 
 PROFILE=""
 BASE_REF=""
-THRESHOLD=60
+THRESHOLD=65
 QUIET="${QUIET:-}"
 
 while [[ $# -gt 0 ]]; do
@@ -45,8 +44,7 @@ if [[ -z "$BASE_REF" ]]; then
     BRANCH="${GITHUB_REF_NAME:-$(git rev-parse --abbrev-ref HEAD)}"
     case "$BRANCH" in
         main) BASE_REF="HEAD~1" ;;
-        dev)  BASE_REF="origin/main" ;;
-        *)    BASE_REF="origin/dev" ;;
+        *)    BASE_REF="origin/main" ;;
     esac
     [[ -z "$QUIET" ]] && echo "Base ref: $BASE_REF (branch: $BRANCH)"
 fi

scripts/iac/aws/basic/main.tf

@@ -0,0 +1,307 @@
+# Spinifex IaC Compatibility Test — "basic" suite
+#
+# Exercises every AWS resource Spinifex currently supports via tofu apply,
+# WITHOUT route tables, NACLs, or other unimplemented networking.
+#
+# Usage:
+#   cd spinifex/scripts/iac/aws/basic
+#   tofu init
+#   tofu plan
+#   tofu apply        # full round-trip against a running Spinifex cluster
+#   tofu destroy      # clean teardown
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "mulgaos_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.mulgaos_endpoint
+    s3  = var.mulgaos_endpoint
+    iam = var.mulgaos_endpoint
+    sts = var.mulgaos_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"]
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 1. VPC + Subnet (no route table needed)
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "test" {
+  cidr_block           = "10.99.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "spx-basic-test-vpc"
+  }
+}
+
+resource "aws_subnet" "test" {
+  vpc_id                  = aws_vpc.test.id
+  cidr_block              = "10.99.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "spx-basic-test-subnet"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 2. Internet Gateway (attach to VPC, but no route table referencing it)
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "test" {
+  vpc_id = aws_vpc.test.id
+
+  tags = {
+    Name = "spx-basic-test-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 3. Security Group — inbound SSH + HTTP, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "test" {
+  name        = "spx-basic-test-sg"
+  description = "Basic test: SSH + HTTP inbound, all outbound"
+  vpc_id      = aws_vpc.test.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "spx-basic-test-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 4. Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "test" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "test" {
+  key_name   = "spx-basic-test-key"
+  public_key = tls_private_key.test.public_key_openssh
+}
+
+resource "local_file" "test_pem" {
+  filename        = "${path.module}/spx-basic-test.pem"
+  content         = tls_private_key.test.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# 5. Placement Group
+# ---------------------------------------------------------------------------
+
+resource "aws_placement_group" "test" {
+  name     = "spx-basic-test-spread"
+  strategy = "spread"
+}
+
+# ---------------------------------------------------------------------------
+# 6. EC2 Instance
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "test" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = "t3.small"
+
+  subnet_id              = aws_subnet.test.id
+  vpc_security_group_ids = [aws_security_group.test.id]
+  key_name               = aws_key_pair.test.key_name
+  placement_group        = aws_placement_group.test.name
+
+  associate_public_ip_address = true
+
+  tags = {
+    Name        = "spx-basic-test-vm"
+    Environment = "test"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 7. EBS Volume + Attachment
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_volume" "data" {
+  availability_zone = data.aws_availability_zones.available.names[0]
+  size              = 10
+  type              = "gp3"
+
+  tags = {
+    Name = "spx-basic-test-data-vol"
+  }
+}
+
+
+# ---------------------------------------------------------------------------
+# 8. EBS Snapshot (of the standalone volume)
+# ---------------------------------------------------------------------------
+
+resource "aws_ebs_snapshot" "data" {
+  volume_id = aws_ebs_volume.data.id
+
+  tags = {
+    Name = "spx-basic-test-snapshot"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# 9. Elastic IP + Association
+#    Requires external IPAM (pool mode) — uncomment when available.
+# ---------------------------------------------------------------------------
+
+# resource "aws_eip" "test" {
+#   domain = "vpc"
+#
+#   tags = {
+#     Name = "spx-basic-test-eip"
+#   }
+# }
+#
+# resource "aws_eip_association" "test" {
+#   allocation_id = aws_eip.test.id
+#   instance_id   = aws_instance.test.id
+# }
+
+# ---------------------------------------------------------------------------
+# 10. Standalone Network Interface (not attached — just tests CRUD)
+# ---------------------------------------------------------------------------
+
+resource "aws_network_interface" "extra" {
+  subnet_id       = aws_subnet.test.id
+  security_groups = [aws_security_group.test.id]
+
+  tags = {
+    Name = "spx-basic-test-eni"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "vpc_id" {
+  value = aws_vpc.test.id
+}
+
+output "instance_id" {
+  value = aws_instance.test.id
+}
+
+output "instance_public_ip" {
+  value = aws_instance.test.public_ip
+}
+
+# output "eip_public_ip" {
+#   value = aws_eip.test.public_ip
+# }
+
+output "volume_id" {
+  value = aws_ebs_volume.data.id
+}
+
+output "snapshot_id" {
+  value = aws_ebs_snapshot.data.id
+}
+
+output "ssh_command" {
+  value = "ssh -i spx-basic-test.pem admin@${aws_instance.test.public_ip}"
+}

scripts/iac/aws/main.tf

@@ -189,12 +189,6 @@ resource "aws_security_group" "web_ssh" {
   }
 }
 
-# 4: Ensure 3 servers on different underlying hardware (spread placement group)
-resource "aws_placement_group" "spread" {
-  name     = "spinifex-test-spread"
-  strategy = "spread"
-}
-
 resource "aws_instance" "vm" {
   count         = 3
   ami           = data.aws_ami.debian12.id
@@ -205,7 +199,6 @@ resource "aws_instance" "vm" {
   key_name               = aws_key_pair.hive_test.key_name
 
   associate_public_ip_address = true
-  placement_group             = aws_placement_group.spread.name
 
   tags = {
     Name = "spinifex-test-vm-${count.index + 1}"

scripts/iac/configure-cluster.sh

@@ -110,7 +110,7 @@ INIT_PID=$!
 # Wait for formation server to be ready
 echo "    Waiting for formation server on $NODE1_IP..."
 attempts=0
-while ! curl -s --connect-timeout 3 "http://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
+while ! curl -sk --connect-timeout 3 "https://$NODE1_IP:4432/formation/health" >/dev/null 2>&1; do
     attempts=$((attempts + 1))
     if [ $attempts -ge 60 ]; then
         echo "Error: Formation server did not start on $NODE1_IP after 5 minutes"
@@ -176,7 +176,7 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do
     printf "    node%d (%s): " "$((i + 1))" "$ip"
     attempts=0
     while true; do
-        health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null || echo "")
+        health=$(curl -sk --connect-timeout 3 "https://$ip:4432/health" 2>/dev/null || echo "")
         if [ -n "$health" ]; then
             echo "healthy"
             break

scripts/iac/proxmox-template/README.md

@@ -7,7 +7,7 @@ Builds a Proxmox VM template with all Spinifex dependencies pre-installed. Cloni
 | Component | Details |
 |---|---|
 | System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
-| Go | 1.26.1 (`make install-go`) |
+| Go | 1.26.2 (`make install-go`) |
 | AWS CLI | v2 (`make install-aws`) |
 | Repositories | spinifex, viperblock, predastore (with `go.work` configured) |
 | Module cache | `go mod download` completed for all repos |

scripts/iac/spinifex-test.sh

@@ -185,7 +185,7 @@ cmd_status() {
         printf "  %-20s mgmt=%-16s data=%-16s " "$name" "$ip" "$data_ip"
 
         local health
-        if health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null); then
+        if health=$(curl -sk --connect-timeout 3 "https://$ip:4432/health" 2>/dev/null); then
             echo "health=$health"
         else
             echo "health=unreachable"

scripts/images/lb-setup.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+# lb-setup.sh — LB-specific chroot setup
+#
+# Runs inside the Alpine image chroot after packages are installed.
+# Sets up HAProxy placeholder config and lb-agent OpenRC init script.
+# The lb-agent polls the AWS gateway for config updates and reports health
+# via SigV4-signed HTTP requests. Credentials come from cloud-init env vars.
+
+set -e
+
+# Create haproxy config directory and placeholder config.
+mkdir -p /etc/haproxy
+cat > /etc/haproxy/haproxy.cfg <<'EOF'
+# Placeholder config — replaced by lb-agent on first config fetch
+global
+    daemon
+    maxconn 256
+
+defaults
+    mode http
+    timeout connect 5s
+    timeout client 30s
+    timeout server 30s
+EOF
+
+# Create lb-agent OpenRC init script
+mkdir -p /etc/init.d
+cat > /etc/init.d/lb-agent <<'INITSCRIPT'
+#!/sbin/openrc-run
+
+description="LB Gateway Config Agent"
+command="/usr/local/bin/lb-agent"
+command_args="--lb-id=${LB_LB_ID:-unknown} --gateway=${LB_GATEWAY_URL} --access-key=${LB_ACCESS_KEY} --secret-key=${LB_SECRET_KEY}"
+command_background=true
+pidfile="/run/lb-agent.pid"
+output_log="/var/log/lb-agent.log"
+error_log="/var/log/lb-agent.log"
+
+depend() {
+    need net
+    after firewall
+}
+INITSCRIPT
+chmod 755 /etc/init.d/lb-agent
+
+# Move networking from boot to default runlevel so cloud-init can write
+# the network config before the networking service starts. Without this,
+# Alpine's networking service races cloud-init and starts with no config.
+rc-update del networking boot 2>/dev/null || true
+rc-update add networking default
+
+# Do NOT enable lb-agent at boot — cloud-init must write
+# /etc/conf.d/lb-agent (with LB_LB_ID, LB_GATEWAY_URL, LB_ACCESS_KEY,
+# LB_SECRET_KEY) before the service starts.
+# Cloud-init runcmd starts the service after write_files.

scripts/images/lb.conf

@@ -0,0 +1,26 @@
+# LB system image manifest
+# Used by build-system-image.sh to create an Alpine AMI with HAProxy + lb-agent
+
+IMAGE_NAME="lb"
+IMAGE_DESCRIPTION="LB VM with HAProxy and lb-agent"
+ALPINE_VERSION="3.21.6"
+IMAGE_SIZE="256M"
+
+# Tag applied to the imported AMI so the daemon can discover this as the
+# system image for ELBv2 load balancers. Must match tags.ManagedByELBv2.
+SYSTEM_TAG="spinifex:managed-by=elbv2"
+
+# Packages to install via apk
+APK_PACKAGES="haproxy curl ca-certificates"
+
+# Binaries to copy into the image (source:destination)
+INSTALL_BINARIES="bin/lb-agent:/usr/local/bin/lb-agent"
+
+# Build command for binaries (run before image build)
+BUILD_COMMANDS="make build-lb-agent"
+
+# OpenRC services to enable at boot
+ENABLE_SERVICES="haproxy"
+
+# Custom setup script (runs inside chroot after packages are installed)
+SETUP_SCRIPT="scripts/images/lb-setup.sh"

scripts/reset-dev-env.sh

@@ -160,15 +160,12 @@ elif [ -n "$WAN_IFACE" ]; then
     fi
 fi
 
-# Chassis ID must match what vpcd derives from the node name in spinifex.toml.
-# vpcd generates "chassis-{nodeName}" (e.g., chassis-node1) and uses it to schedule
-# gateway chassis in the NB DB. ovn-controller registers in the SB DB using the OVS
-# system-id. If they don't match, gateway traffic has no host to schedule to.
-NODE_NAME="node1"
-CHASSIS_ID="chassis-${NODE_NAME}"
+# Chassis ID — let setup-ovn.sh auto-detect from hostname. This ensures the
+# system-id matches what ovn-controller registers in the SBDB, which is what
+# vpcd discovers at startup for gateway scheduling.
 
-echo "Re-initializing OVN (chassis-id: $CHASSIS_ID)"
-./scripts/setup-ovn.sh --management $SETUP_OVN_FLAGS --chassis-id="$CHASSIS_ID"
+echo "Re-initializing OVN"
+./scripts/setup-ovn.sh --management $SETUP_OVN_FLAGS
 
 echo "Initializing platform"
 ADMIN_INIT_ARGS="--region $REGION --az ${REGION}a --node node1 --nodes 1"
@@ -198,7 +195,6 @@ if [ -n "$WAN_IFACE" ] && [ -n "$WAN_GW" ]; then
         ADMIN_INIT_ARGS="$ADMIN_INIT_ARGS --external-mode=pool --external-pool=${POOL_START}-${POOL_END} --external-gateway=${WAN_GW} --external-prefix-len=${WAN_PREFIX}"
     fi
 fi
-./bin/spx admin init $ADMIN_INIT_ARGS
 
 # Re-initialize with --force to regenerate credentials, certs, config, and
 # update ~/.aws/credentials. Must carry the same args so external networking
@@ -212,8 +208,16 @@ if [ ! -f ~/.ssh/spinifex-key.pub ]; then
     ssh-keygen -t ed25519 -f ~/.ssh/spinifex-key -N ""
 fi
 
+# Trust the Spinifex CA certificate so TLS connections to predastore/awsgw work
+if [ -f "$HOME/spinifex/config/ca.pem" ]; then
+    echo "Adding Spinifex CA certificate to system trust store..."
+    sudo cp "$HOME/spinifex/config/ca.pem" /usr/local/share/ca-certificates/spinifex-ca.crt
+    sudo update-ca-certificates
+fi
+
 # Enable pprof for development
-PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
+#PPROF_ENABLED=1 PPROF_OUTPUT=/tmp/spinifex-vm.prof ./scripts/start-dev.sh --build
+./scripts/start-dev.sh --build
 
 export AWS_PROFILE=spinifex
 
@@ -252,4 +256,46 @@ fi
 
 aws ec2 describe-images
 
-echo "Reset complete, fresh AMI imported, proceed to creating instances"
+# --- Launch a smoke-test instance ---
+echo "Launching smoke-test instance..."
+
+# Pick instance type based on CPU vendor
+if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+    INSTANCE_TYPE="t3a.small"
+else
+    INSTANCE_TYPE="t3.small"
+fi
+
+# Get the AMI we just imported
+AMI_ID=$(aws ec2 describe-images --query "Images[0].ImageId" --output text)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "❌ No AMI found, skipping instance launch"
+    exit 0
+fi
+
+# Find a public subnet (first available)
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --filters "Name=map-public-ip-on-launch,Values=true" \
+    --query "Subnets[0].SubnetId" --output text 2>/dev/null)
+# Fallback: just grab the first subnet if no public-tagged one exists
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    SUBNET_ID=$(aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text)
+fi
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "❌ No subnet found, skipping instance launch"
+    exit 0
+fi
+
+echo "  AMI: $AMI_ID"
+echo "  Instance type: $INSTANCE_TYPE"
+echo "  Subnet: $SUBNET_ID"
+
+INSTANCE_ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+
+echo "✅ Reset complete — instance $INSTANCE_ID launched ($INSTANCE_TYPE)"

scripts/setup-ovn.sh

@@ -481,7 +481,19 @@ echo "  ovn-encap-type: geneve"
 echo ""
 echo "Step 5: Starting ovn-controller..."
 
-sudo systemctl start ovn-controller
+# Set ovn-controller file log level to WARN so it doesn't spam the log with
+# connection-retry INFO messages ("OVNSB commit failed") when the SB DB
+# isn't running. Uses a systemd ExecStartPost so it persists across restarts.
+OVN_CTRL_OVERRIDE="/etc/systemd/system/ovn-controller.service.d/log-level.conf"
+sudo mkdir -p "$(dirname "$OVN_CTRL_OVERRIDE")"
+sudo tee "$OVN_CTRL_OVERRIDE" >/dev/null <<'OVERRIDE'
+[Service]
+ExecStartPost=/bin/sh -c 'OVS_RUNDIR=/var/run/ovn exec /usr/bin/ovs-appctl -t ovn-controller vlog/set file:warn'
+OVERRIDE
+sudo systemctl daemon-reload
+echo "  ovn-controller log level: file:warn (via systemd drop-in)"
+
+sudo systemctl restart ovn-controller
 echo "  ovn-controller: started"
 
 # --- Step 6: Sysctl tuning ---
@@ -565,64 +577,49 @@ else
     echo "  systemd override: already exists"
 fi
 
-# Create sudoers rule for network commands that always need root
-# (ip tuntap, ip link set — NET_ADMIN operations)
+# Sudoers rules for spinifex-daemon and spinifex-vpcd are managed by setup.sh
+# (install_sudoers). Skip writing here to avoid conflicts.
 SUDOERS_FILE="/etc/sudoers.d/spinifex-network"
-if [ ! -f "$SUDOERS_FILE" ]; then
-    CURRENT_USER=$(whoami)
-    sudo tee "$SUDOERS_FILE" >/dev/null <<EOF
-# Spinifex VPC networking: allow non-root daemon to manage tap devices and OVS
-$CURRENT_USER ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
-EOF
-    sudo chmod 0440 "$SUDOERS_FILE"
-    echo "  sudoers rule: created ($SUDOERS_FILE)"
+if [ -f "$SUDOERS_FILE" ]; then
+    echo "  sudoers rule: already exists ($SUDOERS_FILE, managed by setup.sh)"
 else
-    echo "  sudoers rule: already exists"
+    echo "  sudoers rule: not found — run setup.sh first, or install manually"
 fi
 
 # --- Step 9: Configure OVN log rotation ---
-# OVN has built-in rotation that renames foo.log → foo.log.log. A previous
-# logrotate config used a *.log glob that caught those .log.log files too,
-# creating .log.log.1 files that accumulated to 27GB. Fix: use explicit
-# filenames so logrotate only touches the primary logs, not OVN's backups.
+# The ovn-common package provides /etc/logrotate.d/ovn-common which handles
+# rotation and vlog/reopen. We just add maxsize + rotate to cap disk usage.
 echo ""
 echo "Step 9: Configuring OVN log rotation..."
 
-# Clean up stale .log.log files from the old double-rotation bug
-if ls /var/log/ovn/*.log.log* 1>/dev/null 2>&1; then
-    sudo rm -f /var/log/ovn/*.log.log*
-    echo "  cleaned up stale .log.log files"
+OVN_LOGROTATE="/etc/logrotate.d/ovn-common"
+if [ -f "$OVN_LOGROTATE" ]; then
+    if ! grep -q 'maxsize' "$OVN_LOGROTATE"; then
+        sudo sed -i '/^\/var\/log\/ovn\/\*\.log {/a\    rotate 5\n    maxsize 100M' "$OVN_LOGROTATE"
+        echo "  added maxsize 100M + rotate 5 to $OVN_LOGROTATE"
+    else
+        echo "  $OVN_LOGROTATE already has maxsize configured"
+    fi
+else
+    echo "  WARNING: $OVN_LOGROTATE not found — install ovn-common package"
 fi
 
-LOGROTATE_FILE="/etc/logrotate.d/ovn-spinifex"
-sudo tee "$LOGROTATE_FILE" >/dev/null <<'LOGROTATE'
-/var/log/ovn/ovn-controller.log
-/var/log/ovn/ovn-northd.log
-/var/log/ovn/ovsdb-server-nb.log
-/var/log/ovn/ovsdb-server-sb.log
-{
-    daily
-    rotate 3
-    maxsize 100M
-    compress
-    missingok
-    notifempty
-    copytruncate
-}
-LOGROTATE
-echo "  logrotate config: $LOGROTATE_FILE (daily, 100M max, 3 rotations, explicit filenames)"
+# Remove our old custom config if present (superseded by patching ovn-common)
+if [ -f /etc/logrotate.d/ovn-spinifex ]; then
+    sudo rm -f /etc/logrotate.d/ovn-spinifex
+    echo "  removed obsolete /etc/logrotate.d/ovn-spinifex"
+fi
 
-# --- Step 10: Disable auto-start on boot ---
-# start-dev.sh / stop-dev.sh manage the OVN lifecycle. Without spinifex services
-# running, ovn-controller spins in a tight reconnect loop burning CPU.
+# --- Step 10: Enable auto-start on boot ---
+# OVN services should start with the system in production. ovn-controller
+# retries when the SB DB isn't ready; file log level is set to WARN (Step 5)
+# to prevent log spam during those retries.
 echo ""
-echo "Step 10: Disabling OVN auto-start on boot (start-dev.sh will manage lifecycle)..."
-sudo systemctl disable openvswitch-switch 2>/dev/null || true
-sudo systemctl disable ovn-controller 2>/dev/null || true
-echo "  openvswitch-switch: disabled on boot"
-echo "  ovn-controller: disabled on boot"
+echo "Step 10: Enabling OVN auto-start on boot..."
+sudo systemctl enable openvswitch-switch 2>/dev/null || true
+sudo systemctl enable ovn-controller 2>/dev/null || true
+echo "  openvswitch-switch: enabled on boot"
+echo "  ovn-controller: enabled on boot"
 
 # --- Step 11: Health check ---
 echo ""

scripts/setup.sh

@@ -89,27 +89,66 @@ detect_arch() {
     info "Detected architecture: $MACHINE ($ARCH)"
 }
 
-# --- Detect service user ---
-detect_service_user() {
-    # Use the real user — either the current user, or SUDO_USER if running via sudo
-    SPINIFEX_USER="${SUDO_USER:-$(whoami)}"
-    SPINIFEX_GROUP="$(id -gn "$SPINIFEX_USER")"
+# --- Create per-service system users ---
+create_service_users() {
+    SPINIFEX_GROUP="spinifex"
 
-    if [ "$SPINIFEX_USER" = "root" ]; then
-        fatal "Cannot determine service user. Do not run as root directly — use: curl ... | bash (as a normal user)"
+    # Create shared group
+    if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
+        $SUDO groupadd --system "$SPINIFEX_GROUP"
     fi
 
-    info "Services will run as: $SPINIFEX_USER:$SPINIFEX_GROUP"
+    # Create per-service users with correct home directories
+    declare -A SERVICE_HOMES=(
+        [nats]="/var/lib/spinifex/nats"
+        [gw]="/var/lib/spinifex/awsgw"
+        [daemon]="/var/lib/spinifex/spinifex"
+        [storage]="/var/lib/spinifex/predastore"
+        [viperblock]="/var/lib/spinifex/viperblock"
+        [vpcd]="/var/lib/spinifex"
+        [ui]="/var/lib/spinifex"
+    )
+    for svc in nats gw daemon storage viperblock vpcd ui; do
+        local user="spinifex-${svc}"
+        if ! id "$user" > /dev/null 2>&1; then
+            $SUDO useradd --system --no-create-home \
+                --home-dir "${SERVICE_HOMES[$svc]}" \
+                --gid "$SPINIFEX_GROUP" \
+                --shell /usr/sbin/nologin \
+                "$user"
+        fi
+    done
 
-    # Allow the service user to manage OVN/OVS and tap devices
-    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
-# Spinifex VPC networking: allow service user to manage tap devices, OVS, and OVN
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+    # Add invoking user to spinifex group for admin CLI access
+    ADMIN_USER="${SUDO_USER:-$(whoami)}"
+    if [ "$ADMIN_USER" != "root" ]; then
+        $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
+    fi
+
+    # KVM access for daemon
+    if getent group kvm > /dev/null 2>&1; then
+        $SUDO usermod -aG kvm spinifex-daemon
+    fi
+
+    info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})"
+}
+
+# --- Install scoped sudoers rules ---
+install_sudoers() {
+    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
+# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs
+spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
+spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
+
+# Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
 SUDOERS
     $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
-    info "Sudoers rules installed for $SPINIFEX_USER"
+    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
+    info "Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd"
 }
 
 # --- Install apt dependencies ---
@@ -196,7 +235,8 @@ download_spinifex() {
         fi
         info "Checksum verified"
     else
-        warn "Checksum not available, skipping verification"
+        rm -rf "$SPINIFEX_TMPDIR"
+        fatal "Checksum not available at $CHECKSUM_URL. Cannot verify download integrity."
     fi
 
     # Extract
@@ -233,18 +273,21 @@ install_files() {
         $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
         info "  /usr/local/share/spinifex/setup-ovn.sh"
     fi
+
 }
 
 # --- Create directories ---
 create_directories() {
     info "Creating directories..."
 
+    # Top-level directories (root-owned, group-readable by spinifex)
     $SUDO mkdir -p /etc/spinifex
-    $SUDO chmod 0700 /etc/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex
+    $SUDO chmod 0750 /etc/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex
 
     $SUDO mkdir -p /var/lib/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex
+    $SUDO chmod 0750 /var/lib/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex
 
     # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/
     if [ ! -e /var/lib/spinifex/config ]; then
@@ -257,21 +300,163 @@ create_directories() {
     fi
 
     $SUDO mkdir -p /var/log/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/log/spinifex
+    $SUDO chmod 0775 /var/log/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex
 
     $SUDO mkdir -p /run/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /run/spinifex
+    $SUDO chmod 0775 /run/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex
+
+    $SUDO mkdir -p /run/spinifex/nbd
+    $SUDO chmod 0770 /run/spinifex/nbd
+    $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex/nbd
+
+    # Per-service config directories
+    $SUDO mkdir -p /etc/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
+    $SUDO chmod 0750 /etc/spinifex/nats
+
+    $SUDO mkdir -p /etc/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
+    $SUDO chmod 0750 /etc/spinifex/predastore
+
+    # Per-service data directories
+    $SUDO mkdir -p /var/lib/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
+    $SUDO chmod 0700 /var/lib/spinifex/nats
+
+    $SUDO mkdir -p /var/lib/spinifex/spinifex
+    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
+    $SUDO chmod 0700 /var/lib/spinifex/spinifex
+
+    $SUDO mkdir -p /var/lib/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
+    $SUDO chmod 0700 /var/lib/spinifex/predastore
 
     $SUDO mkdir -p /var/lib/spinifex/viperblock
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chmod 0700 /var/lib/spinifex/viperblock
 
+    $SUDO mkdir -p /var/lib/spinifex/vpcd
+    $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
+    $SUDO chmod 0700 /var/lib/spinifex/vpcd
+
+    $SUDO mkdir -p /var/lib/spinifex/awsgw
+    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /var/lib/spinifex/awsgw
+    $SUDO chmod 0700 /var/lib/spinifex/awsgw
+
+    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/
+    if [ ! -e /var/lib/spinifex/awsgw/config ]; then
+        $SUDO ln -s /etc/spinifex /var/lib/spinifex/awsgw/config
+    fi
+
+    # Top-level config files (root-owned, group-readable)
     # Generate environment file with install-specific values (e.g. arch-dependent paths)
     $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
 # Generated by setup.sh — install-specific environment variables
 SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
 EOF
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chmod 0640 /etc/spinifex/systemd.env
     info "Generated /etc/spinifex/systemd.env"
+
+    # Service helper scripts (root-owned, group-executable by all service users)
+    if [ -d "$EXTRACT_DIR/scripts" ]; then
+        for script in "$EXTRACT_DIR"/scripts/*.sh; do
+            $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
+                "$script" "/var/lib/spinifex/$(basename "$script")"
+            info "  /var/lib/spinifex/$(basename "$script")"
+        done
+    fi
+}
+
+# --- Fix file ownership for upgrades from v1 ---
+fix_file_ownership() {
+    info "Fixing file ownership for privilege separation..."
+
+    # Per-service data dirs — recursive chown so existing files are accessible
+    for entry in \
+        nats:spinifex-nats \
+        predastore:spinifex-storage \
+        spinifex:spinifex-daemon \
+        viperblock:spinifex-viperblock \
+        vpcd:spinifex-vpcd \
+        awsgw:spinifex-gw; do
+        IFS=: read -r dir svc_user <<< "$entry"
+        if [ -d "/var/lib/spinifex/$dir" ]; then
+            $SUDO chown -R "$svc_user:$SPINIFEX_GROUP" "/var/lib/spinifex/$dir" \
+                || fatal "Failed to set ownership on /var/lib/spinifex/$dir"
+        fi
+    done
+
+    # Per-service config dirs — recursive chown
+    if [ -d /etc/spinifex/nats ]; then
+        $SUDO chown -R "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats \
+            || fatal "Failed to set ownership on /etc/spinifex/nats"
+    fi
+    if [ -d /etc/spinifex/predastore ]; then
+        $SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
+            || fatal "Failed to set ownership on /etc/spinifex/predastore"
+    fi
+
+    # Shared config files — root:spinifex with per-file modes
+    for f in spinifex.toml master.key server.key; do
+        if [ -f "/etc/spinifex/$f" ]; then
+            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
+                || fatal "Failed to set ownership on /etc/spinifex/$f"
+            $SUDO chmod 0640 "/etc/spinifex/$f" \
+                || fatal "Failed to set permissions on /etc/spinifex/$f"
+        fi
+    done
+    for f in server.pem ca.pem; do
+        if [ -f "/etc/spinifex/$f" ]; then
+            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
+                || fatal "Failed to set ownership on /etc/spinifex/$f"
+            $SUDO chmod 0644 "/etc/spinifex/$f" \
+                || fatal "Failed to set permissions on /etc/spinifex/$f"
+        fi
+    done
+
+    # CA private key — root-only
+    if [ -f /etc/spinifex/ca.key ]; then
+        $SUDO chown root:root /etc/spinifex/ca.key \
+            || fatal "Failed to set ownership on /etc/spinifex/ca.key"
+        $SUDO chmod 0600 /etc/spinifex/ca.key \
+            || fatal "Failed to set permissions on /etc/spinifex/ca.key"
+    fi
+
+    # Shared data dirs — root:spinifex 0770 (daemon + admin CLI write, services read).
+    # chmod must be recursive so pre-existing files (e.g. imported images originally
+    # written as 0600 by another user) become group-readable.
+    for d in images amis volumes state; do
+        if [ -d "/var/lib/spinifex/$d" ]; then
+            $SUDO chown -R "root:$SPINIFEX_GROUP" "/var/lib/spinifex/$d" \
+                || fatal "Failed to set ownership on /var/lib/spinifex/$d"
+            $SUDO chmod -R u+rwX,g+rwX,o-rwx "/var/lib/spinifex/$d" \
+                || fatal "Failed to set permissions on /var/lib/spinifex/$d"
+        fi
+    done
+
+    info "File ownership updated"
+}
+
+# --- Run config migrations ---
+run_migrations() {
+    # Only run if spinifex is already initialized (config exists)
+    if [ ! -f /etc/spinifex/spinifex.toml ]; then
+        info "Fresh install detected, skipping migrations"
+        return
+    fi
+
+    if [ "${INSTALL_SPINIFEX_SKIP_MIGRATE:-0}" = "1" ]; then
+        info "INSTALL_SPINIFEX_SKIP_MIGRATE=1, skipping migrations"
+        info "Run 'spx admin upgrade' manually to apply pending migrations"
+        return
+    fi
+
+    info "Running config migrations..."
+    $SUDO /usr/local/bin/spx admin upgrade --yes \
+        || fatal "Config migration failed. See errors above."
 }
 
 # --- Install systemd units ---
@@ -283,15 +468,13 @@ install_systemd() {
     fi
 
     for unit in "$EXTRACT_DIR"/systemd/*; do
-        # Substitute User= and Group= with the detected service user
-        sed "s/^User=spinifex$/User=$SPINIFEX_USER/;s/^Group=spinifex$/Group=$SPINIFEX_GROUP/" \
-            "$unit" | $SUDO tee "/etc/systemd/system/$(basename "$unit")" > /dev/null
-        $SUDO chmod 0644 "/etc/systemd/system/$(basename "$unit")"
+        $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
         info "  /etc/systemd/system/$(basename "$unit")"
     done
 
     $SUDO systemctl daemon-reload
-    info "Systemd units installed (running as $SPINIFEX_USER:$SPINIFEX_GROUP)"
+    $SUDO systemctl enable spinifex.target
+    info "Systemd units installed and enabled (per-service users)"
 }
 
 # --- Install logrotate ---
@@ -332,7 +515,7 @@ print_summary() {
     echo ""
     echo "  Version:      $INSTALLED_VERSION"
     echo "  Architecture: $ARCH"
-    echo "  Service user: $SPINIFEX_USER"
+    echo "  Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
     echo "  Binary:       /usr/local/bin/spx"
     echo "  Config:       /etc/spinifex/"
     echo "  Data:         /var/lib/spinifex/"
@@ -341,16 +524,15 @@ print_summary() {
     echo "  Next steps:"
     echo ""
     echo "  1. Setup OVN networking:"
-    echo "     sudo /usr/local/share/spinifex/setup-ovn.sh --management"
-    echo "     # If WAN is a physical NIC (not a bridge), add:"
-    echo "     #   --wan-bridge=br-wan --wan-iface=<WAN_NIC>   (dedicated WAN NIC)"
-    echo "     #   --macvlan --wan-iface=<WAN_NIC>              (single-NIC, SSH-safe)"
+    echo "     If your WAN interface is already a bridge (e.g. br-wan):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management"
     echo ""
-    echo "  2. Initialize a region:"
-    echo "     sudo spx admin init --region <region> --az <az> --node node1 --nodes 1"
+    echo "     If your WAN is a physical NIC:"
+    echo "       # Dedicated WAN NIC (not your SSH connection):"
+    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
     echo ""
-    echo "  2. Setup OVN networking:"
-    echo "     sudo /usr/local/share/spinifex/setup-ovn.sh --management"
+    echo "  2. Initialize:"
+    echo "     sudo spx admin init --node node1 --nodes 1"
     echo ""
     echo "  3. Start services:"
     echo "     sudo systemctl start spinifex.target"
@@ -372,15 +554,30 @@ main() {
     handle_upgrade
     install_apt_deps
     install_aws_cli
-    detect_service_user
+    create_service_users
+    install_sudoers
     download_spinifex
     install_files
     create_directories
+    fix_file_ownership
     install_systemd
+    run_migrations
     install_logrotate
     rm -rf "$SPINIFEX_TMPDIR"
     restart_if_needed
     print_summary
+
+    # Activate spinifex group membership in the invoking shell. Under curl|bash
+    # stdin is the drained pipe, so redirect from /dev/tty and exec so the new
+    # shell becomes the foreground process. Skip when we can't actually open
+    # /dev/tty (CI, cloud-init, ssh -T — stat passes but open fails with ENXIO).
+    if ! id -Gn 2>/dev/null | grep -qw "$SPINIFEX_GROUP" \
+        && ( : </dev/tty ) 2>/dev/null; then
+        echo ""
+        echo "  Activating '$SPINIFEX_GROUP' group in a subshell — type 'exit' when done."
+        echo ""
+        exec newgrp "$SPINIFEX_GROUP" < /dev/tty
+    fi
 }
 
 main "$@"

scripts/start-dev.sh

@@ -11,6 +11,11 @@
 
 set -euo pipefail
 
+echo "WARNING: start-dev.sh is deprecated for local development." >&2
+echo "  Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2
+echo "  start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2
+echo "" >&2
+
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
@@ -33,6 +38,8 @@ echo "Using configuration directory: $CONFIG_DIR"
 LOGS_DIR="$DATA_DIR/logs"
 WAL_DIR="$DATA_DIR/spinifex"
 
+echo "⚠️  Dev mode: all services run as $(whoami) without privilege separation"
+echo ""
 echo "🚀 Starting Spinifex development environment..."
 echo "Project root: $PROJECT_ROOT"
 echo "Data directory: $DATA_DIR"
@@ -457,7 +464,7 @@ if is_multinode; then
         attempts=0
         max_attempts=5
         while [ $attempts -lt $max_attempts ]; do
-            if curl -s --connect-timeout 3 "http://$peer/health" > /dev/null 2>&1; then
+            if curl -sk --connect-timeout 3 "https://$peer/health" > /dev/null 2>&1; then
                 echo "   $peer: healthy"
                 break
             fi

scripts/start-multi-nats.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-SPINIFEX_CONFIG_PATH=~/node1/config/nats/nats.conf ./bin/spx service nats start

scripts/workload-performance.sh

@@ -5,8 +5,8 @@
 sudo apt update
 sudo apt install -y make gcc git
 
-wget https://go.dev/dl/go1.26.1.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.26.1.linux-amd64.tar.gz
+wget https://go.dev/dl/go1.26.2.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.26.2.linux-amd64.tar.gz
 
 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
 source $HOME/.bashrc

spinifex/admin/admin.go

@@ -40,6 +40,7 @@ type ConfigSettings struct {
 	Region    string
 	NatsToken string
 	DataDir   string
+	LogDir    string
 	ConfigDir string
 
 	// Add more fields as needed
@@ -212,7 +213,9 @@ func CreateServiceDirectories(spxRoot string) {
 		filepath.Join(spxRoot, "nats"),
 		filepath.Join(spxRoot, "predastore"),
 		filepath.Join(spxRoot, "viperblock"),
+		filepath.Join(spxRoot, "vpcd"),
 		filepath.Join(spxRoot, "spinifex"),
+		filepath.Join(spxRoot, "awsgw"),
 	}
 
 	fmt.Println("\n📁 Creating directory structure...")
@@ -241,18 +244,30 @@ func FileExists(path string) bool {
 func ChownRecursive(path, username string) {
 	u, err := user.Lookup(username)
 	if err != nil {
+		slog.Warn("ChownRecursive: user lookup failed, skipping", "user", username, "path", path, "err", err)
+		return
+	}
+	uid, err := strconv.Atoi(u.Uid)
+	if err != nil {
+		slog.Warn("ChownRecursive: invalid UID, skipping", "user", username, "uid", u.Uid, "err", err)
+		return
+	}
+	gid, err := strconv.Atoi(u.Gid)
+	if err != nil {
+		slog.Warn("ChownRecursive: invalid GID, skipping", "user", username, "gid", u.Gid, "err", err)
 		return
 	}
-	uid, _ := strconv.Atoi(u.Uid)
-	gid, _ := strconv.Atoi(u.Gid)
 
 	// Use os.Root to scope filesystem operations and avoid symlink TOCTOU races.
 	// Falls back to direct chown if os.Root is not available.
 	root, rootErr := os.OpenRoot(path)
 	if rootErr != nil {
-		// Fallback: direct chown on the top-level path only
-		if chownErr := os.Chown(path, uid, gid); chownErr != nil { // #nosec G122
-			slog.Debug("chown failed", "path", path, "err", chownErr)
+		// Fallback: direct chown on the top-level path only — subdirectory contents
+		// will retain their original ownership.
+		slog.Warn("ChownRecursive: OpenRoot not available, only top-level directory ownership changed",
+			"path", path, "err", rootErr)
+		if chownErr := os.Lchown(path, uid, gid); chownErr != nil {
+			slog.Warn("chown failed", "path", path, "err", chownErr)
 		}
 		return
 	}
@@ -271,6 +286,81 @@ func ChownRecursive(path, username string) {
 	})
 }
 
+// SetServiceOwnership sets per-service ownership on data/config directories
+// and shared config files to root:spinifex with correct modes.
+// Keep in sync with setup.sh create_directories() and plan doc section 2.
+func SetServiceOwnership() {
+	grp, err := user.LookupGroup("spinifex")
+	if err != nil {
+		slog.Error("SetServiceOwnership: spinifex group not found, skipping all ownership changes", "err", err)
+		fmt.Fprintln(os.Stderr, "WARNING: spinifex group not found — service ownership not set. Run setup.sh first or create the group manually.")
+		return
+	}
+	gid, err := strconv.Atoi(grp.Gid)
+	if err != nil {
+		slog.Error("SetServiceOwnership: invalid spinifex group GID", "gid", grp.Gid, "err", err)
+		fmt.Fprintln(os.Stderr, "WARNING: invalid spinifex group GID — service ownership not set.")
+		return
+	}
+
+	// Per-service directory trees
+	for path, u := range map[string]string{
+		"/etc/spinifex/nats":           "spinifex-nats",
+		"/var/lib/spinifex/nats":       "spinifex-nats",
+		"/etc/spinifex/predastore":     "spinifex-storage",
+		"/var/lib/spinifex/predastore": "spinifex-storage",
+		"/var/lib/spinifex/spinifex":   "spinifex-daemon",
+		"/var/lib/spinifex/viperblock": "spinifex-viperblock",
+		"/var/lib/spinifex/vpcd":       "spinifex-vpcd",
+		"/var/lib/spinifex/awsgw":      "spinifex-gw",
+	} {
+		if _, err := os.Stat(path); err != nil {
+			continue
+		}
+		ChownRecursive(path, u)
+	}
+
+	// Shared data directories — root:spinifex 0770 so daemon + admin CLI can write
+	for _, dir := range []string{
+		"/var/lib/spinifex/images",
+		"/var/lib/spinifex/amis",
+		"/var/lib/spinifex/volumes",
+		"/var/lib/spinifex/state",
+		"/run/spinifex/nbd",
+	} {
+		if _, err := os.Stat(dir); err != nil {
+			continue
+		}
+		if err := os.Lchown(dir, 0, gid); err != nil {
+			slog.Warn("SetServiceOwnership: chown failed", "path", dir, "err", err)
+		}
+		if err := os.Chmod(dir, 0770); err != nil { //nolint:gosec // directories need group-write for daemon + admin CLI
+			slog.Warn("SetServiceOwnership: chmod failed", "path", dir, "err", err)
+		}
+	}
+
+	// Shared config files — root:spinifex, ca.key stays root:root 0600
+	// bootstrap.json lives in the awsgw data dir (not /etc/spinifex),
+	// so /etc/spinifex stays at 0750 (no group-write needed).
+	for path, mode := range map[string]os.FileMode{
+		"/etc/spinifex/spinifex.toml": 0640,
+		"/etc/spinifex/master.key":    0640,
+		"/etc/spinifex/server.pem":    0644,
+		"/etc/spinifex/server.key":    0640,
+		"/etc/spinifex/ca.pem":        0644,
+	} {
+		if _, err := os.Stat(path); err != nil {
+			continue
+		}
+		if err := os.Lchown(path, 0, gid); err != nil {
+			slog.Warn("SetServiceOwnership: chown failed", "path", path, "err", err)
+		}
+		if err := os.Chmod(path, mode); err != nil {
+			slog.Warn("SetServiceOwnership: chmod failed", "path", path, "err", err)
+		}
+	}
+}
+
 // updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
 func UpdateAWSINIFile(path, section string, values map[string]string) error {
 	var cfg *ini.File
@@ -427,53 +517,109 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
 	return nil
 }
 
+// DiscoverLocalIPs enumerates all non-loopback network interface addresses on
+// this machine and returns them as strings. Link-local addresses are excluded.
+func DiscoverLocalIPs() []string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		slog.Warn("failed to enumerate network interfaces", "error", err)
+		return nil
+	}
+
+	seen := make(map[string]struct{})
+	var ips []string
+
+	for _, iface := range ifaces {
+		// Skip loopback and down interfaces.
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			slog.Debug("failed to get addresses for interface", "iface", iface.Name, "error", err)
+			continue
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+				continue
+			}
+			s := ip.String()
+			if _, ok := seen[s]; !ok {
+				seen[s] = struct{}{}
+				ips = append(ips, s)
+			}
+		}
+	}
+	return ips
+}
+
+// DiscoverHostname returns the system hostname suitable for use as a DNS SAN.
+// Returns an empty string if the hostname cannot be determined or is "localhost".
+func DiscoverHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		slog.Warn("failed to determine system hostname", "error", err)
+		return ""
+	}
+	if hostname == "" || hostname == "localhost" {
+		return ""
+	}
+	return hostname
+}
+
 // GenerateSignedCert generates a server certificate signed by the CA.
 // extraIPs are additional IP addresses to include in the certificate's SANs.
+// All non-loopback interface IPs on the local machine are automatically included.
 func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
-	// Load CA certificate
+	return GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, nil)
+}
+
+// GenerateSignedCertWithDNS generates a server certificate signed by the CA.
+// All non-loopback interface IPs and the machine hostname are automatically
+// included. extraIPs and extraDNS allow adding additional SANs.
+func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, extraIPs, extraDNS []string) error {
 	caCertPEM, err := os.ReadFile(caCertPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA cert: %w", err)
 	}
-
 	caCertBlock, _ := pem.Decode(caCertPEM)
 	if caCertBlock == nil {
 		return fmt.Errorf("failed to decode CA cert PEM")
 	}
-
 	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA cert: %w", err)
 	}
 
-	// Load CA private key
 	caKeyPEM, err := os.ReadFile(caKeyPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA key: %w", err)
 	}
-
 	caKeyBlock, _ := pem.Decode(caKeyPEM)
 	if caKeyBlock == nil {
 		return fmt.Errorf("failed to decode CA key PEM")
 	}
-
 	caPrivateKey, err := x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA private key: %w", err)
 	}
-
 	caRSAKey, ok := caPrivateKey.(*rsa.PrivateKey)
 	if !ok {
 		return fmt.Errorf("CA key is not RSA")
 	}
 
-	// Generate server private key
 	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
 		return fmt.Errorf("failed to generate server private key: %w", err)
 	}
 
-	// Create server certificate template
 	notBefore := time.Now()
 	notAfter := notBefore.Add(365 * 24 * time.Hour) // 1 year for server certs
 
@@ -482,20 +628,51 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}
 
-	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	// Build IP list: localhost IPs + auto-discovered interface IPs + explicit extras.
+	seen := map[string]struct{}{"127.0.0.1": {}, "::1": {}}
 	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	addIP := func(s string) {
+		if s == "" || s == "0.0.0.0" {
+			return
+		}
+		if _, ok := seen[s]; ok {
+			return
+		}
+		if parsed := net.ParseIP(s); parsed != nil {
+			seen[s] = struct{}{}
+			ipAddresses = append(ipAddresses, parsed)
+		}
+	}
+	for _, ip := range DiscoverLocalIPs() {
+		addIP(ip)
+	}
 	for _, ip := range extraIPs {
-		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
-			if parsed := net.ParseIP(ip); parsed != nil {
-				ipAddresses = append(ipAddresses, parsed)
-			}
+		addIP(ip)
+	}
+
+	// Build DNS names: always include localhost, plus hostname if available.
+	dnsSeen := map[string]struct{}{"localhost": {}}
+	dnsNames := []string{"localhost"}
+	if hostname := DiscoverHostname(); hostname != "" {
+		if _, ok := dnsSeen[hostname]; !ok {
+			dnsSeen[hostname] = struct{}{}
+			dnsNames = append(dnsNames, hostname)
+		}
+	}
+	for _, dns := range extraDNS {
+		if dns == "" {
+			continue
+		}
+		if _, ok := dnsSeen[dns]; !ok {
+			dnsSeen[dns] = struct{}{}
+			dnsNames = append(dnsNames, dns)
 		}
 	}
 
 	serverTemplate := x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
-			CommonName:   "localhost",
+			CommonName:   "Spinifex Server",
 			Organization: []string{"Spinifex Platform"},
 		},
 		NotBefore:             notBefore,
@@ -503,39 +680,33 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
-		DNSNames:              []string{"localhost"},
+		DNSNames:              dnsNames,
 		IPAddresses:           ipAddresses,
 	}
 
-	// Sign the server certificate with the CA
 	serverDerBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, caCert, &serverPrivateKey.PublicKey, caRSAKey)
 	if err != nil {
 		return fmt.Errorf("failed to create server certificate: %w", err)
 	}
 
-	// Write server certificate to file
 	certOut, err := os.Create(certPath)
 	if err != nil {
 		return fmt.Errorf("failed to create cert file: %w", err)
 	}
 	defer certOut.Close()
-
 	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverDerBytes}); err != nil {
 		return fmt.Errorf("failed to write cert: %w", err)
 	}
 
-	// Write server private key to file
 	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to create key file: %w", err)
 	}
 	defer keyOut.Close()
-
 	privBytes, err := x509.MarshalPKCS8PrivateKey(serverPrivateKey)
 	if err != nil {
 		return fmt.Errorf("failed to marshal private key: %w", err)
 	}
-
 	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
 		return fmt.Errorf("failed to write key: %w", err)
 	}
@@ -672,7 +843,7 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 
 	if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
 		"region":       region,
-		"endpoint_url": fmt.Sprintf("https://%s:9999", endpointHost),
+		"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
 		"ca_bundle":    certPath,
 		"output":       "json",
 	}); err != nil {

spinifex/admin/admin_test.go

@@ -335,7 +335,7 @@ func TestGenerateSignedCert(t *testing.T) {
 		cert, err := x509.ParseCertificate(block.Bytes)
 		require.NoError(t, err)
 		assert.False(t, cert.IsCA)
-		assert.Equal(t, "localhost", cert.Subject.CommonName)
+		assert.Equal(t, "Spinifex Server", cert.Subject.CommonName)
 		assert.Contains(t, cert.DNSNames, "localhost")
 
 		hasLoopback := false
@@ -346,6 +346,9 @@ func TestGenerateSignedCert(t *testing.T) {
 		}
 		assert.True(t, hasLoopback)
 
+		// Auto-discovered IPs should be present (at least loopback + any interface IPs).
+		assert.GreaterOrEqual(t, len(cert.IPAddresses), 2)
+
 		// Verify against CA
 		caCertPEM, _ := os.ReadFile(caCertPath)
 		caBlock, _ := pem.Decode(caCertPEM)
@@ -392,7 +395,19 @@ func TestGenerateSignedCert(t *testing.T) {
 		block, _ := pem.Decode(certPEM)
 		cert, _ := x509.ParseCertificate(block.Bytes)
 
-		assert.Len(t, cert.IPAddresses, 2)
+		// Should have loopback (127.0.0.1, ::1) + auto-discovered interface IPs.
+		// Duplicates (127.0.0.1, ::1) and specials (0.0.0.0, "") must not add extras.
+		// Count unique IPs from a clean run without extras.
+		baseDir := t.TempDir()
+		baseCert := filepath.Join(baseDir, "server.pem")
+		baseKey := filepath.Join(baseDir, "server.key")
+		require.NoError(t, GenerateSignedCert(baseCert, baseKey, caCertPath, caKeyPath))
+		basePEM, _ := os.ReadFile(baseCert)
+		baseBlock, _ := pem.Decode(basePEM)
+		baseParsed, _ := x509.ParseCertificate(baseBlock.Bytes)
+
+		assert.Equal(t, len(baseParsed.IPAddresses), len(cert.IPAddresses),
+			"passing duplicate/special IPs should not add extra entries")
 	})
 
 	t.Run("InvalidCACert", func(t *testing.T) {
@@ -406,6 +421,178 @@ func TestGenerateSignedCert(t *testing.T) {
 	})
 }
 
+func TestDiscoverLocalIPs(t *testing.T) {
+	t.Parallel()
+	ips := DiscoverLocalIPs()
+	// Should find at least one non-loopback IP on any machine with a network interface.
+	assert.NotEmpty(t, ips, "expected at least one non-loopback IP")
+
+	for _, ip := range ips {
+		parsed := net.ParseIP(ip)
+		require.NotNil(t, parsed, "returned IP should be parseable: %s", ip)
+		assert.False(t, parsed.IsLoopback(), "should not include loopback: %s", ip)
+		assert.False(t, parsed.IsLinkLocalUnicast(), "should not include link-local: %s", ip)
+	}
+
+	// No duplicates.
+	seen := make(map[string]struct{})
+	for _, ip := range ips {
+		_, dup := seen[ip]
+		assert.False(t, dup, "duplicate IP: %s", ip)
+		seen[ip] = struct{}{}
+	}
+}
+
+func TestDiscoverHostname(t *testing.T) {
+	t.Parallel()
+	hostname := DiscoverHostname()
+	// We can't assert a specific value, but if non-empty it shouldn't be "localhost".
+	if hostname != "" {
+		assert.NotEqual(t, "localhost", hostname)
+	}
+}
+
+func TestGenerateSignedCertWithDNS_DedupDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Pass "localhost" as extra DNS — should not duplicate the built-in entry.
+	// Also pass empty strings and duplicates — should be ignored.
+	extraDNS := []string{"localhost", "", "example.local", "example.local", ""}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	// Count occurrences of "localhost" — must be exactly 1.
+	localhostCount := 0
+	exampleCount := 0
+	for _, dns := range cert.DNSNames {
+		if dns == "localhost" {
+			localhostCount++
+		}
+		if dns == "example.local" {
+			exampleCount++
+		}
+	}
+	assert.Equal(t, 1, localhostCount, "localhost should appear exactly once")
+	assert.Equal(t, 1, exampleCount, "example.local should appear exactly once")
+	assert.Contains(t, cert.DNSNames, "example.local")
+}
+
+func TestGenerateSignedCertWithDNS_NilSlices(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Both slices nil — should still produce a valid cert with defaults.
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, nil))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.GreaterOrEqual(t, len(cert.IPAddresses), 2, "should have at least loopback IPs")
+}
+
+func TestGenerateSignedCertWithDNS_InvalidCA(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// Missing CA cert file.
+	err := GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "missing-ca.pem"), filepath.Join(dir, "missing-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to read CA cert")
+
+	// Corrupt CA cert.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.pem"), []byte("not-pem"), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.key"), []byte("not-pem"), 0600))
+	err = GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "bad-ca.pem"), filepath.Join(dir, "bad-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+}
+
+func TestGenerateSignedCertWithDNS_ExtraDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	extraIPs := []string{"10.0.0.42"}
+	extraDNS := []string{"spinifex.local", "node1.spinifex.local"}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.Contains(t, cert.DNSNames, "spinifex.local")
+	assert.Contains(t, cert.DNSNames, "node1.spinifex.local")
+
+	hasExtraIP := false
+	for _, ip := range cert.IPAddresses {
+		if ip.Equal(net.ParseIP("10.0.0.42")) {
+			hasExtraIP = true
+		}
+	}
+	assert.True(t, hasExtraIP, "cert should contain extra IP 10.0.0.42")
+
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+func TestGenerateSignedCert_IncludesHostname(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+	require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	hostname := DiscoverHostname()
+	if hostname != "" {
+		assert.Contains(t, cert.DNSNames, hostname,
+			"cert DNS SANs should include the machine hostname")
+	}
+}
+
 func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
 	t.Parallel()
 	dir := t.TempDir()