chore: use ts config files

Replace fragile image-name substring match (-lb-/-alb-) with a
DescribeImages tag filter on spinifex:managed-by=elbv2. Remove the
silent fallback that picks the first image in the store when no LB
image exists — CreateLoadBalancer now returns ServerInternal with a
remediation hint instead of persisting a record in StateFailed.

- daemon: tag filter + error return from the resolver closure
- service_impl: resolver signature is func() (string, error); caches
  only successful results so the next call retries after an error
- CreateLoadBalancer gate now requires launcher + networking; AMI is
  resolved inside the gate and failures short-circuit before persist
- admin images import: add --name+--file combo (catalog metadata with
  a local file source) and a --tag key=value flag for explicit tagging
  of file-only imports
- build-system-image.sh + lb.conf: propagate SYSTEM_TAG via --tag so
  locally built system AMIs get the right tag even when the catalog
  is bypassed

Closes mulga-969

.github/assets/architecture.svg

@@ -0,0 +1,225 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="620" viewBox="0 0 900 620">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#161b22"/>
+      <stop offset="45%" style="stop-color:#1f2028"/>
+      <stop offset="100%" style="stop-color:#161b22"/>
+    </linearGradient>
+    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262633"/>
+      <stop offset="100%" style="stop-color:#7F1D1D14"/>
+    </linearGradient>
+    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#F87171"/>
+      <stop offset="50%" style="stop-color:#DC2626"/>
+      <stop offset="100%" style="stop-color:#7F1D1D"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="620" fill="url(#bg)" rx="12"/>
+
+  <!-- Title bar -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ARCHITECTURE</text>
+  <text x="24" y="52" font-family="'Segoe UI', system-ui, sans-serif" font-size="16" fill="#f5f5f5" font-weight="500">AWS SDK → Gateway → NATS → Service daemons</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">stateless · message-driven · horizontally scalable</text>
+
+  <!-- ═══ CLIENT LAYER ════════════════════════════════════ -->
+  <rect x="16" y="72" width="868" height="78" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
+  <text x="32" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700" letter-spacing="1">CLIENTS</text>
+  <text x="32" y="108" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Standard AWS tooling — unmodified, no shims, no custom SDK</text>
+
+  <g transform="translate(32, 120)" font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <rect x="0" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="43" y="15" fill="#7ee787" text-anchor="middle">aws-cli</text>
+
+    <rect x="94" y="0" width="96" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="142" y="15" fill="#7ee787" text-anchor="middle">Terraform</text>
+
+    <rect x="198" y="0" width="76" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="236" y="15" fill="#7ee787" text-anchor="middle">boto3</text>
+
+    <rect x="282" y="0" width="98" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="331" y="15" fill="#7ee787" text-anchor="middle">aws-sdk-go</text>
+
+    <rect x="388" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="431" y="15" fill="#7ee787" text-anchor="middle">kubectl</text>
+
+    <rect x="482" y="0" width="86" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="525" y="15" fill="#7ee787" text-anchor="middle">Packer</text>
+
+    <rect x="576" y="0" width="120" height="22" rx="4" fill="#3fb950" fill-opacity="0.12" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.8"/>
+    <text x="636" y="15" fill="#7ee787" text-anchor="middle">your application</text>
+  </g>
+
+  <!-- ═══ HTTPS + SigV4 connector ═══════════════════════════ -->
+  <line x1="450" y1="150" x2="450" y2="178" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="390" y="154" width="120" height="18" rx="9" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="166" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#F87171" text-anchor="middle">HTTPS · SigV4 · :9999</text>
+
+  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
+  <rect x="16" y="178" width="868" height="68" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
+  <text x="32" y="200" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY</text>
+  <text x="32" y="215" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
+
+  <g transform="translate(32, 224)" font-family="'SF Mono', ui-monospace, monospace" font-size="9">
+    <rect x="0" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="39" y="11" fill="#FCA5A5" text-anchor="middle">TLS 1.3</text>
+
+    <rect x="86" y="0" width="78" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="125" y="11" fill="#FCA5A5" text-anchor="middle">SigV4 auth</text>
+
+    <rect x="172" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="221" y="11" fill="#FCA5A5" text-anchor="middle">policy eval</text>
+
+    <rect x="278" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="327" y="11" fill="#FCA5A5" text-anchor="middle">req routing</text>
+
+    <rect x="384" y="0" width="88" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="428" y="11" fill="#FCA5A5" text-anchor="middle">rate-limit</text>
+
+    <rect x="480" y="0" width="98" height="16" rx="3" fill="#7F1D1D" fill-opacity="0.3" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="529" y="11" fill="#FCA5A5" text-anchor="middle">audit log</text>
+  </g>
+
+  <!-- ═══ NATS BUS ═══════════════════════════════════════ -->
+  <line x1="450" y1="246" x2="450" y2="272" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="16" y="272" width="868" height="34" rx="6" fill="url(#nats-grad)" opacity="0.92"/>
+  <text x="32" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS BUS</text>
+  <text x="140" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#fee2e2" opacity="0.95">ec2.*   ebs.*   s3.*   vpc.*   iam.*</text>
+  <text x="876" y="293" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">queue groups · fan-out · request/reply</text>
+
+  <!-- ═══ DAEMON FANOUT CONNECTORS ═══════════════════════ -->
+  <g stroke="#DC2626" stroke-width="0.9" stroke-dasharray="3,3" opacity="0.55" fill="none">
+    <line x1="100" y1="306" x2="100" y2="328"/>
+    <line x1="275" y1="306" x2="275" y2="328"/>
+    <line x1="450" y1="306" x2="450" y2="328"/>
+    <line x1="625" y1="306" x2="625" y2="328"/>
+    <line x1="800" y1="306" x2="800" y2="328"/>
+  </g>
+
+  <!-- ═══ SERVICE DAEMONS ROW ═══════════════════════════ -->
+  <!-- EC2 -->
+  <g transform="translate(28, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ec2.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">RunInstances</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">TerminateInstances</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">QMP · cloud-init</text>
+  </g>
+
+  <!-- EBS -->
+  <g transform="translate(204, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">ebs.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">CreateVolume</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">AttachVolume</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">snapshots · WAL</text>
+  </g>
+
+  <!-- S3 -->
+  <g transform="translate(380, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">s3.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">PutObject · GetObject</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Multipart · presigned</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">erasure-coded</text>
+  </g>
+
+  <!-- VPC -->
+  <g transform="translate(556, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">vpc.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Subnets · SGs · RTs</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">ALB · NAT · IGW</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">OVN · OVS</text>
+  </g>
+
+  <!-- IAM -->
+  <g transform="translate(732, 328)">
+    <rect width="144" height="82" rx="7" fill="#7F1D1D" fill-opacity="0.14" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">iam.* daemon</text>
+    <line x1="16" y1="30" x2="128" y2="30" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <text x="72" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Users · Roles · Keys</text>
+    <text x="72" y="62" font-family="'Segoe UI', sans-serif" font-size="10" fill="#f5f5f5" text-anchor="middle">Policy evaluation</text>
+    <text x="72" y="76" font-family="'Segoe UI', sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">audit trail</text>
+  </g>
+
+  <!-- ═══ DAEMON → BACKEND CONNECTORS ═══════════════════ -->
+  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
+    <line x1="100" y1="410" x2="100" y2="440"/>
+    <line x1="276" y1="410" x2="276" y2="440"/>
+    <line x1="452" y1="410" x2="452" y2="440"/>
+    <line x1="628" y1="410" x2="628" y2="440"/>
+    <line x1="804" y1="410" x2="804" y2="440"/>
+  </g>
+
+  <!-- ═══ BACKEND RESOURCES BAR ═════════════════════════ -->
+  <rect x="16" y="440" width="868" height="26" rx="5" fill="url(#bar-grad)"/>
+  <text x="450" y="458" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">BACKEND RESOURCES · local hardware · no cloud dependency</text>
+
+  <!-- ═══ BACKEND ROW ═══════════════════════════════════ -->
+  <!-- QEMU/KVM -->
+  <g transform="translate(28, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">hypervisor</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">host kernel · virtio</text>
+  </g>
+
+  <!-- Viperblock -->
+  <g transform="translate(204, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Viperblock</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">block storage</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">NVMe WAL · NBD</text>
+  </g>
+
+  <!-- Predastore -->
+  <g transform="translate(380, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Predastore</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">object storage</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">Reed–Solomon</text>
+  </g>
+
+  <!-- OVN/OVS -->
+  <g transform="translate(556, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">OVN / OVS</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SDN fabric</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">overlays · flow rules</text>
+  </g>
+
+  <!-- IAM store -->
+  <g transform="translate(732, 474)">
+    <rect width="144" height="62" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="72" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" text-anchor="middle" font-weight="600">Identity store</text>
+    <text x="72" y="40" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">users · keys · rules</text>
+    <text x="72" y="54" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a" text-anchor="middle">local · audit-logged</text>
+  </g>
+
+  <!-- ═══ FOOTER NOTES ══════════════════════════════════ -->
+  <g transform="translate(16, 558)">
+    <rect width="868" height="48" rx="6" fill="#0a0a0a" stroke="#27272a" stroke-width="0.8"/>
+    <text x="24" y="20" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
+      <tspan fill="#F87171" font-weight="600">Queue groups</tspan> load-balance ops across daemon replicas.
+      <tspan fill="#F87171" font-weight="600"> Fan-out</tspan> (no queue group) dispatches describe calls to every node.
+    </text>
+    <text x="24" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">
+      Daemons are <tspan fill="#F87171" font-weight="600">stateless</tspan> — scale horizontally by starting more.
+      No etcd, no Kubernetes, no external control plane. Systemd units, a NATS cluster, and your hardware.
+    </text>
+  </g>
+</svg>

.github/assets/banner.svg

@@ -0,0 +1,154 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="300" viewBox="0 0 900 300">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a;stop-opacity:1" />
+      <stop offset="50%" style="stop-color:#0d1117;stop-opacity:1" />
+      <stop offset="100%" style="stop-color:#0a0a0a;stop-opacity:1" />
+    </linearGradient>
+    <radialGradient id="vignette" cx="50%" cy="50%" r="70%">
+      <stop offset="0%" style="stop-color:#7F1D1D;stop-opacity:0.08"/>
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0"/>
+    </radialGradient>
+    <linearGradient id="mulga-grad" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.95" />
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.95" />
+    </linearGradient>
+    <linearGradient id="rule" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0" />
+      <stop offset="50%" style="stop-color:#F87171;stop-opacity:0.45" />
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0" />
+    </linearGradient>
+    <filter id="node-glow">
+      <feGaussianBlur in="SourceGraphic" stdDeviation="3" />
+    </filter>
+  </defs>
+
+  <!-- Background -->
+  <rect width="900" height="300" fill="url(#bg)" rx="12"/>
+  <rect width="900" height="300" fill="url(#vignette)" rx="12"/>
+
+  <!-- Mesh network background (mulgadc style) -->
+  <g opacity="0.35">
+    <!-- Connections (drawn first, under nodes) -->
+    <g stroke="#7F1D1D" stroke-width="0.6" fill="none">
+      <line x1="80" y1="60" x2="200" y2="40"/>
+      <line x1="80" y1="60" x2="150" y2="140"/>
+      <line x1="200" y1="40" x2="330" y2="90"/>
+      <line x1="150" y1="140" x2="260" y2="180"/>
+      <line x1="150" y1="140" x2="60" y2="220"/>
+      <line x1="330" y1="90" x2="260" y2="180"/>
+      <line x1="260" y1="180" x2="120" y2="260"/>
+      <line x1="60" y1="220" x2="120" y2="260"/>
+
+      <line x1="620" y1="50" x2="760" y2="90"/>
+      <line x1="620" y1="50" x2="700" y2="160"/>
+      <line x1="760" y1="90" x2="850" y2="160"/>
+      <line x1="700" y1="160" x2="820" y2="220"/>
+      <line x1="850" y1="160" x2="820" y2="220"/>
+      <line x1="700" y1="160" x2="600" y2="240"/>
+      <line x1="600" y1="240" x2="820" y2="220"/>
+      <line x1="760" y1="90" x2="700" y2="160"/>
+    </g>
+    <!-- Nodes -->
+    <g fill="#DC2626">
+      <circle cx="80" cy="60" r="2.5"/>
+      <circle cx="200" cy="40" r="2"/>
+      <circle cx="330" cy="90" r="2.5"/>
+      <circle cx="150" cy="140" r="3"/>
+      <circle cx="260" cy="180" r="2"/>
+      <circle cx="60" cy="220" r="2"/>
+      <circle cx="120" cy="260" r="2.5"/>
+
+      <circle cx="620" cy="50" r="2"/>
+      <circle cx="760" cy="90" r="3"/>
+      <circle cx="700" cy="160" r="2.5"/>
+      <circle cx="850" cy="160" r="2"/>
+      <circle cx="820" cy="220" r="2.5"/>
+      <circle cx="600" cy="240" r="2"/>
+    </g>
+    <!-- Pulse rings (static "about-to-pulse" look) -->
+    <g fill="none" stroke="#F87171" stroke-width="0.8" opacity="0.5">
+      <circle cx="150" cy="140" r="6"/>
+      <circle cx="760" cy="90" r="6"/>
+      <circle cx="700" cy="160" r="8"/>
+    </g>
+  </g>
+
+  <!-- Main emblem: cluster-node stack with NATS bus -->
+  <g transform="translate(200, 150)">
+    <!-- Outer glow ring -->
+    <circle cx="0" cy="0" r="64" fill="none" stroke="url(#mulga-grad)" stroke-width="1.5" opacity="0.35" filter="url(#node-glow)"/>
+
+    <!-- Top node -->
+    <rect x="-24" y="-52" width="48" height="22" rx="3" fill="none" stroke="#DC2626" stroke-width="1.8"/>
+    <circle cx="-16" cy="-41" r="1.6" fill="#F87171"/>
+    <circle cx="-8" cy="-41" r="1.6" fill="#F87171"/>
+    <line x1="0" y1="-41" x2="16" y2="-41" stroke="#F87171" stroke-width="0.9" opacity="0.6"/>
+
+    <!-- Middle node (active) -->
+    <rect x="-30" y="-14" width="60" height="28" rx="3" fill="url(#mulga-grad)" opacity="0.22" stroke="#DC2626" stroke-width="2"/>
+    <circle cx="-20" cy="0" r="2" fill="#F87171"/>
+    <circle cx="-10" cy="0" r="2" fill="#F87171"/>
+    <line x1="-2" y1="0" x2="22" y2="0" stroke="#F87171" stroke-width="1.2" opacity="0.8"/>
+
+    <!-- Bottom node -->
+    <rect x="-24" y="30" width="48" height="22" rx="3" fill="none" stroke="#7F1D1D" stroke-width="1.8"/>
+    <circle cx="-16" cy="41" r="1.6" fill="#DC2626"/>
+    <circle cx="-8" cy="41" r="1.6" fill="#DC2626"/>
+    <line x1="0" y1="41" x2="16" y2="41" stroke="#DC2626" stroke-width="0.9" opacity="0.6"/>
+
+    <!-- Radiating mesh ports -->
+    <g stroke-width="0.9" opacity="0.55">
+      <line x1="-30" y1="0" x2="-58" y2="-26" stroke="#DC2626"/>
+      <line x1="-30" y1="0" x2="-64" y2="0" stroke="#DC2626"/>
+      <line x1="-30" y1="0" x2="-58" y2="26" stroke="#DC2626"/>
+      <line x1="30" y1="0" x2="58" y2="-26" stroke="#7F1D1D"/>
+      <line x1="30" y1="0" x2="64" y2="0" stroke="#7F1D1D"/>
+      <line x1="30" y1="0" x2="58" y2="26" stroke="#7F1D1D"/>
+    </g>
+    <g fill="#F87171">
+      <circle cx="-58" cy="-26" r="2.5"/>
+      <circle cx="-64" cy="0" r="2.5"/>
+      <circle cx="-58" cy="26" r="2.5"/>
+    </g>
+    <g fill="#DC2626">
+      <circle cx="58" cy="-26" r="2.5"/>
+      <circle cx="64" cy="0" r="2.5"/>
+      <circle cx="58" cy="26" r="2.5"/>
+    </g>
+  </g>
+
+  <!-- Title -->
+  <text x="340" y="120" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="56" font-weight="700" fill="#f5f5f5" letter-spacing="-1.5">Spinifex</text>
+
+  <!-- Subtitle -->
+  <text x="340" y="155" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="17" fill="#a1a1aa" letter-spacing="0.3">Open-source AWS-compatible platform</text>
+  <text x="340" y="180" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="17" fill="#a1a1aa" letter-spacing="0.3">for secure edge deployments</text>
+
+  <!-- Rule -->
+  <rect x="340" y="200" width="440" height="1" fill="url(#rule)" rx="1"/>
+
+  <!-- Service pills -->
+  <g font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="11" font-weight="500">
+    <rect x="340" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="365" y="231" fill="#F87171" text-anchor="middle">EC2</text>
+
+    <rect x="398" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="423" y="231" fill="#F87171" text-anchor="middle">EBS</text>
+
+    <rect x="456" y="215" width="45" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="478" y="231" fill="#F87171" text-anchor="middle">S3</text>
+
+    <rect x="509" y="215" width="55" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="536" y="231" fill="#F87171" text-anchor="middle">VPC</text>
+
+    <rect x="572" y="215" width="50" height="24" rx="12" fill="#7F1D1D" fill-opacity="0.25" stroke="#DC2626" stroke-width="0.5" stroke-opacity="0.55"/>
+    <text x="597" y="231" fill="#F87171" text-anchor="middle">IAM</text>
+
+    <rect x="630" y="215" width="78" height="24" rx="12" fill="#18181b" stroke="#52525b" stroke-width="0.5" stroke-opacity="0.8"/>
+    <text x="669" y="231" fill="#a1a1aa" text-anchor="middle">Air-Gapped</text>
+  </g>
+
+  <!-- Tagline -->
+  <text x="340" y="265" font-family="'Segoe UI', system-ui, -apple-system, sans-serif" font-size="11" fill="#71717a" letter-spacing="1.8">YOUR INFRASTRUCTURE · YOUR HARDWARE · YOUR CLOUD</text>
+</svg>

.github/assets/design-architecture.svg

@@ -0,0 +1,175 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="720" viewBox="0 0 900 720">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="bar-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#161b22"/>
+      <stop offset="45%" style="stop-color:#1f2028"/>
+      <stop offset="100%" style="stop-color:#161b22"/>
+    </linearGradient>
+    <linearGradient id="client-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262633"/>
+      <stop offset="100%" style="stop-color:#7F1D1D14"/>
+    </linearGradient>
+    <linearGradient id="nats-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#F87171"/>
+      <stop offset="50%" style="stop-color:#DC2626"/>
+      <stop offset="100%" style="stop-color:#7F1D1D"/>
+    </linearGradient>
+    <linearGradient id="daemon-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D0a"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="720" fill="url(#bg)" rx="12"/>
+
+  <!-- Title -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">HIGH-LEVEL ARCHITECTURE</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">SDK → Gateway → NATS → Daemons → VMs</text>
+
+  <!-- ═══ AWS SDK / CLI ══════════════════════════════════ -->
+  <rect x="16" y="50" width="868" height="76" rx="8" fill="url(#client-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.5"/>
+  <text x="32" y="72" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700" letter-spacing="1">AWS SDK / CLI</text>
+  <text x="32" y="90" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">Users interact with Spinifex using standard AWS SDKs and CLIs against a custom endpoint URL</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <rect x="32" y="98" width="480" height="20" rx="4" fill="#0a0a0a" stroke="#3fb950" stroke-width="0.6" stroke-opacity="0.4"/>
+    <text x="42" y="112" fill="#7ee787">$ aws ec2 run-instances --endpoint-url https://localhost:9999 ...</text>
+  </g>
+
+  <!-- Connector: HTTPS + SigV4 -->
+  <line x1="450" y1="126" x2="450" y2="154" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="380" y="128" width="140" height="20" rx="10" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="142" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" text-anchor="middle">HTTPS · SigV4 Auth</text>
+
+  <!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
+  <rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
+  <text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
+  <text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
+
+  <!-- Two-column feature bullets -->
+  <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+    <text x="40" y="200">● TLS termination</text>
+    <text x="40" y="220">● EC2 Query Protocol parsing</text>
+    <text x="40" y="240">● XML response generation</text>
+
+    <text x="460" y="200">● AWS SigV4 authentication</text>
+    <text x="460" y="220">● Routes to service handlers</text>
+    <text x="460" y="240">● NATS client connection</text>
+  </g>
+
+  <!-- Connector: JSON over NATS -->
+  <line x1="450" y1="258" x2="450" y2="286" stroke="#DC2626" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.7"/>
+  <rect x="380" y="260" width="140" height="20" rx="10" fill="#0a0a0a" stroke="#DC2626" stroke-width="0.8" stroke-opacity="0.5"/>
+  <text x="450" y="274" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" text-anchor="middle">JSON over NATS</text>
+
+  <!-- ═══ NATS BROKER ═══════════════════════════════════ -->
+  <rect x="16" y="286" width="868" height="124" rx="8" fill="#1a0a0a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <rect x="16" y="286" width="868" height="28" rx="8" fill="url(#nats-grad)" opacity="0.9"/>
+  <text x="32" y="305" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#f5f5f5" font-weight="700" letter-spacing="1.5">NATS MESSAGE BROKER</text>
+  <text x="876" y="305" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#fee2e2" text-anchor="end" opacity="0.9">request/reply · queue groups · fan-out</text>
+
+  <!-- Two-column topics/queue groups -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="40" y="332" fill="#F87171" font-weight="600">Topics</text>
+    <text x="40" y="350" fill="#f5f5f5">ec2.RunInstances</text>
+    <text x="40" y="368" fill="#f5f5f5">ec2.DescribeInstances</text>
+    <text x="40" y="386" fill="#f5f5f5">ec2.cmd.{instanceID}</text>
+    <text x="40" y="404" fill="#f5f5f5">ebs.mount  ·  ebs.unmount</text>
+
+    <text x="460" y="332" fill="#F87171" font-weight="600">Delivery mode</text>
+    <text x="460" y="350" fill="#a1a1aa">spinifex-workers <tspan fill="#71717a">(load-balanced)</tspan></text>
+    <text x="460" y="368" fill="#a1a1aa">no queue group <tspan fill="#71717a">(fan-out to all nodes)</tspan></text>
+    <text x="460" y="386" fill="#a1a1aa">per-instance <tspan fill="#71717a">(start / stop / terminate)</tspan></text>
+    <text x="460" y="404" fill="#a1a1aa">Viperblock request/reply</text>
+  </g>
+
+  <!-- Fan-out connectors to 3 daemons -->
+  <g stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6" fill="none">
+    <line x1="450" y1="410" x2="450" y2="430"/>
+    <line x1="450" y1="430" x2="155" y2="452"/>
+    <line x1="450" y1="430" x2="450" y2="452"/>
+    <line x1="450" y1="430" x2="745" y2="452"/>
+  </g>
+
+  <!-- ═══ DAEMON ROW ════════════════════════════════════ -->
+  <!-- Daemon 1 -->
+  <g transform="translate(32, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node 1</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon 2 -->
+  <g transform="translate(327, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node 2</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon N -->
+  <g transform="translate(622, 452)">
+    <rect width="246" height="124" rx="8" fill="url(#daemon-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.7"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">Spinifex Daemon</text>
+    <text x="226" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="end">Node N</text>
+    <line x1="16" y1="32" x2="230" y2="32" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+    <g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
+      <text x="20" y="52">● NATS subscriber</text>
+      <text x="20" y="70">● Resource manager</text>
+      <text x="20" y="88">● QEMU/KVM launcher</text>
+      <text x="20" y="106">● QMP monitoring</text>
+    </g>
+    <text x="20" y="119" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a">daemon.go</text>
+  </g>
+
+  <!-- Daemon → VM connectors -->
+  <g stroke="#71717a" stroke-width="0.8" stroke-dasharray="2,3" opacity="0.55" fill="none">
+    <line x1="155" y1="576" x2="155" y2="610"/>
+    <line x1="450" y1="576" x2="450" y2="610"/>
+    <line x1="745" y1="576" x2="745" y2="610"/>
+  </g>
+
+  <!-- ═══ VM BAR + ROW ═══════════════════════════════════ -->
+  <rect x="16" y="610" width="868" height="22" rx="5" fill="url(#bar-grad)"/>
+  <text x="450" y="625" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle" letter-spacing="1.5">WORKLOADS · QEMU/KVM virtual machines per node</text>
+
+  <!-- VM boxes -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <rect x="32" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="155" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="155" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="155" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node 1 workloads</text>
+
+    <rect x="327" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="450" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="450" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="450" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node 2 workloads</text>
+
+    <rect x="622" y="642" width="246" height="60" rx="7" fill="#27272a" fill-opacity="0.7" stroke="#71717a" stroke-width="1" stroke-opacity="0.6"/>
+    <text x="745" y="665" fill="#e4e4e7" text-anchor="middle" font-weight="600">QEMU / KVM VMs</text>
+    <text x="745" y="683" fill="#a1a1aa" text-anchor="middle" font-size="9.5">virtio-blk · virtio-net · cloud-init</text>
+    <text x="745" y="695" fill="#71717a" text-anchor="middle" font-size="9">Node N workloads</text>
+  </g>
+</svg>

.github/assets/diagrams/tf-bastion.svg

@@ -0,0 +1,46 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="180" viewBox="0 0 900 180">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="180" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="28" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">BASTION · only entry point into a private subnet</text>
+
+  <!-- WAN -->
+  <g transform="translate(40, 56)">
+    <rect width="200" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">WAN</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">operator / laptop</text>
+  </g>
+
+  <!-- Arrow: SSH -->
+  <line x1="240" y1="86" x2="320" y2="86" stroke="#58a6ff" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="316,82 328,86 316,90" fill="#58a6ff" opacity="0.8"/>
+  <rect x="244" y="62" width="80" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="284" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">SSH</text>
+
+  <!-- Bastion -->
+  <g transform="translate(338, 56)">
+    <rect width="220" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="110" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Bastion</text>
+    <text x="110" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">public subnet</text>
+  </g>
+
+  <!-- Arrow: SSH (private IP) -->
+  <line x1="558" y1="86" x2="638" y2="86" stroke="#a855f7" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="634,82 646,86 634,90" fill="#a855f7" opacity="0.8"/>
+  <rect x="552" y="62" width="140" height="18" rx="9" fill="#0a0a0a" stroke="#a855f7" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="622" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#c084fc" text-anchor="middle">SSH (private IP)</text>
+
+  <!-- App server -->
+  <g transform="translate(656, 56)">
+    <rect width="220" height="60" rx="8" fill="#a855f71a" stroke="#a855f7" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="110" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">App Server</text>
+    <text x="110" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">private subnet · no internet</text>
+  </g>
+
+  <text x="450" y="156" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" text-anchor="middle" font-style="italic">App server has no route to the internet — all access hops through the bastion.</text>
+</svg>

.github/assets/diagrams/tf-nginx-alb.svg

@@ -0,0 +1,98 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="480" viewBox="0 0 900 480">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="priv-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f71f"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="480" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · VPC WITH PUBLIC AND PRIVATE SUBNETS</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · multi-AZ</text>
+
+  <!-- Internet -->
+  <rect x="350" y="54" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to IGW -->
+  <line x1="450" y1="94" x2="450" y2="116" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,112 454,112 450,120" fill="#71717a" opacity="0.7"/>
+
+  <!-- IGW -->
+  <rect x="380" y="122" width="140" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="450" y="142" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">IGW</text>
+  <text x="450" y="158" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">nginx-alb-igw</text>
+
+  <!-- Fan-out to two public subnets -->
+  <line x1="450" y1="166" x2="450" y2="184" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="230" y1="184" x2="670" y2="184" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="230" y1="184" x2="230" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="670" y1="184" x2="670" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <!-- public_a -->
+  <g transform="translate(108, 202)">
+    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_a</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+
+    <rect x="20" y="70" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="85" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">NAT GW</text>
+  </g>
+
+  <!-- public_b -->
+  <g transform="translate(548, 202)">
+    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_b</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="56" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="71" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+  </g>
+
+  <!-- SNAT arrow from public_a NAT → private row -->
+  <line x1="230" y1="306" x2="230" y2="340" stroke="#F87171" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.65"/>
+  <polygon points="226,336 234,336 230,344" fill="#F87171" opacity="0.8"/>
+  <rect x="256" y="314" width="240" height="20" rx="10" fill="#0a0a0a" stroke="#F87171" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="376" y="328" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">SNAT for private subnets (egress)</text>
+
+  <!-- private_a -->
+  <g transform="translate(108, 350)">
+    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_a</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_1</text>
+    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
+  </g>
+
+  <!-- private_b -->
+  <g transform="translate(548, 350)">
+    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_b</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_2</text>
+    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
+  </g>
+</svg>

.github/assets/diagrams/tf-s3-webapp.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="180" viewBox="0 0 900 180">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="180" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="28" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">S3 WEBAPP · Flask on EC2 with Predastore backend</text>
+
+  <!-- Browser -->
+  <g transform="translate(40, 56)">
+    <rect width="200" height="60" rx="8" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Browser</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">end user</text>
+  </g>
+
+  <!-- Arrow: HTTP -->
+  <line x1="240" y1="86" x2="320" y2="86" stroke="#58a6ff" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="316,82 328,86 316,90" fill="#58a6ff" opacity="0.8"/>
+  <rect x="244" y="62" width="80" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="284" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">HTTP</text>
+
+  <!-- EC2 Instance -->
+  <g transform="translate(338, 56)">
+    <rect width="240" height="60" rx="8" fill="#DC26261a" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.65"/>
+    <text x="120" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" text-anchor="middle" font-weight="700">EC2 Instance</text>
+    <text x="120" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">Flask webapp · port 80</text>
+  </g>
+
+  <!-- Arrow: S3 API (boto3) -->
+  <line x1="578" y1="86" x2="658" y2="86" stroke="#a855f7" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+  <polygon points="654,82 666,86 654,90" fill="#a855f7" opacity="0.8"/>
+  <rect x="570" y="62" width="144" height="18" rx="9" fill="#0a0a0a" stroke="#a855f7" stroke-width="0.6" stroke-opacity="0.55"/>
+  <text x="642" y="75" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#c084fc" text-anchor="middle">S3 API (boto3)</text>
+
+  <!-- Predastore -->
+  <g transform="translate(676, 56)">
+    <rect width="200" height="60" rx="8" fill="#a855f71a" stroke="#a855f7" stroke-width="1.3" stroke-opacity="0.6"/>
+    <text x="100" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">Predastore</text>
+    <text x="100" y="46" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">S3 store · port 8443</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-config-layers.svg

@@ -0,0 +1,78 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="330" viewBox="0 0 900 330">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="cluster-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pool-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D08"/>
+    </linearGradient>
+    <linearGradient id="node-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="330" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">CONFIGURATION LEVELS · spinifex.toml</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">three scopes: cluster · pool · node</text>
+
+  <!-- File badge -->
+  <rect x="24" y="50" width="140" height="24" rx="6" fill="#161b22" stroke="#52525b" stroke-width="0.6"/>
+  <text x="94" y="67" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#e4e4e7" text-anchor="middle">spinifex.toml</text>
+
+  <!-- spine -->
+  <line x1="52" y1="74" x2="52" y2="298" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+
+  <!-- ═══ Level 1: [network] cluster-wide ═══════════════════ -->
+  <line x1="52" y1="108" x2="92" y2="108" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="86" width="764" height="56" rx="8" fill="url(#cluster-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="108" y="108" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">[network]</text>
+  <text x="220" y="108" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">Cluster-wide · which mode?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="128" fill="#a1a1aa">└─</text>
+    <text x="128" y="128" fill="#7ee787">external_mode</text>
+    <text x="224" y="128" fill="#a1a1aa">=</text>
+    <text x="238" y="128" fill="#f5f5f5">"pool"</text>
+  </g>
+
+  <!-- ═══ Level 2: [[network.external_pools]] ═══════════════ -->
+  <line x1="52" y1="176" x2="92" y2="176" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="154" width="764" height="78" rx="8" fill="url(#pool-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="108" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700">[[network.external_pools]]</text>
+  <text x="328" y="176" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">IP ranges · what IPs to use?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="196" fill="#a1a1aa">├─</text>
+    <text x="128" y="196" fill="#FCA5A5">name</text>
+    <text x="158" y="196" fill="#71717a">·</text>
+    <text x="166" y="196" fill="#FCA5A5">range_start</text>
+    <text x="246" y="196" fill="#71717a">·</text>
+    <text x="254" y="196" fill="#FCA5A5">range_end</text>
+
+    <text x="108" y="212" fill="#a1a1aa">├─</text>
+    <text x="128" y="212" fill="#FCA5A5">gateway</text>
+    <text x="182" y="212" fill="#71717a">·</text>
+    <text x="190" y="212" fill="#FCA5A5">prefix_len</text>
+
+    <text x="108" y="228" fill="#a1a1aa">└─</text>
+    <text x="128" y="228" fill="#FCA5A5">region</text>
+    <text x="174" y="228" fill="#71717a">·</text>
+    <text x="182" y="228" fill="#FCA5A5">az</text>
+    <text x="202" y="228" fill="#71717a">(optional)</text>
+  </g>
+
+  <!-- ═══ Level 3: [nodes.NAME.vpcd] ════════════════════════ -->
+  <line x1="52" y1="268" x2="92" y2="268" stroke="#52525b" stroke-width="1" opacity="0.6"/>
+  <rect x="92" y="246" width="764" height="56" rx="8" fill="url(#node-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+  <text x="108" y="268" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" font-weight="700">[nodes.NAME.vpcd]</text>
+  <text x="266" y="268" font-family="'Segoe UI', sans-serif" font-size="11" fill="#f5f5f5">Per-node · which NIC?</text>
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10">
+    <text x="108" y="288" fill="#a1a1aa">└─</text>
+    <text x="128" y="288" fill="#8ab4f8">external_interface</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-datapath-localnet.svg

@@ -0,0 +1,63 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="110" viewBox="0 0 900 110">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="110" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="22" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">DATA PATH · VM to WAN via OVN localnet</text>
+
+  <!-- VM TAP -->
+  <g transform="translate(24, 40)">
+    <rect width="118" height="48" rx="7" fill="#a855f71a" stroke="#a855f7" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="59" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" text-anchor="middle" font-weight="700">VM TAP</text>
+    <text x="59" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">guest interface</text>
+  </g>
+  <polygon points="148,60 160,64 148,68" fill="#71717a" opacity="0.7"/>
+  <line x1="142" y1="64" x2="148" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-int -->
+  <g transform="translate(166, 40)">
+    <rect width="102" height="48" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="51" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" text-anchor="middle" font-weight="700">br-int</text>
+    <text x="51" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS bridge</text>
+  </g>
+  <polygon points="274,60 286,64 274,68" fill="#71717a" opacity="0.7"/>
+  <line x1="268" y1="64" x2="274" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- OVN logical pipeline -->
+  <g transform="translate(292, 40)">
+    <rect width="192" height="48" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="96" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">OVN logical pipeline</text>
+    <text x="96" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">switch → router → SNAT/DNAT</text>
+  </g>
+  <polygon points="490,60 502,64 490,68" fill="#71717a" opacity="0.7"/>
+  <line x1="484" y1="64" x2="490" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- localnet -->
+  <g transform="translate(508, 40)">
+    <rect width="120" height="48" rx="7" fill="#DC26260d" stroke="#DC2626" stroke-width="1" stroke-opacity="0.5" stroke-dasharray="4,3"/>
+    <text x="60" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">localnet</text>
+    <text x="60" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">OVN port type</text>
+  </g>
+  <polygon points="634,60 646,64 634,68" fill="#71717a" opacity="0.7"/>
+  <line x1="628" y1="64" x2="634" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-wan -->
+  <g transform="translate(652, 40)">
+    <rect width="110" height="48" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="55" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+    <text x="55" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS bridge</text>
+  </g>
+  <polygon points="768,60 780,64 768,68" fill="#71717a" opacity="0.7"/>
+  <line x1="762" y1="64" x2="768" y2="64" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- WAN -->
+  <g transform="translate(786, 40)">
+    <rect width="90" height="48" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="45" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" text-anchor="middle" font-weight="700">WAN</text>
+    <text x="45" y="38" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">uplink</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-dhcp-conversations.svg

@@ -0,0 +1,74 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="360" viewBox="0 0 900 360">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="host-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+    <linearGradient id="vm-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="360" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">TWO DHCP CONVERSATIONS · completely independent</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">pool mode · static or dhcp source</text>
+
+  <!-- ═══ HOST ↔ ROUTER DHCP ══════════════════════════════ -->
+  <rect x="24" y="54" width="852" height="138" rx="10" fill="url(#host-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+  <text x="44" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" font-weight="700" letter-spacing="0.8">HOST ↔ ROUTER DHCP</text>
+  <text x="876" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="end">on br-ext · veth pair → br-wan</text>
+  <line x1="24" y1="88" x2="876" y2="88" stroke="#58a6ff" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <!-- step 1 -->
+    <circle cx="54" cy="108" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="112" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">1</text>
+    <text x="74" y="112" fill="#f5f5f5">Spinifex asks router:</text>
+    <text x="228" y="112" fill="#a1a1aa">"give me a public IP for this VM"</text>
+
+    <!-- step 2 -->
+    <circle cx="54" cy="132" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="136" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">2</text>
+    <text x="74" y="136" fill="#f5f5f5">Router responds:</text>
+    <text x="196" y="136" fill="#8ab4f8" font-weight="600">192.168.1.75</text>
+
+    <!-- step 3 -->
+    <circle cx="54" cy="156" r="9" fill="#58a6ff" fill-opacity="0.15" stroke="#58a6ff" stroke-width="1"/>
+    <text x="54" y="160" fill="#58a6ff" text-anchor="middle" font-weight="700" font-size="10">3</text>
+    <text x="74" y="160" fill="#f5f5f5">Spinifex stores IP, tells OVN:</text>
+    <text x="274" y="160" fill="#F87171" font-weight="600">dnat_and_snat(192.168.1.75 ↔ 172.31.0.4)</text>
+  </g>
+
+  <text x="44" y="181" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" font-style="italic">Only used in DHCP source mode. Static mode picks from range.</text>
+
+  <!-- separator -->
+  <line x1="24" y1="206" x2="876" y2="206" stroke="#27272a" stroke-width="0.8"/>
+  <text x="450" y="220" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#52525b" text-anchor="middle" letter-spacing="2">INDEPENDENT · no packets cross between</text>
+
+  <!-- ═══ VM ↔ OVN DHCP ═══════════════════════════════════ -->
+  <rect x="24" y="234" width="852" height="110" rx="10" fill="url(#vm-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="44" y="258" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700" letter-spacing="0.8">VM ↔ OVN DHCP</text>
+  <text x="876" y="258" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#7ee787" text-anchor="end">on tap → br-int → OVN logical switch</text>
+  <line x1="24" y1="268" x2="876" y2="268" stroke="#3fb950" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <circle cx="54" cy="288" r="9" fill="#3fb950" fill-opacity="0.15" stroke="#3fb950" stroke-width="1"/>
+    <text x="54" y="292" fill="#3fb950" text-anchor="middle" font-weight="700" font-size="10">1</text>
+    <text x="74" y="292" fill="#f5f5f5">cloud-init asks:</text>
+    <text x="190" y="292" fill="#a1a1aa">"what's my IP?"</text>
+
+    <circle cx="54" cy="312" r="9" fill="#3fb950" fill-opacity="0.15" stroke="#3fb950" stroke-width="1"/>
+    <text x="54" y="316" fill="#3fb950" text-anchor="middle" font-weight="700" font-size="10">2</text>
+    <text x="74" y="316" fill="#f5f5f5">OVN DHCP responds:</text>
+    <text x="226" y="316" fill="#a1a1aa">"you are</text>
+    <text x="284" y="316" fill="#7ee787" font-weight="600">172.31.0.4</text>
+    <text x="356" y="316" fill="#a1a1aa">"</text>
+  </g>
+
+  <text x="44" y="336" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a" font-style="italic">VM never sees 192.168.1.x or the router.</text>
+</svg>

.github/assets/diagrams/vpc-direct-bridge.svg

@@ -0,0 +1,44 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="170" viewBox="0 0 900 170">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="170" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">DIRECT BRIDGE MODE · dedicated WAN NIC</text>
+
+  <!-- Row 1: Management NIC (independent, no WAN bridge path) -->
+  <g transform="translate(24, 42)">
+    <rect width="852" height="44" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+    <text x="20" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">Management NIC</text>
+    <text x="20" y="36" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa">host IP · SSH · spinifex services · overlay (Geneve)</text>
+    <text x="840" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#7ee787" text-anchor="end" font-weight="600">independent path</text>
+  </g>
+
+  <!-- Row 2: WAN NIC → br-wan → localnet -->
+  <g transform="translate(24, 100)">
+    <rect width="160" height="44" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="80" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">WAN NIC</text>
+    <text x="80" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">dedicated physical</text>
+  </g>
+  <polygon points="198,118 210,122 198,126" fill="#71717a" opacity="0.7"/>
+  <line x1="186" y1="122" x2="198" y2="122" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(214, 100)">
+    <rect width="220" height="44" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="110" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+    <text x="110" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">OVS port (NIC added directly)</text>
+  </g>
+  <polygon points="448,118 460,122 448,126" fill="#71717a" opacity="0.7"/>
+  <line x1="434" y1="122" x2="448" y2="122" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(464, 100)">
+    <rect width="180" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">OVN localnet</text>
+    <text x="90" y="36" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="middle">external switch port</text>
+  </g>
+
+  <text x="670" y="130" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#F87171" font-weight="600">distributed NAT · any MAC · all frames</text>
+</svg>

.github/assets/diagrams/vpc-host-flow.svg

@@ -0,0 +1,91 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="440" viewBox="0 0 900 440">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="host-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#27272a88"/>
+      <stop offset="100%" style="stop-color:#27272a22"/>
+    </linearGradient>
+    <linearGradient id="br-int-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f71f"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+    <linearGradient id="br-wan-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#58a6ff1f"/>
+      <stop offset="100%" style="stop-color:#58a6ff08"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="440" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NETWORK FLOW · bare-metal host</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">br-int overlay · br-wan WAN uplink</text>
+
+  <!-- ═══ Bare-Metal Host container ═══════════════════════ -->
+  <rect x="40" y="60" width="820" height="284" rx="12" fill="url(#host-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.7"/>
+  <text x="58" y="84" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#e4e4e7" font-weight="700" letter-spacing="1">BARE-METAL HOST</text>
+  <line x1="40" y1="96" x2="860" y2="96" stroke="#52525b" stroke-opacity="0.5" stroke-width="0.5"/>
+
+  <!-- br-int bridge (left) -->
+  <g transform="translate(80, 122)">
+    <rect width="260" height="176" rx="10" fill="url(#br-int-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" font-weight="700">br-int</text>
+    <text x="240" y="24" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="end">(overlay)</text>
+    <line x1="20" y1="34" x2="240" y2="34" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+      <rect x="30" y="52" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="70" fill="#d8b4fe" text-anchor="middle">tap-eni1</text>
+
+      <rect x="30" y="88" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="106" fill="#d8b4fe" text-anchor="middle">tap-eni2</text>
+
+      <rect x="30" y="124" width="200" height="28" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.4" stroke-width="0.6"/>
+      <text x="130" y="142" fill="#d8b4fe" text-anchor="middle">tap-eni3</text>
+    </g>
+  </g>
+
+  <!-- OVN NAT pipeline arrow in middle -->
+  <g transform="translate(352, 200)">
+    <line x1="0" y1="10" x2="148" y2="10" stroke="#DC2626" stroke-width="1.4" stroke-dasharray="4,3" opacity="0.75"/>
+    <polygon points="148,5 160,10 148,15" fill="#DC2626" opacity="0.8"/>
+    <text x="78" y="0" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">OVN</text>
+    <text x="78" y="28" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">NAT pipeline</text>
+  </g>
+
+  <!-- br-wan bridge (right) -->
+  <g transform="translate(520, 122)">
+    <rect width="260" height="176" rx="10" fill="url(#br-wan-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" font-weight="700">br-wan</text>
+    <text x="240" y="24" font-family="'Segoe UI', sans-serif" font-size="10" fill="#8ab4f8" text-anchor="end">(WAN)</text>
+    <line x1="20" y1="34" x2="240" y2="34" stroke="#58a6ff" stroke-opacity="0.3" stroke-width="0.5"/>
+
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+      <rect x="30" y="52" width="200" height="60" rx="5" fill="#58a6ff1a" stroke="#58a6ff" stroke-opacity="0.45" stroke-width="0.8"/>
+      <text x="130" y="74" fill="#8ab4f8" text-anchor="middle" font-weight="600">spx-ext-{nic}</text>
+      <text x="130" y="92" fill="#a1a1aa" text-anchor="middle" font-size="10">(macvlan)</text>
+    </g>
+  </g>
+
+  <!-- macvlan bridge mode label -->
+  <text x="650" y="314" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="middle" font-style="italic">macvlan bridge mode</text>
+
+  <!-- WAN NIC inside host, bottom -->
+  <line x1="650" y1="298" x2="650" y2="320" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <g transform="translate(500, 320)">
+    <rect width="300" height="20" rx="5" fill="#161b22" stroke="#52525b" stroke-width="0.6"/>
+    <text x="150" y="14" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">WAN NIC · host IP unchanged</text>
+  </g>
+
+  <!-- connector out -->
+  <line x1="650" y1="344" x2="650" y2="380" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="646,376 654,376 650,384" fill="#71717a" opacity="0.7"/>
+
+  <!-- Physical WAN -->
+  <g transform="translate(500, 388)">
+    <rect width="300" height="40" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+    <text x="150" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" text-anchor="middle" font-weight="700">Physical WAN</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-macvlan.svg

@@ -0,0 +1,57 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="320" viewBox="0 0 900 320">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="320" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">MACVLAN FALLBACK · single-NIC host</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">WAN NIC == SSH NIC · host IP preserved</text>
+
+  <!-- WAN NIC (parent) -->
+  <g transform="translate(280, 58)">
+    <rect width="340" height="54" rx="8" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.3" stroke-opacity="0.65"/>
+    <text x="170" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" text-anchor="middle" font-weight="700">WAN NIC</text>
+    <text x="170" y="42" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">host IP unchanged · parent interface</text>
+  </g>
+
+  <!-- Split point -->
+  <line x1="450" y1="112" x2="450" y2="132" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="260" y1="132" x2="640" y2="132" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="260" y1="132" x2="260" y2="148" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <line x1="640" y1="132" x2="640" y2="148" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- Child 1: host traffic -->
+  <g transform="translate(120, 148)">
+    <rect width="280" height="48" rx="7" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="140" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" text-anchor="middle" font-weight="700">host traffic</text>
+    <text x="140" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">SSH · management · overlay</text>
+  </g>
+
+  <!-- Child 2: macvlan -->
+  <g transform="translate(500, 148)">
+    <rect width="280" height="48" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="140" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">spx-ext-{nic}</text>
+    <text x="140" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5" text-anchor="middle">macvlan · bridge mode · no IP</text>
+  </g>
+
+  <!-- br-wan -->
+  <line x1="640" y1="196" x2="640" y2="216" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="636,212 644,212 640,220" fill="#71717a" opacity="0.7"/>
+
+  <g transform="translate(500, 220)">
+    <rect width="280" height="40" rx="7" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="140" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+  </g>
+
+  <!-- localnet -->
+  <line x1="640" y1="260" x2="640" y2="276" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="636,272 644,272 640,280" fill="#71717a" opacity="0.7"/>
+
+  <g transform="translate(500, 280)">
+    <rect width="280" height="30" rx="6" fill="#DC26260d" stroke="#DC2626" stroke-width="1" stroke-opacity="0.5" stroke-dasharray="4,3"/>
+    <text x="140" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="600">localnet → OVN ext switch</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-network-path.svg

@@ -0,0 +1,131 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="wan-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="ovs-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff22"/>
+      <stop offset="100%" style="stop-color:#58a6ff0a"/>
+    </linearGradient>
+    <linearGradient id="router-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC262622"/>
+      <stop offset="100%" style="stop-color:#7F1D1D0a"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="priv-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f722"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="560" fill="url(#bg)" rx="12"/>
+
+  <!-- Title -->
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NETWORK PATH</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">OVN logical topology · bridge mapping · NAT</text>
+
+  <!-- WAN -->
+  <rect x="300" y="54" width="300" height="40" rx="8" fill="url(#wan-grad)" stroke="#3fb950" stroke-width="1" stroke-opacity="0.55"/>
+  <text x="450" y="78" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Physical Network (WAN)</text>
+
+  <!-- connector -->
+  <line x1="450" y1="94" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+
+  <!-- br-wan bridge -->
+  <rect x="340" y="112" width="220" height="52" rx="8" fill="url(#ovs-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.65"/>
+  <text x="450" y="134" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#58a6ff" text-anchor="middle" font-weight="700">br-wan</text>
+  <text x="450" y="152" font-family="'Segoe UI', sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">OVS bridge · physical uplink</text>
+
+  <!-- connector with bridge-mappings label -->
+  <line x1="450" y1="164" x2="450" y2="186" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <rect x="316" y="170" width="268" height="18" rx="9" fill="#0a0a0a" stroke="#58a6ff" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="450" y="183" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#8ab4f8" text-anchor="middle">ovn-bridge-mappings: external:br-wan</text>
+
+  <!-- VPC Logical Router (the big box) -->
+  <rect x="60" y="208" width="780" height="196" rx="10" fill="url(#router-grad)" stroke="#DC2626" stroke-width="1.4" stroke-opacity="0.75"/>
+  <text x="78" y="232" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#F87171" font-weight="700" letter-spacing="1">VPC LOGICAL ROUTER</text>
+  <line x1="60" y1="242" x2="840" y2="242" stroke="#DC2626" stroke-opacity="0.25" stroke-width="0.5"/>
+
+  <!-- Router ports row -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="11">
+    <rect x="82" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="142" y="270" fill="#F87171" text-anchor="middle" font-weight="600">Gateway port</text>
+    <text x="142" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">public IP</text>
+
+    <text x="215" y="282" fill="#71717a">─────</text>
+
+    <rect x="260" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="320" y="270" fill="#F87171" text-anchor="middle" font-weight="600">ext switch</text>
+    <text x="320" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">OVN logical</text>
+
+    <text x="393" y="282" fill="#71717a">─────</text>
+
+    <rect x="438" y="254" width="120" height="42" rx="5" fill="#0d1117" stroke="#F87171" stroke-opacity="0.6" stroke-width="0.8"/>
+    <text x="498" y="270" fill="#F87171" text-anchor="middle" font-weight="600">localnet port</text>
+    <text x="498" y="286" fill="#a1a1aa" text-anchor="middle" font-size="9.5">→ br-wan</text>
+  </g>
+
+  <!-- NAT rules + Routes, two columns on right -->
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <text x="598" y="260" fill="#F87171" font-weight="600">NAT rules</text>
+    <text x="598" y="276" fill="#f5f5f5">SNAT:</text>
+    <text x="644" y="276" fill="#a1a1aa">VPC CIDR → gw IP</text>
+    <text x="598" y="292" fill="#f5f5f5">dnat_and_snat:</text>
+    <text x="704" y="292" fill="#a1a1aa">pub ↔ priv</text>
+  </g>
+
+  <g font-family="'SF Mono', ui-monospace, monospace" font-size="10.5">
+    <text x="82" y="330" fill="#F87171" font-weight="600">Routes</text>
+    <text x="82" y="346" fill="#f5f5f5">0.0.0.0/0</text>
+    <text x="154" y="346" fill="#a1a1aa">→ WAN gateway</text>
+  </g>
+
+  <!-- Subnet branches inside router -->
+  <line x1="150" y1="362" x2="150" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="150" y1="386" x2="305" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="305" y1="386" x2="305" y2="404" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <line x1="150" y1="386" x2="515" y2="386" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+  <line x1="515" y1="386" x2="515" y2="404" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
+
+  <!-- Subnet A (public) -->
+  <g transform="translate(215, 404)">
+    <rect width="180" height="124" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">subnet-a</text>
+    <text x="90" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">(public)</text>
+    <line x1="16" y1="46" x2="164" y2="46" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="90" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#f5f5f5" text-anchor="middle">10.0.1.0/24</text>
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa">
+      <circle cx="40" cy="88" r="2.5" fill="#3fb950"/>
+      <text x="52" y="92">eni-1</text>
+      <circle cx="40" cy="108" r="2.5" fill="#3fb950"/>
+      <text x="52" y="112">eni-2</text>
+    </g>
+  </g>
+
+  <!-- Subnet B (private) -->
+  <g transform="translate(425, 404)">
+    <rect width="180" height="124" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#c084fc" text-anchor="middle" font-weight="700">subnet-b</text>
+    <text x="90" y="38" font-family="'Segoe UI', sans-serif" font-size="10" fill="#d8b4fe" text-anchor="middle">(private)</text>
+    <line x1="16" y1="46" x2="164" y2="46" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="90" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#f5f5f5" text-anchor="middle">10.0.2.0/24</text>
+    <g font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa">
+      <circle cx="40" cy="88" r="2.5" fill="#a855f7"/>
+      <text x="52" y="92">eni-3</text>
+      <circle cx="40" cy="108" r="2.5" fill="#a855f7"/>
+      <text x="52" y="112">eni-4</text>
+    </g>
+  </g>
+
+  <!-- Footnote -->
+  <text x="24" y="548" font-family="'Segoe UI', sans-serif" font-size="10" fill="#71717a">Cross-host traffic uses Geneve tunnels (UDP 6081) over the overlay NIC · ovn-controller programs OpenFlow rules on br-int</text>
+</svg>

.github/assets/diagrams/vpc-private-subnet-flow.svg

@@ -0,0 +1,56 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="110" viewBox="0 0 900 110">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="110" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">PRIVATE SUBNET · no default route</text>
+
+  <!-- Instance -->
+  <g transform="translate(40, 42)">
+    <rect width="180" height="44" rx="7" fill="#a855f71a" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" text-anchor="middle" font-weight="700">Instance</text>
+    <text x="90" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">10.0.2.5</text>
+  </g>
+
+  <!-- arrow -->
+  <g stroke="#DC2626" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="220" y1="64" x2="260" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="260,60 268,64 260,68" fill="#DC2626" opacity="0.6"/>
+
+  <!-- Router -->
+  <g transform="translate(275, 42)">
+    <rect width="180" height="44" rx="7" fill="#DC26261a" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="90" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" text-anchor="middle" font-weight="700">VPC Router</text>
+    <text x="90" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">lookup 0.0.0.0/0</text>
+  </g>
+
+  <!-- arrow -->
+  <g stroke="#DC2626" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="455" y1="64" x2="495" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="495,60 503,64 495,68" fill="#DC2626" opacity="0.6"/>
+
+  <!-- [no default route] -->
+  <g transform="translate(510, 42)">
+    <rect width="196" height="44" rx="7" fill="#0a0a0a" stroke="#71717a" stroke-width="1" stroke-opacity="0.7" stroke-dasharray="4,3"/>
+    <text x="98" y="20" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" text-anchor="middle" font-weight="600">[no default route]</text>
+    <text x="98" y="34" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="middle">no IGW attached</text>
+  </g>
+
+  <!-- arrow to dropped -->
+  <g stroke="#71717a" stroke-width="1.2" opacity="0.6" fill="none">
+    <line x1="706" y1="64" x2="746" y2="64" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="746,60 754,64 746,68" fill="#71717a" opacity="0.6"/>
+
+  <!-- dropped -->
+  <g transform="translate(760, 42)">
+    <rect width="116" height="44" rx="7" fill="#27272a" stroke="#52525b" stroke-width="1" stroke-opacity="0.7"/>
+    <text x="58" y="27" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" text-anchor="middle" font-weight="600">dropped</text>
+  </g>
+</svg>

.github/assets/diagrams/vpc-public-subnet-flow.svg

@@ -0,0 +1,74 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="170" viewBox="0 0 900 170">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="170" fill="url(#bg)" rx="10"/>
+
+  <text x="24" y="24" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" letter-spacing="2" font-weight="600">PUBLIC SUBNET · 1:1 NAT managed by OVN</text>
+
+  <!-- OUTBOUND row -->
+  <text x="24" y="52" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" font-weight="700">Outbound</text>
+
+  <g transform="translate(112, 38)">
+    <rect width="180" height="40" rx="6" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="90" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#7ee787" text-anchor="middle" font-weight="700">Instance</text>
+    <text x="90" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">10.0.1.5</text>
+  </g>
+
+  <g stroke="#3fb950" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="292" y1="58" x2="352" y2="58" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="352,54 360,58 352,62" fill="#3fb950" opacity="0.7"/>
+
+  <g transform="translate(368, 38)">
+    <rect width="130" height="40" rx="6" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="65" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">SNAT</text>
+  </g>
+
+  <g stroke="#3fb950" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="498" y1="58" x2="558" y2="58" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="558,54 566,58 558,62" fill="#3fb950" opacity="0.7"/>
+
+  <g transform="translate(574, 38)">
+    <rect width="272" height="40" rx="6" fill="#3fb9501a" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="136" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#7ee787" text-anchor="middle" font-weight="700">exits as 203.0.113.10</text>
+    <text x="136" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">source IP rewritten</text>
+  </g>
+
+  <!-- INBOUND row -->
+  <text x="24" y="110" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#58a6ff" font-weight="700">Inbound</text>
+
+  <g transform="translate(112, 96)">
+    <rect width="180" height="40" rx="6" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="90" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#8ab4f8" text-anchor="middle" font-weight="700">WAN client</text>
+    <text x="90" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">→ 203.0.113.10</text>
+  </g>
+
+  <g stroke="#58a6ff" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="292" y1="116" x2="352" y2="116" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="352,112 360,116 352,120" fill="#58a6ff" opacity="0.7"/>
+
+  <g transform="translate(368, 96)">
+    <rect width="130" height="40" rx="6" fill="#DC26261a" stroke="#DC2626" stroke-width="1.1" stroke-opacity="0.65"/>
+    <text x="65" y="25" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" text-anchor="middle" font-weight="700">DNAT</text>
+  </g>
+
+  <g stroke="#58a6ff" stroke-width="1.1" opacity="0.6" fill="none">
+    <line x1="498" y1="116" x2="558" y2="116" stroke-dasharray="3,3"/>
+  </g>
+  <polygon points="558,112 566,116 558,120" fill="#58a6ff" opacity="0.7"/>
+
+  <g transform="translate(574, 96)">
+    <rect width="272" height="40" rx="6" fill="#58a6ff1a" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+    <text x="136" y="18" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#8ab4f8" text-anchor="middle" font-weight="700">arrives at 10.0.1.5</text>
+    <text x="136" y="32" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#a1a1aa" text-anchor="middle">dest IP rewritten</text>
+  </g>
+
+  <!-- Footnote -->
+  <text x="24" y="158" font-family="'Segoe UI', sans-serif" font-size="9" fill="#71717a">The instance OS only sees its private IP (10.0.1.5) — the public address is managed entirely by OVN's dnat_and_snat rule.</text>
+</svg>

.github/assets/platform.svg

@@ -0,0 +1,144 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="560" viewBox="0 0 900 560">
+  <defs>
+    <linearGradient id="pf-bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="hw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#52525b;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#52525b;stop-opacity:0.06"/>
+    </linearGradient>
+    <linearGradient id="os-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f7;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#a855f7;stop-opacity:0.05"/>
+    </linearGradient>
+    <linearGradient id="spx-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC2626;stop-opacity:0.28"/>
+      <stop offset="100%" style="stop-color:#7F1D1D;stop-opacity:0.1"/>
+    </linearGradient>
+    <linearGradient id="app-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb950;stop-opacity:0.18"/>
+      <stop offset="100%" style="stop-color:#3fb950;stop-opacity:0.05"/>
+    </linearGradient>
+  </defs>
+
+  <rect width="900" height="560" fill="url(#pf-bg)" rx="12"/>
+
+  <!-- Subtle mesh dots background -->
+  <g opacity="0.12" fill="#7F1D1D">
+    <circle cx="40" cy="40" r="1"/><circle cx="120" cy="30" r="1"/><circle cx="220" cy="50" r="1"/>
+    <circle cx="320" cy="25" r="1"/><circle cx="440" cy="45" r="1"/><circle cx="560" cy="30" r="1"/>
+    <circle cx="680" cy="50" r="1"/><circle cx="780" cy="30" r="1"/><circle cx="860" cy="45" r="1"/>
+    <circle cx="60" cy="530" r="1"/><circle cx="160" cy="520" r="1"/><circle cx="260" cy="540" r="1"/>
+    <circle cx="380" cy="525" r="1"/><circle cx="500" cy="540" r="1"/><circle cx="620" cy="520" r="1"/>
+    <circle cx="740" cy="540" r="1"/><circle cx="840" cy="525" r="1"/>
+  </g>
+
+  <!-- Title -->
+  <text x="450" y="32" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" text-anchor="middle" letter-spacing="2.8">THE MULGA PLATFORM</text>
+  <text x="450" y="58" font-family="'Segoe UI', system-ui, sans-serif" font-size="20" font-weight="600" fill="#f5f5f5" text-anchor="middle">AWS-style workflows, on your own stack</text>
+
+  <!-- Layer 4: Applications (top) -->
+  <g transform="translate(60, 95)">
+    <rect width="780" height="88" rx="10" fill="url(#app-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#3fb950" letter-spacing="1.5" font-weight="600">LAYER 4 · APPLICATIONS &amp; TOOLING</text>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">Your workloads — unmodified AWS tooling</text>
+
+    <g transform="translate(20, 60)" font-family="'Segoe UI', sans-serif" font-size="10.5">
+      <rect x="0" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="35" y="14" fill="#7ee787" text-anchor="middle">aws-cli</text>
+
+      <rect x="78" y="0" width="78" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="117" y="14" fill="#7ee787" text-anchor="middle">Terraform</text>
+
+      <rect x="164" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="208" y="14" fill="#7ee787" text-anchor="middle">AWS SDKs</text>
+
+      <rect x="260" y="0" width="88" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="304" y="14" fill="#7ee787" text-anchor="middle">Kubernetes</text>
+
+      <rect x="356" y="0" width="70" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="391" y="14" fill="#7ee787" text-anchor="middle">Packer</text>
+
+      <rect x="434" y="0" width="98" height="20" rx="10" fill="#3fb950" fill-opacity="0.14" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.5"/>
+      <text x="483" y="14" fill="#7ee787" text-anchor="middle">Your Services</text>
+    </g>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 188)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 3: Spinifex (highlighted) -->
+  <g transform="translate(60, 215)">
+    <rect width="780" height="140" rx="10" fill="url(#spx-grad)" stroke="#DC2626" stroke-width="1.5" stroke-opacity="0.75"/>
+    <text x="20" y="26" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#F87171" letter-spacing="1.5" font-weight="600">LAYER 3 · SPINIFEX · AWS ORCHESTRATION</text>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="15" fill="#f5f5f5" font-weight="500">AWS Gateway → NATS → Service Daemons · SigV4, IAM, VPC-aware</text>
+
+    <g transform="translate(20, 66)" font-family="'Segoe UI', sans-serif">
+      <!-- EC2 -->
+      <rect x="0" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="59" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EC2</text>
+      <text x="59" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">QEMU/KVM VMs</text>
+      <text x="59" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">cloud-init · AMIs</text>
+
+      <!-- EBS -->
+      <rect x="126" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="185" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">EBS</text>
+      <text x="185" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Viperblock</text>
+      <text x="185" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">WAL · NBD · snapshots</text>
+
+      <!-- S3 -->
+      <rect x="252" y="0" width="118" height="58" rx="6" fill="#DC2626" fill-opacity="0.1" stroke="#DC2626" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="311" y="18" font-size="13" fill="#F87171" text-anchor="middle" font-weight="600">S3</text>
+      <text x="311" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">Predastore</text>
+      <text x="311" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Reed–Solomon EC</text>
+
+      <!-- VPC -->
+      <rect x="378" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
+      <text x="437" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">VPC</text>
+      <text x="437" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">OVN / OVS</text>
+      <text x="437" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">subnets · SG · ALB</text>
+
+      <!-- IAM -->
+      <rect x="504" y="0" width="118" height="58" rx="6" fill="#7F1D1D" fill-opacity="0.22" stroke="#F87171" stroke-opacity="0.55" stroke-width="1"/>
+      <text x="563" y="18" font-size="13" fill="#FCA5A5" text-anchor="middle" font-weight="600">IAM</text>
+      <text x="563" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">SigV4 · policies</text>
+      <text x="563" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">users · roles · keys</text>
+
+      <!-- Control -->
+      <rect x="630" y="0" width="110" height="58" rx="6" fill="#27272a" fill-opacity="0.8" stroke="#71717a" stroke-opacity="0.6" stroke-width="1"/>
+      <text x="685" y="18" font-size="13" fill="#e4e4e7" text-anchor="middle" font-weight="600">Control</text>
+      <text x="685" y="34" font-size="9.5" fill="#f5f5f5" text-anchor="middle">NATS bus</text>
+      <text x="685" y="48" font-size="9.5" fill="#a1a1aa" text-anchor="middle">spx · daemons</text>
+    </g>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 360)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 2: OS -->
+  <g transform="translate(60, 387)">
+    <rect width="780" height="62" rx="10" fill="url(#os-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.5"/>
+    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" letter-spacing="1.5" font-weight="600">LAYER 2 · OPERATING SYSTEM</text>
+    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Debian / Ubuntu LTS · systemd · KVM · Linux bridge · headless autoinstall</text>
+  </g>
+
+  <!-- Down arrow -->
+  <g transform="translate(450, 454)" opacity="0.45">
+    <line x1="0" y1="0" x2="0" y2="14" stroke="#a1a1aa" stroke-width="1"/>
+    <polygon points="-4,12 4,12 0,18" fill="#a1a1aa"/>
+  </g>
+
+  <!-- Layer 1: Hardware -->
+  <g transform="translate(60, 481)">
+    <rect width="780" height="62" rx="10" fill="url(#hw-grad)" stroke="#71717a" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="22" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#a1a1aa" letter-spacing="1.5" font-weight="600">LAYER 1 · HARDWARE</text>
+    <text x="20" y="44" font-family="'Segoe UI', sans-serif" font-size="14" fill="#f5f5f5">Single node · multi-node cluster · edge appliance · air-gapped rack · commodity x86 / ARM</text>
+  </g>
+</svg>

.github/workflows/e2e.yml

@@ -27,7 +27,7 @@ jobs:
   e2e_single:
     name: Single-Node E2E
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 25
+    timeout-minutes: 30
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
     steps:
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -48,9 +48,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - name: Clone and build
+      - name: Build install artifacts
         working-directory: mulga/scripts/tofu-cluster
-        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
@@ -62,13 +66,14 @@ jobs:
           ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
           tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-      - name: Bootstrap Cluster
+      - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          SPX_PREBUILT=true \
-          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
           SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
-          ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1
+          ./bootstrap-install.sh env3 \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run E2E Tests
         working-directory: mulga/scripts/tofu-cluster
@@ -78,7 +83,7 @@ jobs:
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
           SSH_KEY="${SSH_KEY/#\~/$HOME}"
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh && ./tests/e2e/run-cert-e2e.sh'
+            'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh  && ./run-lb-e2e.sh'
 
       - name: Teardown
         if: always()
@@ -101,7 +106,7 @@ jobs:
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -154,7 +159,7 @@ jobs:
   e2e_real_multi:
     name: Real Multi-Node E2E
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 25
+    timeout-minutes: 35
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
     steps:
@@ -164,7 +169,7 @@ jobs:
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
-          app-id: ${{ secrets.APP_ID }}
+          client-id: ${{ secrets.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
           repositories: mulga
 
@@ -175,9 +180,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
-      - name: Clone and build
+      - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
-        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
@@ -189,13 +198,14 @@ jobs:
           ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
           tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
 
-      - name: Bootstrap Cluster
+      - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
         run: |
-          SPX_PREBUILT=true \
-          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
           SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
-          ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1
+          ./bootstrap-install.sh env2 \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run E2E Tests
         working-directory: mulga/scripts/tofu-cluster
@@ -212,8 +222,10 @@ jobs:
             NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
           done
 
+          NODE2_IP=$(echo "$INVENTORY" | jq -r '.nodes[1].wan_ip')
+
           ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}"
+            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
 
       - name: Teardown
         if: always()

.github/workflows/release.yml

@@ -98,7 +98,7 @@ jobs:
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
         with:
           draft: true
           files: |
@@ -161,28 +161,18 @@ jobs:
           TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
           ./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
 
-      - name: Run Smoke Test
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          INVENTORY=$(tofu output -json inventory)
-          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-            -i "$SSH_KEY" \
-            smoke-test-install.sh \
-            "tf-user@${WAN_IP}:/tmp/smoke-test.sh"
-          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
-            -i "$SSH_KEY" "tf-user@${WAN_IP}" \
-            'bash /tmp/smoke-test.sh'
-
       - name: Promote Release
         if: success()
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release edit "$TAG" --repo "$REPO" --draft=false
-          echo "Release ${TAG} promoted to latest"
+          if [[ "$TAG" =~ -(dev|rc|alpha|beta) ]]; then
+            gh release edit "$TAG" --repo "$REPO" --draft=false --prerelease
+            echo "Release ${TAG} promoted as prerelease"
+          else
+            gh release edit "$TAG" --repo "$REPO" --draft=false
+            echo "Release ${TAG} promoted to latest"
+          fi
 
       - name: Mark Release Failed
         if: failure()

.github/workflows/ui.yml

@@ -30,7 +30,7 @@ jobs:
           sparse-checkout: spinifex/services/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v5
+        uses: pnpm/action-setup@v6
         with:
           package_json_file: spinifex/services/spinifexui/frontend/package.json
 

.gitignore

@@ -7,6 +7,9 @@
 !example.tfvars
 scripts/iac/proxmox/clusters/
 
+# Generated SSH private keys from terraform examples (e.g. local_file.nginx_pem)
+*.pem
+
 # Go workspace
 go.work
 go.work.sum

.golangci.yml

@@ -22,6 +22,7 @@ linters:
     - errname # checks that sentinel errors are prefixed with Err, error types suffixed with Error
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
+    - forcetypeassert # flags unchecked type assertions that can panic
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - goprintffuncname # checks that printf-like functions are named with f at the end
     - gosec # inspects source code for security problems
@@ -29,11 +30,14 @@ linters:
     - ineffassign # detects assignments to existing variables that are not used
     - makezero # finds slice declarations with non-zero initial length followed by append
     - mirror # catches wrong mirror patterns in bytes/strings usage
+    - musttag # checks structs passed to marshal/unmarshal have proper tags
     - modernize # modernizes code to use newer Go idioms (go fix analyzers)
     - nakedret # finds naked returns in functions
     - nilerr # catches code that returns nil even after checking err != nil
     - nilnesserr # catches returning a different nil error variable after err != nil check
     - nolintlint # reports ill-formed or insufficient nolint directives
+    - nosprintfhostport # flags fmt.Sprintf host:port — use net.JoinHostPort for IPv6 safety
+    - perfsprint # detects fmt.Sprintf calls replaceable with faster alternatives
     - predeclared # catches shadowing of Go builtins (len, cap, error, etc.)
     - reassign # checks that package variables are not reassigned
     - recvcheck # checks for receiver type consistency (pointer vs value)
@@ -91,6 +95,12 @@ linters:
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
+    perfsprint:
+      concat-loop: false
+      err-error: false # don't flag err.Error()
+      errorf: false # don't convert fmt.Errorf to errors.New
+      strconcat: false
+
     nakedret:
       max-func-lines: 0
 
@@ -124,5 +134,3 @@ linters:
         linters: [gosec] # subprocess execution — legitimate in VM orchestration
       - text: "G304"
         linters: [gosec] # file path from variable — necessary for config loading
-      - text: "G402"
-        linters: [gosec] # TLS config — we handle TLS ourselves

Makefile

@@ -63,18 +63,18 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
-build-alb-agent:
-	@echo -e "\n....Building alb-agent (static)"
-	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/alb-agent cmd/alb-agent/main.go
+build-lb-agent:
+	@echo -e "\n....Building lb-agent (static)"
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
-build-system-image: ## Build a system image from manifest (use IMAGE=alb)
+build-system-image: ## Build a system image from manifest (use IMAGE=lb)
 ifndef IMAGE
-	$(error IMAGE is required. Usage: make build-system-image IMAGE=alb)
+	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
 endif
 	./scripts/build-system-image.sh scripts/images/$(IMAGE).conf
 
-build-alb-image: ## Build ALB Alpine image
-	$(MAKE) build-system-image IMAGE=alb
+build-lb-image: ## Build LB Alpine image
+	$(MAKE) build-system-image IMAGE=lb
 
 go_run:
 	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
@@ -116,6 +116,17 @@ run:
 	$(MAKE) go_build
 	$(MAKE) go_run
 
+# Fast iteration: build + install binary + restart all services
+deploy: build
+	sudo install -m 755 bin/spx /usr/local/bin/spx
+	sudo systemctl daemon-reload
+	sudo systemctl restart spinifex.target
+
+# Re-run setup.sh after changing systemd units, helper scripts, or logrotate config.
+# Not needed for code-only changes — use deploy for those.
+reinstall:
+	scripts/dev-install.sh
+
 clean:
 	rm -f ./bin/$(GO_PROJECT_NAME)
 	rm -rf spinifex/services/spinifexui/frontend/dist
@@ -132,9 +143,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.1 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.2 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.1.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.2.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -207,7 +218,8 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-alb-agent build-system-image build-alb-image go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
+.PHONY: build build-ui build-lb-agent build-system-image build-lb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
+	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
 	distro distro-amd64 distro-arm64 distro-clean

README.md

@@ -1,25 +1,55 @@
+<p align="center">
+  <img src=".github/assets/banner.svg" alt="Spinifex — AWS-Compatible Infrastructure Stack" width="900">
+</p>
+
+<p align="center">
+  <a href="https://go.dev/"><img src="https://img.shields.io/badge/Go-1.22+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://github.com/mulgadc/spinifex/actions"><img src="https://img.shields.io/badge/CI-e2e-58a6ff?style=flat-square" alt="CI"></a>
+</p>
+
+<p align="center">
+  <a href="#what-is-spinifex">What is Spinifex?</a> ·
+  <a href="#the-platform">Platform</a> ·
+  <a href="#core-components">Components</a> ·
+  <a href="#architecture-at-a-glance">Architecture</a> ·
+  <a href="#installation">Installation</a> ·
+  <a href="https://docs.mulgadc.com">Docs</a>
+</p>
+
+---
+
 # Spinifex: An Open Source AWS-Compatible Stack for Bare-Metal, Edge, and On-Prem Deployments
 
-(Developer Preview) - Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com/) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
+Spinifex developed by [Mulga Defense Corporation](https://mulgadc.com) is an open source infrastructure platform that brings the core services of AWS—like EC2, VPC, EBS, and S3—to environments where running in the cloud isn't an option. Whether you're deploying to edge sites, private data-centers, or need to operate in low-connectivity or highly contested environments, Spinifex gives you AWS-style workflows on your own hardware.
 
 ## What is Spinifex?
 
 Spinifex replicates essential AWS primitives—virtual machines, block volumes, and object storage—using lightweight, self-contained components that are easy to deploy and integrate.
 
-It’s designed for developers and operators who need:
+It's designed for developers and operators who need:
 
 - Cloud-like infrastructure without the cloud
 - Minimal dependencies, full control
 - Drop-in compatibility with tools like the AWS CLI, SDKs, and Terraform
 - A secure cloud environment you control and own the entire hardware, network and software stack
 
-You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren’t feasible.
+You can run Spinifex on a few servers in a rack, a field site, or anywhere centralized cloud services aren't feasible.
+
+## The Platform
+
+<p align="center">
+  <img src=".github/assets/platform.svg" alt="The Mulga Platform — Hardware to Applications" width="900">
+</p>
+
+From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
 ## Core Components
 
 ### Spinifex (Compute Service – EC2 Alternative)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU’s QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
 
 - EC2-like VM management on bare metal
 - Launches with cloud-init metadata support
@@ -27,7 +57,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 ### Viperblock (Block Storage – EBS Alternative)
 
-[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It’s built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
+[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
 
 - Fast, durable virtual disks
 - Replication for resilience
@@ -43,11 +73,19 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+## Architecture at a Glance
+
+<p align="center">
+  <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture — AWS Gateway, NATS bus, service daemons" width="900">
+</p>
+
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless — scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
+
 ## Key Features
 
 AWS-Compatible Interfaces – Provision infrastructure with awscli, Terraform, or SDKs you already use.
 
-- Designed for Control – Run on your own terms, whether that’s in a datacenter, remote site, or sensitive environment.
+- Designed for Control – Run on your own terms, whether that's in a datacenter, remote site, or sensitive environment.
 - Minimal Dependencies – Each service is standalone and avoids complex orchestration layers.
 - Works Offline – No reliance on centralized cloud services or external networks.
 - Open Source – Licensed under AGPL 3.0. Fork it, extend it, or deploy it as-is.
@@ -61,11 +99,11 @@ Installation requires an Ubuntu / Debian system. See the detailed documentation
 The installation is straightforward to set up and running on a single node for testing purposes.
 
 ```bash
-curl https://install.mulgadc.com | bash
+curl -fsSL https://install.mulgadc.com | bash
 
 sudo /usr/local/share/spinifex/setup-ovn.sh --management
 
-sudo spx admin init
+sudo spx admin init --node node1 --nodes 1
 
 sudo systemctl start spinifex.target
 
@@ -76,7 +114,7 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/source-install/) documentation
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
 
 ### Component Repositories
 
@@ -87,6 +125,26 @@ Spinifex coordinates these independent components:
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
+## Spinifex UI
+
+Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+
+<p align="center">
+  <img src="docs/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+</p>
+
+The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+
+```bash
+open https://YOUR_NODE_IP:3000
+```
+
+- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
+- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+
+For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+
 ## Development Philosophy
 
 ### Built by Engineers, For Engineers

build/Dockerfile.distro

@@ -42,6 +42,7 @@ RUN cd viperblock && go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-s
 RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
 RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
 RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
+RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
 RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
 
 FROM scratch AS export

build/scripts/predastore-start.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Helper script for spinifex-predastore.service
+# Detects host, port, backend, and node-id from spinifex.toml
+# then execs spx. Falls back to single-node defaults if no config found.
+
+CONF=/etc/spinifex/spinifex.toml
+BIND="0.0.0.0:8443"
+BACKEND="filesystem"
+NODE_ID="0"
+
+if [ -f "$CONF" ]; then
+    H=$(awk -F'"' '/\[nodes\..*\.predastore\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$H" ]; then
+        BIND="$H"
+        BACKEND="distributed"
+    fi
+
+    N=$(awk -F'= *' '/node_id/{gsub(/ /,"",$2);print $2;exit}' "$CONF")
+    if [ -n "$N" ] && [ "$N" != "0" ]; then
+        NODE_ID="$N"
+    fi
+fi
+
+export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}"
+export SPINIFEX_PREDASTORE_PORT="${BIND##*:}"
+export SPINIFEX_PREDASTORE_BACKEND="$BACKEND"
+export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID"
+
+exec /usr/local/bin/spx service predastore start

build/scripts/wait-for-nats.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Wait for NATS to be reachable before starting a dependent service.
+# Reads the NATS address from spinifex.toml [nodes.*.nats] section,
+# falls back to nats.conf, then to 127.0.0.1:4222.
+
+CONF=/etc/spinifex/spinifex.toml
+NATS_CONF=/etc/spinifex/nats/nats.conf
+NATS_HOST=""
+NATS_PORT="4222"
+TIMEOUT=30
+
+# Try spinifex.toml first
+if [ -f "$CONF" ]; then
+    NATS_BIND=$(awk -F'"' '/\[nodes\..*\.nats\]/{f=1} f&&/^host/{print $2;exit}' "$CONF")
+    if [ -n "$NATS_BIND" ]; then
+        NATS_HOST="${NATS_BIND%%:*}"
+        NATS_PORT="${NATS_BIND##*:}"
+    fi
+fi
+
+# Fall back to nats.conf
+if [ -z "$NATS_HOST" ] && [ -f "$NATS_CONF" ]; then
+    NATS_HOST=$(grep "^listen:" "$NATS_CONF" | sed 's/listen: *//;s/:.*//')
+fi
+
+# Final fallback
+NATS_HOST="${NATS_HOST:-127.0.0.1}"
+
+for i in $(seq 1 "$TIMEOUT"); do
+    if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "Timed out waiting for NATS at $NATS_HOST:$NATS_PORT" >&2
+exit 1

build/systemd/spinifex-awsgw.service

@@ -2,20 +2,52 @@
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-gw
 Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service awsgw start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
-Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/
 Environment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-daemon.service

@@ -2,12 +2,13 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-daemon
 Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
@@ -16,5 +17,38 @@ Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
 
+# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
+ProtectSystem=full
+PrivateDevices=no
+DevicePolicy=closed
+DeviceAllow=/dev/kvm rw
+DeviceAllow=/dev/net/tun rw
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+# AF_PACKET is required by dhcpcd (raw L2 BOOTP before the iface has an IP).
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-nats.service

@@ -5,7 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-nats
 Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
 Restart=on-failure
@@ -13,5 +13,36 @@ RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/nats
+ReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-predastore.service

@@ -2,13 +2,14 @@
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-storage
 Group=spinifex
-ExecStart=/usr/local/bin/spx service predastore start
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/var/lib/spinifex/predastore-start.sh
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
@@ -17,5 +18,36 @@ Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
 Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-ui.service

@@ -5,12 +5,45 @@ After=spinifex-awsgw.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-ui
 Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
+Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
+Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
+
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key
+ReadWritePaths=/var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
 
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -2,12 +2,13 @@
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-predastore.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-viperblock
 Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service viperblock start
 Restart=on-failure
 RestartSec=5
@@ -16,5 +17,36 @@ EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
 
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env
+ReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
 [Install]
 WantedBy=spinifex.target

build/systemd/spinifex-vpcd.service

@@ -2,17 +2,48 @@
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-daemon.service
-Requires=spinifex-nats.service
+Wants=spinifex-nats.service
 
 [Service]
 Type=simple
-User=spinifex
+User=spinifex-vpcd
 Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service vpcd start
 Restart=on-failure
 RestartSec=5
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
+
+# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
+ProtectSystem=full
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml
+ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
 
 [Install]
 WantedBy=spinifex.target

cmd/alb-agent/main.go

@@ -1,62 +0,0 @@
-// alb-agent runs inside ALB VMs and manages HAProxy configuration.
-//
-// It connects to NATS, subscribes to config updates for its load balancer,
-// writes HAProxy configs to disk, and reloads HAProxy on changes. It also
-// responds to health pings so the daemon can verify the agent is alive.
-//
-// Usage:
-//
-//	alb-agent --lb-id=lb-xxxxx --nats=nats://10.0.0.1:4222
-//
-// The LB ID defaults to the hostname (set by cloud-init to the LB ID).
-// The NATS URL defaults to nats://127.0.0.1:4222.
-package main
-
-import (
-	"flag"
-	"log/slog"
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/mulgadc/spinifex/spinifex/albagent"
-)
-
-func main() {
-	var (
-		lbID    string
-		natsURL string
-	)
-
-	hostname, _ := os.Hostname()
-
-	flag.StringVar(&lbID, "lb-id", hostname, "Load balancer ID (defaults to hostname)")
-	flag.StringVar(&natsURL, "nats", "nats://127.0.0.1:4222", "NATS server URL")
-	flag.Parse()
-
-	if lbID == "" {
-		slog.Error("--lb-id is required (or set hostname to the LB ID)")
-		os.Exit(1)
-	}
-
-	slog.Info("Starting ALB agent", "lbId", lbID, "nats", natsURL)
-
-	agent, err := albagent.New(lbID, natsURL)
-	if err != nil {
-		slog.Error("Failed to create agent", "err", err)
-		os.Exit(1)
-	}
-
-	if err := agent.Start(); err != nil {
-		slog.Error("Failed to start agent", "err", err)
-		os.Exit(1)
-	}
-
-	// Wait for shutdown signal
-	sigCh := make(chan os.Signal, 1)
-	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
-	sig := <-sigCh
-
-	slog.Info("Received signal, shutting down", "signal", sig)
-	agent.Stop()
-}

cmd/lb-agent/main.go

@@ -0,0 +1,90 @@
+// lb-agent runs inside load balancer VMs and manages HAProxy configuration.
+//
+// It polls the AWS gateway for config updates and reports health via heartbeats.
+// All communication uses SigV4-signed HTTP requests.
+//
+// Usage:
+//
+//	lb-agent --lb-id=lb-xxxxx --gateway=https://192.168.1.33:9999 --access-key=AKIA... --secret-key=...
+//
+// Flags default to environment variables set by cloud-init:
+// LB_LB_ID, LB_GATEWAY_URL, LB_ACCESS_KEY, LB_SECRET_KEY, LB_REGION.
+package main
+
+import (
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/mulgadc/spinifex/spinifex/lbagent"
+)
+
+func main() {
+	var (
+		lbID       string
+		gatewayURL string
+		accessKey  string
+		secretKey  string
+		region     string
+	)
+
+	hostname, _ := os.Hostname()
+
+	flag.StringVar(&lbID, "lb-id", envOrDefault("LB_LB_ID", hostname), "Load balancer ID")
+	flag.StringVar(&gatewayURL, "gateway", os.Getenv("LB_GATEWAY_URL"), "Gateway URL (e.g. https://192.168.1.33:9999)")
+	flag.StringVar(&accessKey, "access-key", os.Getenv("LB_ACCESS_KEY"), "AWS access key ID")
+	flag.StringVar(&secretKey, "secret-key", os.Getenv("LB_SECRET_KEY"), "AWS secret access key")
+	flag.StringVar(&region, "region", envOrDefault("LB_REGION", "us-east-1"), "AWS region for SigV4 signing")
+	flag.Parse()
+
+	if lbID == "" {
+		slog.Error("--lb-id is required (or set LB_LB_ID)")
+		os.Exit(1)
+	}
+	if gatewayURL == "" {
+		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
+		os.Exit(1)
+	}
+	if accessKey == "" || secretKey == "" {
+		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
+		os.Exit(1)
+	}
+
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+
+	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
+	if err != nil {
+		slog.Error("Failed to create agent", "err", err)
+		os.Exit(1)
+	}
+
+	// Start poll loop in a goroutine — it blocks until Stop is called.
+	errCh := make(chan error, 1)
+	go func() {
+		if err := agent.Start(); err != nil {
+			errCh <- err
+		}
+	}()
+
+	// Wait for shutdown signal or fatal error.
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+
+	select {
+	case sig := <-sigCh:
+		slog.Info("Received signal, shutting down", "signal", sig)
+	case err := <-errCh:
+		slog.Error("Agent error", "err", err)
+	}
+
+	agent.Stop()
+}
+
+func envOrDefault(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}

cmd/spinifex/cmd/admin.go

@@ -25,6 +25,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"maps"
 	"net"
 	"net/http"
 	"os"
@@ -32,6 +33,7 @@ import (
 	"path"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -216,6 +218,9 @@ func init() {
 
 	adminCmd.AddCommand(certCmd)
 	certCmd.AddCommand(certRenewCmd)
+
+	adminCmd.AddCommand(upgradeCmd)
+	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
@@ -274,6 +279,7 @@ func init() {
 	imagesImportCmd.Flags().String("version", "", "Specified distro version (e.g 12)")
 	imagesImportCmd.Flags().String("arch", "", "Specified distro arch (e.g aarch64, arm64, x86_64)")
 	imagesImportCmd.Flags().String("platform", "Linux/UNIX", "Specified platform (e.g Linux/UNIX, Windows)")
+	imagesImportCmd.Flags().StringSlice("tag", nil, "Tag to apply to the imported AMI as key=value (repeatable; e.g. --tag spinifex:managed-by=elbv2)")
 	imagesImportCmd.Flags().Bool("force", false, "Force command execution (overwrites existing files)")
 }
 
@@ -308,35 +314,54 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Determine, if name specified, or file
+	// --name pulls metadata (including Tags) from the catalog; --file supplies
+	// the local image source. When both are set, the catalog provides metadata
+	// and the file is used directly (no download). When only --file is set,
+	// metadata comes from flags and no catalog tags are applied.
 	imageName, _ := cmd.Flags().GetString("name")
+	localFile, _ := cmd.Flags().GetString("file")
+
+	if imageName == "" && localFile == "" {
+		fmt.Fprintf(os.Stderr, "Either --name or --file is required to import an image\n")
+		os.Exit(1)
+	}
 
 	if imageName != "" {
 		var exists bool
-		// Confirm the image exists
 		image, exists = utils.AvailableImages[imageName]
-
 		if !exists {
 			fmt.Fprintf(os.Stderr, "Image name not found in available images")
 			os.Exit(1)
 		}
-	} else {
-		imageFile, _ = cmd.Flags().GetString("file")
+	}
 
-		if imageFile == "" {
-			fmt.Fprintf(os.Stderr, "File required to import image")
-			os.Exit(1)
-		}
-
-		if _, err := os.Stat(imageFile); err != nil {
+	if localFile != "" {
+		if _, err := os.Stat(localFile); err != nil {
 			fmt.Fprintf(os.Stderr, "File could not be found: %s", err)
 			os.Exit(1)
 		}
+		imageFile = localFile
+		if imageName == "" {
+			image.Distro, _ = cmd.Flags().GetString("distro")
+			image.Version, _ = cmd.Flags().GetString("version")
+			image.Arch, _ = cmd.Flags().GetString("arch")
+			image.Platform, _ = cmd.Flags().GetString("platform")
+		}
+	}
 
-		image.Distro, _ = cmd.Flags().GetString("distro")
-		image.Version, _ = cmd.Flags().GetString("version")
-		image.Arch, _ = cmd.Flags().GetString("arch")
-		image.Platform, _ = cmd.Flags().GetString("platform")
+	// --tag k=v (repeatable) merges user-supplied tags into the AMI. Overrides
+	// catalog tags on key collision so operators can re-tag a known image.
+	tagFlags, _ := cmd.Flags().GetStringSlice("tag")
+	for _, kv := range tagFlags {
+		k, v, ok := strings.Cut(kv, "=")
+		if !ok || k == "" {
+			fmt.Fprintf(os.Stderr, "Invalid --tag %q: expected key=value\n", kv)
+			os.Exit(1)
+		}
+		if image.Tags == nil {
+			image.Tags = map[string]string{}
+		}
+		image.Tags[k] = v
 	}
 
 	if image.Distro == "" {
@@ -372,7 +397,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Created config directory: %s\n", imagePath)
 
 	// Next, if the file is selected to download, fetch it, extract disk image, and save to path
-	if imageName != "" {
+	if imageName != "" && localFile == "" {
 		// Download the file to the image path
 		filename := path.Base(image.URL)
 		imageFile = fmt.Sprintf("%s/%s", imagePath, filename)
@@ -437,6 +462,13 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	manifest.AMIMetadata.ImageOwnerAlias = "system"
 	manifest.AMIMetadata.VolumeSizeGiB = utils.SafeInt64ToUint64(imageStat.Size() / 1024 / 1024 / 1024)
 
+	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
+	// onto the imported AMI so the UI can filter them out.
+	if len(image.Tags) > 0 {
+		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+	}
+
 	// Volume Data
 	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
 	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
@@ -546,9 +578,9 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 // TODO: Move all logic to a module, use minimal application logic in viper commands
 func runAdminInit(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
-		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
-		fmt.Fprintln(os.Stderr, "   Run with sudo, or manually: sudo cp ~/spinifex/config/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates")
-		fmt.Fprintln(os.Stderr)
+		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin init' is not running as root.")
+		fmt.Fprintln(os.Stderr, "   Service user setup and CA certificate installation will be skipped.")
+		fmt.Fprintln(os.Stderr, "   For production deployments, run with sudo.")
 	}
 
 	force, _ := cmd.Flags().GetBool("force")
@@ -785,14 +817,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	bootstrapDir := filepath.Join(spxRoot, "awsgw")
+	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
+	if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Println("\n🔐 Generated IAM master key")
 	fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
+	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
 
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
@@ -873,7 +911,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
 	var predastoreNodeID int
@@ -933,6 +971,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   spxRoot,
+		LogDir:    LogDirFor(spxRoot),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -1014,14 +1053,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(spxRoot)
 
-	// In production layout (running as root), fix ownership so the service user
-	// can read config and write data. SUDO_USER identifies the operator account.
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
 	if os.Getuid() == 0 {
-		sudoUser := os.Getenv("SUDO_USER")
-		if sudoUser != "" {
-			admin.ChownRecursive(configDir, sudoUser)
-			admin.ChownRecursive(spxRoot, sudoUser)
-		}
+		admin.SetServiceOwnership()
 	}
 
 	// Print success message
@@ -1051,13 +1086,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		fmt.Fprintf(os.Stderr, "❌ Error generating IAM master key: %v\n", err)
 		os.Exit(1)
 	}
-	bootstrapResult, err := writeBootstrapFiles(configDir, masterKey, accessKey, secretKey, accountID)
+	bootstrapDir := filepath.Join(spxRoot, "awsgw")
+	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
 	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
@@ -1141,7 +1177,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Generate multi-node predastore config
 	var predastoreNodeID int
@@ -1171,6 +1207,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		Region:    region,
 		NatsToken: natsToken,
 		DataDir:   spxRoot,
+		LogDir:    LogDirFor(spxRoot),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -1246,14 +1283,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	admin.CreateServiceDirectories(spxRoot)
 
-	// In production layout (running as root), fix ownership so the service user
-	// can read config and write data. SUDO_USER identifies the operator account.
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
 	if os.Getuid() == 0 {
-		sudoUser := os.Getenv("SUDO_USER")
-		if sudoUser != "" {
-			admin.ChownRecursive(configDir, sudoUser)
-			admin.ChownRecursive(spxRoot, sudoUser)
-		}
+		admin.SetServiceOwnership()
 	}
 
 	// Keep formation server running briefly so joining nodes can fetch complete status
@@ -1279,8 +1312,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 func runAdminJoin(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
-		fmt.Fprintln(os.Stderr, "⚠️  Not running as root — CA certificate will not be installed to system trust store.")
-		fmt.Fprintln(os.Stderr)
+		fmt.Fprintln(os.Stderr, "⚠️  Warning: 'spx admin join' is not running as root.")
+		fmt.Fprintln(os.Stderr, "   Service user setup and CA certificate installation will be skipped.")
+		fmt.Fprintln(os.Stderr, "   For production deployments, run with sudo.")
 	}
 
 	node, _ := cmd.Flags().GetString("node")
@@ -1500,12 +1534,17 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "❌ Error decoding master key: %v\n", err)
 		os.Exit(1)
 	}
-	if err := writeBootstrapFilesWithAdmin(configDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID, creds.AdminAccessKey, creds.AdminSecretKey); err != nil {
+	bootstrapDir := filepath.Join(dataDir, "awsgw")
+	if err := writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKeyBytes, creds.AccessKey, creds.SecretKey, creds.AccountID, creds.AdminAccessKey, creds.AdminSecretKey); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
 		os.Exit(1)
 	}
+	if err := writeSystemCredentials(configDir, creds.AccessKey, creds.SecretKey); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error writing system credentials: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Println("✅ IAM master key received from leader")
-	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(configDir, "bootstrap.json"))
+	fmt.Printf("✅ Bootstrap file written: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 
 	// Generate server cert signed by CA with this node's bind IP
 	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
@@ -1533,7 +1572,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	portStr := fmt.Sprintf("%d", port)
+	portStr := strconv.Itoa(port)
 
 	// Generate multi-node predastore config
 	var predastoreNodeID int
@@ -1569,6 +1608,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		Region:    creds.Region,
 		NatsToken: creds.NatsToken,
 		DataDir:   dataDir,
+		LogDir:    LogDirFor(dataDir),
 		ConfigDir: configDir,
 
 		Node:          node,
@@ -1648,14 +1688,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	admin.CreateServiceDirectories(dataDir)
 
-	// In production layout (running as root), fix ownership so the service user
-	// can read config and write data. SUDO_USER identifies the operator account.
+	// In production layout (running as root), set per-service ownership on
+	// directories and shared config files (root:spinifex with correct modes).
 	if os.Getuid() == 0 {
-		sudoUser := os.Getenv("SUDO_USER")
-		if sudoUser != "" {
-			admin.ChownRecursive(configDir, sudoUser)
-			admin.ChownRecursive(dataDir, sudoUser)
-		}
+		admin.SetServiceOwnership()
 	}
 
 	// Print cluster summary
@@ -1665,10 +1701,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	for name, n := range statusResp.Nodes {
 		fmt.Printf("     - %s (%s)\n", name, n.BindIP)
 	}
-	fmt.Println("\n📋 Next steps:")
-	fmt.Println("   1. Start services:")
-	fmt.Println("      ./scripts/start-dev.sh")
-	fmt.Println()
 }
 
 // buildRemoteNodes converts formation NodeInfo into RemoteNode entries,
@@ -1801,7 +1833,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 			endpointHost = h
 		}
 	}
-	endpointURL := fmt.Sprintf("https://%s:9999", endpointHost)
+	endpointURL := "https://" + net.JoinHostPort(endpointHost, "9999")
 
 	credPath := filepath.Join(homeDir, ".aws", "credentials")
 	configPath := filepath.Join(homeDir, ".aws", "config")
@@ -1902,8 +1934,9 @@ type writeBootstrapResult struct {
 }
 
 // writeBootstrapFiles generates new admin credentials and writes the bootstrap
-// files (master.key + bootstrap.json). Used by init flows (single and multi-node).
-func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKey, accountID string) (*writeBootstrapResult, error) {
+// files (master.key to configDir, bootstrap.json to bootstrapDir).
+// Used by init flows (single and multi-node).
+func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, accessKey, secretKey, accountID string) (*writeBootstrapResult, error) {
 	adminAccessKey, err := admin.GenerateAWSAccessKey()
 	if err != nil {
 		return nil, fmt.Errorf("generate admin access key: %w", err)
@@ -1912,7 +1945,7 @@ func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKe
 	if err != nil {
 		return nil, fmt.Errorf("generate admin secret key: %w", err)
 	}
-	if err := writeBootstrapFilesWithAdmin(configDir, masterKey, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey); err != nil {
+	if err := writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey); err != nil {
 		return nil, err
 	}
 	return &writeBootstrapResult{
@@ -1921,9 +1954,31 @@ func writeBootstrapFiles(configDir string, masterKey []byte, accessKey, secretKe
 	}, nil
 }
 
+// writeSystemCredentials writes the system access key to a plaintext JSON file.
+// The daemon reads this at startup to inject credentials into ALB VM cloud-init
+// for SigV4-authenticated communication with the AWS gateway.
+func writeSystemCredentials(configDir, accessKey, secretKey string) error {
+	creds := struct {
+		AccessKeyID     string `json:"access_key_id"`
+		SecretAccessKey string `json:"secret_access_key"`
+	}{
+		AccessKeyID:     accessKey,
+		SecretAccessKey: secretKey,
+	}
+	data, err := json.MarshalIndent(creds, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshalling system credentials: %w", err)
+	}
+	return os.WriteFile(filepath.Join(configDir, "system-credentials.json"), data, 0600)
+}
+
 // writeBootstrapFilesWithAdmin writes the bootstrap files using the provided
-// admin credentials. Used by join flows where admin creds come from the leader.
-func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string) error {
+// admin credentials. master.key goes to configDir, bootstrap.json goes to
+// bootstrapDir (the awsgw data directory) so it stays outside /etc/spinifex.
+func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []byte, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string) error {
+	if err := os.MkdirAll(bootstrapDir, 0700); err != nil {
+		return fmt.Errorf("create bootstrap directory %s: %w", bootstrapDir, err)
+	}
 	if err := handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), masterKey); err != nil {
 		return fmt.Errorf("saving master key: %w", err)
 	}
@@ -1951,7 +2006,7 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey,
 		},
 	}
 
-	return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd)
+	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }
 
 // detectedWanBridge returns the OVS bridge name that OVN bridge-mappings should

cmd/spinifex/cmd/admin_upgrade.go

@@ -0,0 +1,92 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/spf13/cobra"
+)
+
+var upgradeCmd = &cobra.Command{
+	Use:   "upgrade",
+	Short: "Apply pending config migrations",
+	Long: `Apply pending configuration file migrations for upgrades between Spinifex versions.
+
+When run interactively (without --yes), shows pending migrations and prompts
+for confirmation. When called from setup.sh with --yes, applies immediately.
+
+Operators can skip migrations during install by setting INSTALL_SPINIFEX_SKIP_MIGRATE=1,
+then run 'spx admin upgrade' manually to review and apply.`,
+	Run: runAdminUpgrade,
+}
+
+func runAdminUpgrade(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+	dataDir, _ := cmd.Root().Flags().GetString("spinifex-dir")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	// Check that the installation exists.
+	spinifexToml := configDir + "/spinifex.toml"
+	if _, err := os.Stat(spinifexToml); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "No Spinifex installation found at %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	// Show current versions.
+	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading config versions: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Reading current config versions...")
+	for name, v := range versions {
+		if v == 0 {
+			fmt.Printf("  %-20s %d (no version marker)\n", name+":", v)
+		} else {
+			fmt.Printf("  %-20s %d\n", name+":", v)
+		}
+	}
+	fmt.Println()
+
+	// Check for pending migrations.
+	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking pending migrations: %v\n", err)
+		os.Exit(1)
+	}
+
+	if len(pending) == 0 {
+		fmt.Println("No pending config migrations.")
+		return
+	}
+
+	fmt.Println("Pending config migrations:")
+	for _, p := range pending {
+		fmt.Printf("  [config] %s %d → %d: %s\n", p.Target, p.FromVersion, p.ToVersion, p.Description)
+	}
+	fmt.Println()
+
+	// Prompt for confirmation unless --yes.
+	if !yes {
+		fmt.Print("Apply? [y/N] ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer != "y" && answer != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	// Apply migrations.
+	if err := migrate.DefaultRegistry.RunAllConfig(configDir, dataDir); err != nil {
+		fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("\nDone. Restart services to apply: sudo systemctl restart spinifex.target")
+}

cmd/spinifex/cmd/get.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -76,9 +77,15 @@ func loadConfigAndConnect() (*config.ClusterConfig, *nats.Conn, error) {
 	}
 
 	// Backfill BaseDir from config file path when not set in the config.
-	// Config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
 	if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" {
-		nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
+		if isProductionLayout() {
+			// Production: BaseDir is /var/lib/spinifex (data dir).
+			// /var/lib/spinifex/config is a symlink to /etc/spinifex.
+			nodeConfig.BaseDir = DefaultDataDir()
+		} else {
+			// Dev: config lives at <baseDir>/config/spinifex.toml, so baseDir is two levels up.
+			nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
+		}
 		cfg.Nodes[cfg.Node] = nodeConfig
 	}
 
@@ -215,7 +222,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 				resp.Region,
 				resp.AZ,
 				formatUptime(resp.Uptime),
-				fmt.Sprintf("%d", resp.VMCount),
+				strconv.Itoa(resp.VMCount),
 				strings.Join(resp.Services, ","),
 			})
 		} else {
@@ -295,7 +302,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 			v.InstanceID,
 			v.Status,
 			v.InstanceType,
-			fmt.Sprintf("%d", v.VCPU),
+			strconv.Itoa(v.VCPU),
 			formatMemGB(v.MemoryGB),
 			v.Node,
 			v.Host,

cmd/spinifex/cmd/paths.go

@@ -7,7 +7,7 @@ import (
 )
 
 // DefaultConfigDir returns the default configuration directory.
-// Production: /etc/spinifex (when running as root AND /etc/spinifex exists)
+// Production: /etc/spinifex (when /etc/spinifex exists)
 // Development: ~/spinifex/config
 func DefaultConfigDir() string {
 	if isProductionLayout() {
@@ -26,6 +26,16 @@ func DefaultDataDir() string {
 	return filepath.Join(realUserHomeDir(), "spinifex")
 }
 
+// LogDirFor returns the log directory for a given data directory.
+// Production: /var/log/spinifex (matches systemd ReadWritePaths)
+// Development: <dataDir>/logs (supports custom per-node data dirs)
+func LogDirFor(dataDir string) string {
+	if isProductionLayout() {
+		return "/var/log/spinifex"
+	}
+	return filepath.Join(dataDir, "logs")
+}
+
 // realUserHomeDir returns the home directory of the real (non-sudo) user.
 // When running under sudo, SUDO_USER is set to the invoking user — resolve
 // their home directory so config/data land in the right place.
@@ -45,13 +55,12 @@ func DefaultConfigFile() string {
 }
 
 // isProductionLayout returns true when running in a production install.
-// Detected by: running as root AND /etc/spinifex directory already exists
-// (created by setup.sh during binary install).
+// Detected by: /etc/spinifex directory exists (created by setup.sh during
+// binary install). No root check — allows non-root users (e.g. tf-user) to
+// run CLI commands like `spx get nodes` without sudo or --config flags.
 func isProductionLayout() bool {
-	if os.Getuid() == 0 {
-		if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
-			return true
-		}
+	if info, err := os.Stat("/etc/spinifex"); err == nil && info.IsDir() {
+		return true
 	}
 	return false
 }

cmd/spinifex/cmd/paths_test.go

@@ -7,9 +7,9 @@ import (
 )
 
 func TestDefaultConfigDir_Dev(t *testing.T) {
-	// Non-root user without /etc/spinifex should get dev paths
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	// Without /etc/spinifex should get dev paths
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	homeDir, _ := os.UserHomeDir()
@@ -21,8 +21,8 @@ func TestDefaultConfigDir_Dev(t *testing.T) {
 }
 
 func TestDefaultDataDir_Dev(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	homeDir, _ := os.UserHomeDir()
@@ -34,8 +34,8 @@ func TestDefaultDataDir_Dev(t *testing.T) {
 }
 
 func TestDefaultConfigFile(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	got := DefaultConfigFile()
@@ -46,12 +46,12 @@ func TestDefaultConfigFile(t *testing.T) {
 	}
 }
 
-func TestIsProductionLayout_NonRoot(t *testing.T) {
-	if os.Getuid() == 0 {
-		t.Skip("test requires non-root user")
+func TestIsProductionLayout_NoEtcSpinifex(t *testing.T) {
+	if _, err := os.Stat("/etc/spinifex"); err == nil {
+		t.Skip("test requires /etc/spinifex to not exist")
 	}
 
 	if isProductionLayout() {
-		t.Error("isProductionLayout() = true for non-root user, want false")
+		t.Error("isProductionLayout() = true without /etc/spinifex, want false")
 	}
 }

cmd/spinifex/cmd/service.go

@@ -245,13 +245,13 @@ var viperblockStartCmd = &cobra.Command{
 
 		accessKey := viper.GetString("access-key")
 		if accessKey != "" {
-			fmt.Println("Overwriting access-key:", accessKey)
+			fmt.Println("Overwriting access-key: ****")
 			nodeConfig.Predastore.AccessKey = accessKey
 		}
 
 		secretKey := viper.GetString("secret-key")
 		if secretKey != "" {
-			fmt.Println("Overwriting secret-key:", secretKey)
+			fmt.Println("Overwriting secret-key: ****")
 			nodeConfig.Predastore.SecretKey = secretKey
 		}
 
@@ -287,6 +287,7 @@ var viperblockStartCmd = &cobra.Command{
 
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
 			PluginPath: pluginPath,
 			S3Host:     nodeConfig.Predastore.Host,
 			Bucket:     nodeConfig.Predastore.Bucket,
@@ -733,6 +734,12 @@ var vpcdStartCmd = &cobra.Command{
 			}
 		}
 
+		baseDir := viper.GetString("base-dir")
+		if baseDir != "" {
+			fmt.Println("Overwriting vpcd base-dir to:", baseDir)
+			nodeConfig.BaseDir = baseDir
+		}
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -860,7 +867,7 @@ func init() {
 
 	serviceCmd.AddCommand(viperblockCmd)
 
-	viperblockCmd.PersistentFlags().String("s3-host", "0.0.0.0:8443", "Predastore (S3) host URI")
+	viperblockCmd.PersistentFlags().String("s3-host", "", "Predastore (S3) host URI")
 	viper.BindEnv("s3-host", "SPINIFEX_VIPERBLOCK_S3_HOST")
 	viper.BindPFlag("s3-host", predastoreCmd.PersistentFlags().Lookup("s3-host"))
 

cmd/spinifex/cmd/templates/nats.conf

@@ -1,3 +1,4 @@
+# spinifex-config-version: 1
 # NATS Server Configuration
 # Generated by Spinifex admin init
 
@@ -16,7 +17,7 @@ cluster {
 
   routes = [
 {{- range .ClusterRoutes }}
-    "nats://{{ . }}",
+    "nats-route://{{$.NatsToken}}@{{ . }}",
 {{- end }}
   ]
 }
@@ -26,7 +27,7 @@ server_name: spinifex-nats-{{.Node}}
 
 # Authorization
 authorization {
-#  token: "{{.NatsToken}}"
+  token: "{{.NatsToken}}"
 }
 
 # JetStream for persistent messaging and KV store
@@ -40,7 +41,7 @@ jetstream {
 debug: false
 trace: false
 logtime: true
-log_file: "{{.DataDir}}/logs/nats.log"
+log_file: "{{.LogDir}}/nats.log"
 
 # Limits
 max_connections: 1000

cmd/spinifex/cmd/top.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/types"
@@ -99,7 +100,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 				resp.Node,
 				fmt.Sprintf("%d/%d", resp.AllocVCPU, resp.TotalVCPU),
 				fmt.Sprintf("%s/%s", formatMemGB(resp.AllocMemGB), formatMemGB(resp.TotalMemGB)),
-				fmt.Sprintf("%d", resp.VMCount),
+				strconv.Itoa(resp.VMCount),
 			})
 
 			for _, cap := range resp.InstanceTypes {
@@ -146,8 +147,8 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 		agg := capacityMap[name]
 		capTable = append(capTable, []string{
 			name,
-			fmt.Sprintf("%d", agg.Available),
-			fmt.Sprintf("%d", agg.VCPU),
+			strconv.Itoa(agg.Available),
+			strconv.Itoa(agg.VCPU),
 			formatMemGB(agg.MemoryGB),
 		})
 	}

docs/COMMANDS.md

@@ -72,6 +72,12 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|---------------|-------------|------------|--------|
 | `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
 
+### Upgrade Management
+
+| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------|---------------|-------------|------------|--------|
+| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Existing installation (`spinifex.toml` must exist in config dir) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | 1. No installation present (error, suggests `spx admin init`)<br>2. No pending migrations (prints "No pending config migrations")<br>3. Pending migrations with interactive confirmation (y/yes applies, anything else aborts)<br>4. Pending migrations with `--yes` (applies without prompt)<br>5. Migration failure surfaces error and exits non-zero | **DONE** |
+
 ### Account Management
 
 | Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
@@ -547,16 +553,16 @@ STS provides temporary credentials and identity verification. `GetCallerIdentity
 | `assume-role` | — | `--role-arn`, `--role-session-name`, `--duration-seconds`, `--policy`, `--external-id`, `--serial-number`, `--token-code`, `--tags`, `--transitive-tag-keys` | Role must exist, caller must be allowed by trust policy | Gateway validates RoleArn + RoleSessionName (required) → looks up role → evaluates trust policy (AssumeRolePolicyDocument) against caller → generates temporary credentials (AccessKeyId with ASIA prefix, SecretAccessKey, SessionToken) → credentials expire after DurationSeconds (default 3600, max 43200) → return Credentials + AssumedRoleUser (ARN, AssumedRoleId) | 1. Assume role with valid trust policy<br>2. Trust policy denies caller (AccessDenied)<br>3. Non-existent role (NoSuchEntity)<br>4. Expired session token rejected at SigV4<br>5. Custom duration<br>6. Session tags | **NOT STARTED** |
 | `get-session-token` | — | `--duration-seconds`, `--serial-number`, `--token-code` | Valid IAM user credentials (not role) | Gateway validates caller is IAM user (not assumed role) → generates temporary credentials (ASIA prefix) → session inherits user's permissions → return Credentials | 1. Get token for IAM user<br>2. Duration within range<br>3. Reject if caller is assumed role | **NOT STARTED** |
 
-### ELBv2 (Elastic Load Balancing v2 — Application Load Balancer)
+### ELBv2 (Elastic Load Balancing v2 — Application & Network Load Balancer)
 
-All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key format: `lb.{lbId}`, `tg.{tgId}`, `listener.{listenerId}`. The data plane uses a system-managed ALB VM running HAProxy, launched automatically during CreateLoadBalancer. HAProxy configuration is pushed via NATS (`elbv2.alb.{lbId}.config` topic) whenever listeners or targets change. Agent readiness is polled via `elbv2.alb.{lbId}.ping`.
+All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key format: `lb.{lbId}`, `tg.{tgId}`, `listener.{listenerId}`. The data plane uses a system-managed LB VM running HAProxy, launched automatically during CreateLoadBalancer. HAProxy configuration is pushed via NATS (`elbv2.lb.{lbId}.config` topic) whenever listeners or targets change. Agent readiness is polled via `elbv2.lb.{lbId}.ping`.
 
 #### ELBv2 - Load Balancers
 
 | Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
 |---------|-------------------|---------------|---------------|-------------|------------|--------|
-| `create-load-balancer` | `--name`, `--subnets`, `--security-groups`, `--scheme` (internet-facing/internal), `--tags`, `--ip-address-type` (ipv4 only) | `--type` (hardcoded application), `--customer-owned-ipv4-pool`, `--dry-run` | None (subnets optional) | Gateway validates Name (required) → NATS `elbv2.CreateLoadBalancer` with `spinifex-workers` queue group → daemon validates scheme (internet-facing or internal, InvalidScheme if other) → checks duplicate name in account (DuplicateLoadBalancerName) → generates LB ID → builds DNS name `{name}-{lbId}.{region}.elb.spinifex.local` → if subnets provided, creates one ENI per subnet with `spinifex:managed-by=elbv2` tag (rolls back on failure) → if system AMI available, launches t3.nano ALB VM with cloud-init running `/usr/local/bin/alb-agent` → state=provisioning → background goroutine polls agent ping for ~5min → on agent ready: state=active, on timeout: state=failed → if no VM launcher: state=active immediately → persists to KV → returns LoadBalancer with ARN, DNSName, State | 1. Create with name (happy path)<br>2. Internal scheme<br>3. Invalid scheme (InvalidScheme)<br>4. Duplicate name (DuplicateLoadBalancerName)<br>5. Missing name (MissingParameter)<br>6. With tags<br>7. With subnets (ENI creation) | **DONE** |
-| `delete-load-balancer` | `--load-balancer-arn` | `--dry-run` | LB must exist | Gateway validates LoadBalancerArn (required) → NATS `elbv2.DeleteLoadBalancer` → daemon verifies LB exists and belongs to account → terminates ALB VM (async background) → deletes all listeners for this LB → deletes system-managed ENIs → deletes LB record from KV → returns success | 1. Delete existing LB<br>2. Not found (LoadBalancerNotFound)<br>3. Cascades listener cleanup<br>4. ENI cleanup | **DONE** |
+| `create-load-balancer` | `--name`, `--subnets`, `--security-groups`, `--scheme` (internet-facing/internal), `--tags`, `--ip-address-type` (ipv4 only) | `--type` (hardcoded application), `--customer-owned-ipv4-pool`, `--dry-run` | None (subnets optional) | Gateway validates Name (required) → NATS `elbv2.CreateLoadBalancer` with `spinifex-workers` queue group → daemon validates scheme (internet-facing or internal, InvalidScheme if other) → checks duplicate name in account (DuplicateLoadBalancerName) → generates LB ID → builds DNS name `{name}-{lbId}.{region}.elb.spinifex.local` → if subnets provided, creates one ENI per subnet with `spinifex:managed-by=elbv2` tag (rolls back on failure) → if system AMI available, launches sys.micro LB VM with cloud-init running `/usr/local/bin/lb-agent` → state=provisioning → background goroutine polls agent ping for ~5min → on agent ready: state=active, on timeout: state=failed → if no VM launcher: state=active immediately → persists to KV → returns LoadBalancer with ARN, DNSName, State | 1. Create with name (happy path)<br>2. Internal scheme<br>3. Invalid scheme (InvalidScheme)<br>4. Duplicate name (DuplicateLoadBalancerName)<br>5. Missing name (MissingParameter)<br>6. With tags<br>7. With subnets (ENI creation) | **DONE** |
+| `delete-load-balancer` | `--load-balancer-arn` | `--dry-run` | LB must exist | Gateway validates LoadBalancerArn (required) → NATS `elbv2.DeleteLoadBalancer` → daemon verifies LB exists and belongs to account → terminates LB VM (async background) → deletes all listeners for this LB → deletes system-managed ENIs → deletes LB record from KV → returns success | 1. Delete existing LB<br>2. Not found (LoadBalancerNotFound)<br>3. Cascades listener cleanup<br>4. ENI cleanup | **DONE** |
 | `describe-load-balancers` | `--load-balancer-arns`, `--names` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeLoadBalancers` → daemon lists all LB records from KV → filters by account ID → applies ARN filter and/or Name filter → returns LoadBalancer list with State, DNSName, Scheme, VpcId, AvailabilityZones | 1. List all LBs<br>2. Filter by name<br>3. Account isolation | **DONE** |
 
 #### ELBv2 - Target Groups
@@ -583,6 +589,12 @@ All ELBv2 resources are stored in the `spinifex-elbv2` JetStream KV bucket. Key
 | `delete-listener` | `--listener-arn` | `--dry-run` | Listener must exist | Gateway validates ListenerArn (required) → NATS `elbv2.DeleteListener` → daemon verifies listener exists and belongs to account → deletes from KV → fetches parent LB → calls `pushConfig(lb)` to regenerate HAProxy config without this listener → returns success | 1. Delete listener<br>2. Not found (ListenerNotFound) | **DONE** |
 | `describe-listeners` | `--load-balancer-arn`, `--listener-arns` | `--page-size`, `--marker`, `--dry-run` | None | NATS `elbv2.DescribeListeners` → daemon lists listeners → if LoadBalancerArn specified, filters to that LB → applies ARN filter → account isolation → returns Listener list | 1. Filter by LB ARN<br>2. Account isolation | **DONE** |
 
+#### ELBv2 - Tags
+
+| Command | Implemented Flags | Missing Flags | Prerequisites | Basic Logic | Test Cases | Status |
+|---------|-------------------|---------------|---------------|-------------|------------|--------|
+| `describe-tags` | `--resource-arns` (loadbalancer, targetgroup, listener) | — | Resource(s) must exist | Gateway validates non-empty ResourceArns → NATS `elbv2.DescribeTags` → daemon parses each ARN's resource segment to dispatch by type → looks up record (LB/TG/listener) → enforces account isolation (cross-account → not-found) → returns TagDescriptions in input order with sorted-by-key Tag list. Listener records don't store tags yet, so they always return an empty Tags slice (matches AWS behaviour for untagged resources). Required by Terraform AWS provider post-create refresh on `aws_lb`, `aws_lb_target_group`, and `aws_lb_listener`. | 1. LB tags round-trip<br>2. TG tags round-trip<br>3. Listener returns empty Tags (no error)<br>4. Multiple ARNs in one call<br>5. Unknown LB/TG/listener (per-type not-found)<br>6. Cross-account ARN (not-found, no leak)<br>7. Invalid/non-ELBv2 ARN (InvalidParameterValue)<br>8. Empty/nil ResourceArns (MissingParameter)<br>9. Untagged LB returns empty Tags | **DONE** |
+
 #### ELBv2 - Not Implemented
 
 | Feature | Description | Priority | Status |

docs/DESIGN.md

@@ -4,55 +4,9 @@ Spinifex is an AWS-compatible infrastructure platform for bare-metal, edge, and
 
 ## High-Level Architecture
 
-```
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                              AWS SDK / CLI                                  │
-│                    (aws ec2 run-instances --endpoint-url ...)               │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                                      │ HTTPS (SigV4 Auth)
-                                      ▼
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                           AWS Gateway (Port 9999)                           │
-│                                                                             │
-│   • TLS termination                    • AWS SigV4 authentication           │
-│   • EC2 Query Protocol parsing         • Routes to service handlers         │
-│   • XML response generation            • NATS client connection             │
-│                                                                             │
-│   File: spinifex/services/awsgw/awsgw.go                                        │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                                      │ JSON over NATS
-                                      ▼
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                            NATS Message Broker                              │
-│                                                                             │
-│   Topics:                              Queue Groups:                        │
-│   • ec2.RunInstances                   • spinifex-workers (load balanced)       │
-│   • ec2.DescribeInstances              • No queue (fan-out to all nodes)    │
-│   • ec2.cmd.{instanceID}              (per-instance start/stop/terminate)   │
-│   • ebs.mount / ebs.unmount                                                 │
-└─────────────────────────────────────────────────────────────────────────────┘
-                                      │
-                    ┌─────────────────┼─────────────────┐
-                    ▼                 ▼                 ▼
-┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
-│   Spinifex Daemon        │ │   Spinifex Daemon        │ │   Spinifex Daemon        │
-│   (Node 1)           │ │   (Node 2)           │ │   (Node N)           │
-│                      │ │                      │ │                      │
-│ • NATS subscriber    │ │ • NATS subscriber    │ │ • NATS subscriber    │
-│ • Resource manager   │ │ • Resource manager   │ │ • Resource manager   │
-│ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │ │ • QEMU/KVM launcher  │
-│ • QMP monitoring     │ │ • QMP monitoring     │ │ • QMP monitoring     │
-│                      │ │                      │ │                      │
-│ File: daemon.go      │ │ File: daemon.go      │ │ File: daemon.go      │
-└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
-          │                       │                       │
-          ▼                       ▼                       ▼
-┌──────────────────────┐ ┌──────────────────────┐ ┌──────────────────────┐
-│   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │ │   QEMU/KVM VMs       │
-└──────────────────────┘ └──────────────────────┘ └──────────────────────┘
-```
+<p align="center">
+  <img src="../.github/assets/design-architecture.svg" alt="Spinifex high-level architecture — AWS SDK, gateway, NATS broker, per-node daemons, QEMU/KVM VMs" width="900">
+</p>
 
 ## Request Flow
 

docs/INSTALL_ORIGINAL.md

@@ -48,7 +48,7 @@ sudo apt install nbdkit nbdkit-plugin-dev pkg-config qemu-system qemu-utils qemu
 
 **Note:** OVN and Open vSwitch are required for VPC networking (virtual switches, routers, DHCP, Geneve overlay). The packages are installed above, but the setup and configuration step (`setup-ovn.sh`) is covered later — see the [Setup OVN](#setup-ovn) section below.
 
-Ensure the Go toolkit is installed for version 1.26.1 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
+Ensure the Go toolkit is installed for version 1.26.2 or higher. Recommended to install the latest directly from [https://go.dev/dl/](https://go.dev/dl/).
 
 Confirm Go is correctly installed, and set in your $PATH.
 
@@ -940,9 +940,9 @@ All nodes should show `Ready` status. Use `./bin/spx get vms` to see running ins
 Check individual daemon health endpoints:
 
 ```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
-curl -s http://$SPINIFEX_NODE2:4432/health
-curl -s http://$SPINIFEX_NODE3:4432/health
+curl -sk https://$SPINIFEX_NODE1:4432/health
+curl -sk https://$SPINIFEX_NODE2:4432/health
+curl -sk https://$SPINIFEX_NODE3:4432/health
 ```
 
 Check NATS cluster routing (from any node):

docs/TAG-CONVENTIONS.md

@@ -0,0 +1,66 @@
+# Spinifex Tag Conventions
+
+Spinifex uses a small set of reserved tag keys to mark resources that are
+owned and managed by the platform itself (as opposed to resources created
+directly by customer AWS API calls). The UI uses these tags to hide
+system-owned resources from customer-facing listings so operators do not
+mistake them for user workloads and accidentally terminate or modify
+them.
+
+## `spinifex:managed-by`
+
+Identifies the Spinifex platform component that owns a resource.
+
+| Value   | Component                       | Tagged resource types                 |
+|---------|---------------------------------|---------------------------------------|
+| `elbv2` | ALB / HAProxy load balancers    | LB VM (`instance`), LB ENI, LB AMI    |
+
+The tag is applied by the backend at create/launch time:
+
+- **Instance** — set in
+  `spinifex/spinifex/daemon/daemon_system_instance.go` when
+  `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto
+  the stored `vm.VM.ManagedBy` field for fast lookup.
+- **ENI** — set in
+  `spinifex/spinifex/handlers/elbv2/service_impl.go` when
+  `CreateLoadBalancer` calls `CreateNetworkInterface`.
+- **AMI** — declared on the catalog entry in
+  `spinifex/spinifex/utils/images.go` and copied into
+  `AMIMetadata.Tags` by the image importer in
+  `spinifex/cmd/spinifex/cmd/admin.go`.
+
+Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`;
+new system components should register their value there and reuse the
+constants rather than hardcoding strings.
+
+## UI behaviour
+
+The admin **Nodes** page and the **Images** (AMIs) page filter out
+resources that carry `spinifex:managed-by`. Engineers who need to see
+system resources for debugging can append `?system=1` to the URL — the
+filter is bypassed and a hidden-count banner is replaced with the full
+listing. Direct navigation to a system AMI detail page without
+`?system=1` renders the "Image not found" fallback.
+
+The EC2 **Instances** page does not need UI-side filtering: backend
+SigV4 authentication scopes `DescribeInstances` by account ID, and
+system instances live under the global system account
+(`000000000000`), so customers never see them.
+
+## `spinifex:lb-arn`
+
+Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`.
+Stores the ARN of the parent load balancer so reconciliation can map an
+ENI back to its owner without a separate lookup. Customer-facing
+filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is
+informational.
+
+## Adding a new system component
+
+1. Add a new value constant next to `ManagedByELBv2` in
+   `spinifex/spinifex/tags/tags.go`.
+2. Tag every resource the component creates (instance, ENI, AMI,
+   volume, …) with `spinifex:managed-by=<value>`.
+3. Add the value to the table above so operators can recognise it.
+4. No frontend changes are required — the UI filters on the key, not
+   the value.

docs/install-multi-node/README.md

@@ -152,7 +152,7 @@ Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import a
 The init command must still be running when join executes. If init exited, re-run with `--force`.
 
 ```bash
-curl -s http://$SPINIFEX_NODE1:4432/health
+curl -sk https://$SPINIFEX_NODE1:4432/health
 ```
 
 ### OVN Chassis Not Registering

docs/install-source/README.md

@@ -32,7 +32,7 @@ This guide builds Spinifex from source. For production deployments, the [binary
 **Requirements:**
 
 - Ubuntu 22.04+ / Debian 12+
-- Go 1.26.1+
+- Go 1.26.2+
 - GCC, make, pkg-config
 - QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
 
@@ -75,21 +75,23 @@ If your WAN is a physical NIC, choose one:
 ./scripts/setup-ovn.sh --management --macvlan --wan-iface=enp0s3
 ```
 
-## Step 4. Initialize
+## Step 4. Development initialisation
 
 ```bash
-sudo ./bin/spx admin init --node node1 --nodes 1
+./scripts/dev-install.sh
 ```
 
-Save the admin credentials printed during init.
+Note, this will bootstrap a single node development environment
 
-## Step 5. Start Services
+## Step 5. Start services
 
 ```bash
-./scripts/start-dev.sh
+sudo systemctl status spinifex.target
 ```
 
-## Step 6. Verify
+Start all services
+
+## Step 6. Verify Installation
 
 ```bash
 export AWS_PROFILE=spinifex

docs/setting-up-your-cluster/README.md

@@ -26,6 +26,7 @@ resources:
 - [4. Launch an Instance](#4-launch-an-instance)
 - [5. Connect via SSH](#5-connect-via-ssh)
 - [6. Managing Instances](#6-managing-instances)
+- [7. Launching the Web UI](#7-launching-the-web-ui)
 - [Troubleshooting](#troubleshooting)
 
 ---
@@ -166,14 +167,25 @@ Query available instance types for your hardware:
 aws ec2 describe-instance-types
 ```
 
-Launch an instance in the public subnet:
+Launch an instance in the public subnet, note this will select an instance type with 2 VCPU and 1024MB of memory.
+
+> **Note:** Replace `AMI_NAME` with the previously imported image above with the `ami-` prefix.
 
 ```bash
-AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
+AMI_NAME="ami-ubuntu-24.04-x86_64"
+
+AMI_ID=$(aws ec2 describe-images \
+  --filters "Name=name,Values=$AMI_NAME" \
+  --query 'Images | sort_by(@, &CreationDate) | [-1].ImageId' \
+  --output text)
+
+INSTANCE_TYPE=$(aws ec2 describe-instance-types \
+  --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
+  --output text)
 
 INSTANCE_ID=$(aws ec2 run-instances \
   --image-id $AMI_ID \
-  --instance-type t3.small \
+  --instance-type $INSTANCE_TYPE \
   --key-name spinifex-key \
   --subnet-id $SUBNET_ID \
   --count 1 \
@@ -256,6 +268,51 @@ On a multi-node cluster, instances are distributed across nodes:
 spx get vms
 ```
 
+## 7. Launching the Web UI
+
+Spinifex ships with a built-in web console — an alternative to the AWS CLI, SDKs, and Terraform, analogous to the AWS Management Console. Every action in the UI is a standard AWS SigV4 API call, so the same IAM policies and audit behaviour apply.
+
+![Spinifex Web UI](../spinifex-ui.jpg)
+
+### Open the Console
+
+The UI is served by each node on port `3000` over TLS. Replace `YOUR_NODE_IP` with the address of the node you installed Spinifex on (or `localhost` if you're on the node itself):
+
+```
+https://YOUR_NODE_IP:3000
+```
+
+### Trust the Self-Signed Certificate (required)
+
+On first load, your browser will show a TLS warning — Spinifex generates a self-signed certificate at install time. This is expected.
+
+1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
+2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
+3. Install the certificate as a **trusted root** on your workstation:
+
+   - **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
+   - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
+   - **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
+   - **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
+
+4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
+
+> **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
+
+### Log In with AWS Credentials
+
+The console authenticates against the AWS credentials in `~/.aws/credentials` on the node where Spinifex is installed:
+
+```ini
+[spinifex]
+aws_access_key_id     = AKIA...
+aws_secret_access_key = ...
+```
+
+At the Spinifex login screen, paste the **Access Key ID** and **Secret Access Key** from the `[spinifex]` profile (or whichever profile maps to the IAM user or role you want to use). Additional users and policies can be managed through the UI or via `aws iam` commands — see [IAM Users and Policies](/docs/iam-users-and-policies).
+
+Once logged in, you have browser-based access to every Spinifex feature: launch and manage instances, attach EBS volumes, browse S3 buckets, configure VPCs and security groups, and manage IAM users and keys — all backed by the same AWS-compatible control plane the CLI uses.
+
 ## Additional Options
 
 ### Private Subnets (No Public IP)
@@ -280,10 +337,6 @@ spx admin account create --name myteam
 export AWS_PROFILE=spinifex-myteam
 ```
 
-### Web UI
-
-The Spinifex web interface is available at `https://localhost:3000` when services are running.
-
 ## Troubleshooting
 
 ### Instance Stuck in Pending

docs/terraform/bastion-private-subnet/README.md

@@ -36,12 +36,9 @@ Deploy a VPC with public and private subnets where the private subnet has no int
 
 **Architecture:**
 
-```
-WAN ──SSH──▶ Bastion (public subnet)
-                 │
-                 ▼ SSH (private IP)
-              App Server (private subnet, no internet)
-```
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-bastion.svg" alt="Bastion architecture — WAN reaches bastion in public subnet, SSH hop to app server in private subnet" width="900">
+</p>
 
 **What you'll learn:**
 

docs/terraform/nginx-alb/README.md

@@ -0,0 +1,231 @@
+---
+title: "Nginx Web Server (Load Balanced)"
+description: "Deploy a VPC with two private EC2 instances running Nginx, fronted by an internet-facing ALB using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - nginx
+  - ec2
+  - elbv2
+  - alb
+  - vpc
+  - workbook
+resources:
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "OpenTofu"
+    url: "https://opentofu.org/"
+---
+
+# Terraform: Nginx Web Servers with ALB
+
+> Deploys two Nginx web server instances behind an Application Load Balancer on Spinifex.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Deploy two Nginx web servers behind an internet-facing Application Load Balancer on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC with public and private subnets, an internet gateway and NAT Gateway, route tables, security group, SSH key pair, an application load balancer (ALB) and two EC2 instances with cloud-init user-data that installs and starts Nginx. Only the ALB is reachable from outside the VPC — the Nginx instances live in the **private subnets** and reach the internet only for cloud-init bootstrapping via the NAT Gateway.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-nginx-alb.svg" alt="Nginx + ALB VPC — IGW, two public subnets with ALB ENIs and NAT GW, two private subnets hosting nginx" width="900">
+</p>
+
+**What you'll learn:**
+
+- Configuring the AWS Terraform provider to target Spinifex
+- Creating a VPC with public + private subnets, an IGW and a NAT Gateway
+- Provisioning a multi-AZ internet-facing ALB fronting private workers
+- Provisioning an EC2 instance with cloud-init user-data
+- Generating SSH key pairs with the TLS provider
+
+**What gets created**
+
+| Resource | Name | Purpose |
+|---|---|---|
+| VPC | `nginx-alb-vpc` | Isolated network (10.20.0.0/16) |
+| Public Subnets | `nginx-alb-public-a`, `nginx-alb-public-b` | Two AZs hosting the ALB and NAT Gateway |
+| Private Subnets | `nginx-alb-private-a`, `nginx-alb-private-b` | Two AZs hosting the Nginx workers |
+| Internet Gateway | `nginx-alb-igw` | Routes internet traffic for the public subnets |
+| Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
+| NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
+| Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
+| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Debian 12 with Nginx via cloud-init (private subnets) |
+| ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
+| Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
+| Listener | HTTP :80 | Forwards traffic to the target group |
+
+**Prerequisites:**
+
+- Spinifex installed and running (see [Installing Spinifex](/docs/install))
+- A Debian 12 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- OpenTofu or Terraform installed
+- `AWS_PROFILE=spinifex` configured
+
+## Instructions
+
+### Step 1. Get the Template
+
+Clone the Terraform examples from the Spinifex repository:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform
+cd docs/terraform/nginx-alb
+```
+
+Or create a `main.tf` file and paste the full configuration below.
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Install Load Balancer AMI
+
+Next install the load balancer AMI images, which is used as the AMI disk image for launching the load-balancer and a requirement.
+
+```bash
+spx admin images import --name lb-alpine-3.21.6-x86_64
+```
+
+### Step 3. Deploy
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+```
+
+
+### Step 4. Specify instance and apply
+
+Next, depending on your architecture and CPU/memory requirements you must specify an instance type to launch.
+
+Either specify an instance type directly (e.g Intel)
+
+```bash
+# AMD instance
+export TF_VAR_instance_type="t3a.small"
+
+# Or, Intel
+export TF_VAR_instance_type="t3.small"
+```
+
+Or alternatively, using the AWS CLI tool query your instance for available types (e.g Intel, AMD, ARM) that support 2 vCPUs and 1 GB RAM.
+
+```bash
+TF_VAR_instance_type=$(aws ec2 describe-instance-types \
+  --query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
+  --output text)
+```
+
+Next, apply and launch the template:
+
+```bash
+tofu apply
+```
+
+### Step 5. Verify
+
+> **Note:** EC2 instances can take 30+ seconds to boot after apply, and the NAT Gateway must be `available` before cloud-init on the workers can reach the apt repository. If the ALB returns 5xx or HTTP is unreachable, wait and retry — the target group health checks need a moment to mark both instances healthy once Nginx has installed.
+
+The ALB is internet-facing, but the DNS name Spinifex returns (`*.elb.spinifex.local`) will not resolve from your host. Fetch the ALB's public IP with the AWS CLI:
+
+```bash
+ALB_IP=$(aws elbv2 describe-load-balancers --names nginx-alb \
+  --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+  --output text)
+echo "ALB public IP: $ALB_IP"
+```
+
+Then hit the ALB — successive requests should alternate between Server 1 and Server 2:
+
+```bash
+curl http://$ALB_IP
+curl http://$ALB_IP
+```
+
+Open `http://$ALB_IP` in your browser and refresh to see the page alternate content served from each instance.
+
+The Nginx instances themselves only have private IPs (see the `instance_1_private_ip` / `instance_2_private_ip` outputs) and are only reachable from inside the VPC — go through the ALB.
+
+Check target health via AWS CLI:
+
+```bash
+TG_ARN=$(aws elbv2 describe-target-groups \
+  --query 'TargetGroups[0].TargetGroupArn' \
+  --output text)
+
+echo "Target Group ARN: $TG_ARN"
+
+aws elbv2 describe-target-health --target-group-arn $TG_ARN
+```
+
+### Cleanup
+
+```bash
+tofu destroy
+```
+
+## Troubleshooting
+
+### AMI Not Found
+
+Ensure you have imported a Debian 12 image. Check available AMIs:
+
+```bash
+aws ec2 describe-images --owners 000000000000 --profile spinifex
+```
+
+If missing import:
+
+```bash
+spx admin images import --name debian-12-x86_64
+```
+
+### Provider Connection Refused
+
+Verify Spinifex services are running:
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
+
+### ALB Returns 5xx / Targets Unhealthy
+
+Give the instances a moment to finish cloud-init (Nginx has to install before it can answer health checks). Check target health:
+
+```bash
+TG_ARN=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
+  --query 'TargetGroups[0].TargetGroupArn' --output text)
+aws elbv2 describe-target-health --target-group-arn "$TG_ARN"
+```
+
+If targets stay unhealthy, verify the instances are running:
+
+```bash
+aws ec2 describe-instances --profile spinifex
+```
+
+If cloud-init on the workers never finished, confirm the NAT Gateway is `available` (the private subnets rely on it for outbound apt access):
+
+```bash
+aws ec2 describe-nat-gateways --query 'NatGateways[].[NatGatewayId,State]'
+```
+
+### Nginx Not Responding
+
+The Nginx instances have no public IP, so you can't SSH in directly from your host. If you need to inspect cloud-init logs, launch a small jump host in the same VPC or run commands via the Spinifex console, then:
+
+```bash
+ssh -i nginx-alb-demo.pem ec2-user@<instance_private_ip>
+sudo journalctl -u cloud-init --no-pager
+sudo systemctl status nginx
+```

docs/terraform/nginx-alb/main.tf

@@ -0,0 +1,523 @@
+# Example: Nginx Web Servers with ALB on Spinifex
+#
+# Deploys a VPC with two public subnets (ALB + NAT Gateway) and two private
+# subnets hosting Nginx EC2 instances. An internet-facing Application Load
+# Balancer distributes HTTP traffic between the private workers, and the
+# NAT Gateway gives them outbound internet access so cloud-init can install
+# Nginx from the Debian apt repository.
+#
+# Demonstrates: VPC, public and private subnets, internet gateway, NAT
+# Gateway with Elastic IP, route tables, security group, key pair, cloud-init
+# user-data, EC2 instances, ALB, target group, and listener.
+#
+# Usage:
+#   cd spinifex/docs/terraform/nginx-alb
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#
+# After apply, fetch the ALB's public IP (the *.elb.spinifex.local DNS
+# name does not resolve from your host):
+#
+#   aws elbv2 describe-load-balancers --names nginx-alb \
+#     --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+#     --output text
+#
+# Then:
+#   curl http://<alb_public_ip>    # Load-balanced Nginx (alternates between instances)
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.65.0, < 5.66.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "instance_type" {
+  type = string
+  default = "t3.small"
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://localhost:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+# ---------------------------------------------------------------------------
+# Provider — point the AWS provider at Spinifex
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2                    = var.spinifex_endpoint
+    iam                    = var.spinifex_endpoint
+    sts                    = var.spinifex_endpoint
+    elasticloadbalancingv2 = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# ---------------------------------------------------------------------------
+# Data sources
+# ---------------------------------------------------------------------------
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+data "aws_ami" "debian12" {
+  most_recent = true
+  owners      = ["000000000000"] # Spinifex system images
+
+  filter {
+    name   = "name"
+    values = ["*debian-12*"]
+  }
+}
+
+# ---------------------------------------------------------------------------
+# SSH Key Pair
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "nginx" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "nginx" {
+  key_name   = "nginx-alb-demo"
+  public_key = tls_private_key.nginx.public_key_openssh
+}
+
+resource "local_file" "nginx_pem" {
+  filename        = "${path.module}/nginx-alb-demo.pem"
+  content         = tls_private_key.nginx.private_key_openssh
+  file_permission = "0600"
+}
+
+# ---------------------------------------------------------------------------
+# VPC
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "nginx-alb-vpc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Internet Gateway
+# ---------------------------------------------------------------------------
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "nginx-alb-igw"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Public Subnets (two AZs for the ALB and NAT Gateway)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "public_a" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.1.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-public-a"
+  }
+}
+
+resource "aws_subnet" "public_b" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.2.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-public-b"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Private Subnets (two AZs for the Nginx workers)
+# ---------------------------------------------------------------------------
+
+resource "aws_subnet" "private_a" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.11.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-private-a"
+  }
+}
+
+resource "aws_subnet" "private_b" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.20.12.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "nginx-alb-private-b"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# NAT Gateway — outbound internet for the private subnets
+#
+# Background plumbing: the private-subnet workers need outbound connectivity
+# during cloud-init so apt-get can install Nginx. A single NAT Gateway in
+# public_a provides SNAT for both private subnets via the private route table.
+# ---------------------------------------------------------------------------
+
+resource "aws_eip" "nat" {
+  domain = "vpc"
+
+  tags = {
+    Name = "nginx-alb-nat-eip"
+  }
+}
+
+resource "aws_nat_gateway" "main" {
+  allocation_id = aws_eip.nat.id
+  subnet_id     = aws_subnet.public_a.id
+
+  tags = {
+    Name = "nginx-alb-nat"
+  }
+
+  depends_on = [aws_internet_gateway.igw]
+}
+
+# ---------------------------------------------------------------------------
+# Route Tables — public subnets egress via IGW, private subnets via NAT GW
+# ---------------------------------------------------------------------------
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "nginx-alb-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public_a" {
+  subnet_id      = aws_subnet.public_a.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table_association" "public_b" {
+  subnet_id      = aws_subnet.public_b.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = aws_nat_gateway.main.id
+  }
+
+  tags = {
+    Name = "nginx-alb-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private_a" {
+  subnet_id      = aws_subnet.private_a.id
+  route_table_id = aws_route_table.private.id
+}
+
+resource "aws_route_table_association" "private_b" {
+  subnet_id      = aws_subnet.private_b.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# Security Group — SSH + HTTP inbound, all outbound
+# ---------------------------------------------------------------------------
+
+resource "aws_security_group" "web" {
+  name        = "nginx-alb-sg"
+  description = "Allow SSH and HTTP inbound"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "HTTP"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "All outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nginx-alb-sg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# EC2 Instances — two Nginx servers with distinct landing pages
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx_1" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = var.instance_type
+
+  subnet_id              = aws_subnet.private_a.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  # Workers pull nginx from apt via the NAT Gateway — creation must wait
+  # until the NAT Gateway is available so cloud-init can reach the repos.
+  depends_on = [aws_nat_gateway.main]
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    apt-get update -y
+    apt-get install -y nginx
+
+    INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id 2>/dev/null || hostname)
+    cat > /var/www/html/index.html <<HTML
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex ALB Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p><strong>Instance:</strong> $INSTANCE_ID (Server 1)</p>
+      <p>This Nginx server is behind an Application Load Balancer.</p>
+      <hr>
+      <p><small>Provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-alb-1"
+  }
+}
+
+resource "aws_instance" "nginx_2" {
+  ami           = data.aws_ami.debian12.id
+  instance_type = var.instance_type
+
+  subnet_id              = aws_subnet.private_b.id
+  vpc_security_group_ids = [aws_security_group.web.id]
+  key_name               = aws_key_pair.nginx.key_name
+
+  # Workers pull nginx from apt via the NAT Gateway — creation must wait
+  # until the NAT Gateway is available so cloud-init can reach the repos.
+  depends_on = [aws_nat_gateway.main]
+
+  user_data_base64 = base64encode(<<-USERDATA
+    #!/bin/bash
+    set -euo pipefail
+
+    apt-get update -y
+    apt-get install -y nginx
+
+    INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id 2>/dev/null || hostname)
+    cat > /var/www/html/index.html <<HTML
+    <!DOCTYPE html>
+    <html>
+    <head><title>Spinifex ALB Demo</title></head>
+    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
+      <h1>Hello from Spinifex!</h1>
+      <p><strong>Instance:</strong> $INSTANCE_ID (Server 2)</p>
+      <p>This Nginx server is behind an Application Load Balancer.</p>
+      <hr>
+      <p><small>Provisioned via cloud-init user-data.</small></p>
+    </body>
+    </html>
+    HTML
+
+    systemctl enable nginx
+    systemctl restart nginx
+  USERDATA
+  )
+
+  tags = {
+    Name = "nginx-alb-2"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Application Load Balancer
+# ---------------------------------------------------------------------------
+
+resource "aws_lb" "web" {
+  name               = "nginx-alb"
+  internal           = false
+  load_balancer_type = "application"
+  security_groups    = [aws_security_group.web.id]
+  subnets            = [aws_subnet.public_a.id, aws_subnet.public_b.id]
+
+  tags = {
+    Name = "nginx-alb"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Target Group — HTTP health-checked on port 80
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_target_group" "nginx" {
+  name     = "nginx-alb-tg"
+  port     = 80
+  protocol = "HTTP"
+  vpc_id   = aws_vpc.main.id
+
+  health_check {
+    path                = "/"
+    protocol            = "HTTP"
+    healthy_threshold   = 2
+    unhealthy_threshold = 3
+    timeout             = 5
+    interval            = 10
+  }
+
+  tags = {
+    Name = "nginx-alb-tg"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Register both instances as targets
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_target_group_attachment" "nginx_1" {
+  target_group_arn = aws_lb_target_group.nginx.arn
+  target_id        = aws_instance.nginx_1.id
+  port             = 80
+}
+
+resource "aws_lb_target_group_attachment" "nginx_2" {
+  target_group_arn = aws_lb_target_group.nginx.arn
+  target_id        = aws_instance.nginx_2.id
+  port             = 80
+}
+
+# ---------------------------------------------------------------------------
+# Listener — forward HTTP :80 to the target group
+# ---------------------------------------------------------------------------
+
+resource "aws_lb_listener" "http" {
+  load_balancer_arn = aws_lb.web.arn
+  port              = 80
+  protocol          = "HTTP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.nginx.arn
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "note" {
+  value = <<-EOT
+    EC2 instances can take 30+ seconds to boot after apply — if HTTP is
+    unreachable, wait and retry.
+
+    The Nginx instances have private IPs only. The ALB DNS name ends in
+    .elb.spinifex.local and will not resolve from your host, so fetch the
+    ALB's public IP with:
+
+      aws elbv2 describe-load-balancers --names nginx-alb \
+        --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
+        --output text
+
+    Then: curl http://<that-ip>
+  EOT
+}
+
+output "alb_name" {
+  value = aws_lb.web.name
+}
+
+output "alb_arn" {
+  value = aws_lb.web.arn
+}
+
+output "alb_dns_name" {
+  value = aws_lb.web.dns_name
+}
+
+output "instance_1_id" {
+  value = aws_instance.nginx_1.id
+}
+
+output "instance_1_private_ip" {
+  value = aws_instance.nginx_1.private_ip
+}
+
+output "instance_2_id" {
+  value = aws_instance.nginx_2.id
+}
+
+output "instance_2_private_ip" {
+  value = aws_instance.nginx_2.private_ip
+}

docs/terraform/s3-webapp/README.md

@@ -38,12 +38,9 @@ Deploy an EC2 instance running a Flask file-sharing web application backed by S3
 
 **Architecture:**
 
-```
-Browser ──HTTP──▶ EC2 Instance (Flask webapp, port 80)
-                      │
-                      ▼ S3 API (boto3)
-                  Predastore (port 8443)
-```
+<p align="center">
+  <img src="../../../.github/assets/diagrams/tf-s3-webapp.svg" alt="S3 webapp — browser to Flask EC2 instance to Predastore via S3 API" width="900">
+</p>
 
 **What you'll learn:**
 

docs/update/README.md

@@ -0,0 +1,124 @@
+---
+title: "Updating Spinifex"
+description: "Upgrade an existing Spinifex installation to the latest release."
+category: "Getting Started"
+tags:
+  - update
+  - upgrade
+  - migrate
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Single-Node Install"
+    url: "https://docs.mulgadc.com/docs/install"
+---
+
+# Updating Spinifex
+
+> Upgrade an existing Spinifex installation to the latest release.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Manual Upgrade](#manual-upgrade)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Updating Spinifex is the same command used to install it. The installer detects an existing installation, downloads the latest binary and runs any pending configuration migrations before restarting services.
+
+For operators who want to review migrations before they are applied, a manual upgrade path is also supported.
+
+## Instructions
+
+## Step 1. Re-run the Installer
+
+```bash
+curl -fsSL https://install.mulgadc.com | bash
+```
+
+That's it. The installer will:
+
+1. Download and install the latest Spinifex binary.
+2. Reinstall systemd units so new services are picked up.
+3. Run any pending configuration migrations automatically (equivalent to `spx admin upgrade --yes`).
+4. Restart `spinifex.target` if the services were already running.
+
+## Step 2. Verify
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+If this returns a list of instance types, your upgrade is complete.
+
+## Manual Upgrade
+
+If you prefer to review pending migrations before they are applied, Spinifex supports running `spx admin init` to allow you to verify config file migrations.
+
+## Step 1. Install the New Binary Without Running Migrations
+
+```bash
+curl -fsSL https://install.mulgadc.com | INSTALL_SPINIFEX_SKIP_MIGRATE=1 bash
+```
+
+The installer will download the new binary and reinstall systemd units, but will **not** apply any configuration migrations.
+
+## Step 2. Review Pending Migrations
+
+```bash
+sudo spx admin upgrade
+```
+
+The command prints the current version of each config file, the migrations that would be applied, and a `from → to` description for each. It then prompts for confirmation before making any changes. Answer `n` to abort without touching your config.
+
+## Step 3. Apply Migrations
+
+When you are ready, answer `y` at the prompt, or re-run with `--yes` to apply non-interactively:
+
+```bash
+sudo spx admin upgrade --yes
+```
+
+## Step 4. Restart Services
+
+Migrations modify config files on disk but do not restart running services. Apply the new config with:
+
+```bash
+sudo systemctl restart spinifex.target
+```
+
+## Troubleshooting
+
+### No Pending Config Migrations
+
+```
+No pending config migrations.
+```
+
+Your config is already at the latest version. Nothing to do.
+
+### No Spinifex Installation Found
+
+```
+No Spinifex installation found at /etc/spinifex
+Run 'spx admin init' first.
+```
+
+`spx admin upgrade` requires an initialized installation. If this is a fresh host, follow the [Single-Node Install](/docs/install) guide instead.
+
+### Migration Failure
+
+If a migration fails, the installer and `spx admin upgrade` exit non-zero and leave the config in its prior state where possible. Review the error output, then re-run `sudo spx admin upgrade` once the underlying issue is resolved.
+
+### Services Did Not Pick Up New Config
+
+Migrations edit config files on disk but the running daemons continue to use the config they loaded at start-up. Restart with:
+
+```bash
+sudo systemctl restart spinifex.target
+```

docs/vpc-networking/README.md

@@ -38,38 +38,9 @@ Spinifex maps AWS VPC concepts directly to OVN constructs:
 
 ## Network Path
 
-```
-                        Physical Network (WAN)
-                               │
-                        ┌──────┴──────┐
-                        │ br-wan │  OVS bridge (physical uplink)
-                        └──────┬──────┘
-                               │  ovn-bridge-mappings: external:br-wan
-                               │
-                    ┌──────────┴───────────────────────────┐
-                    │     VPC Logical Router               │
-                    │                                      │
-                    │  Gateway port ──── ext switch ─────── localnet port
-                    │    (public IP)     (OVN)              (maps to br-wan)
-                    │                                      │
-                    │  NAT rules:                          │
-                    │    SNAT: VPC CIDR → gateway IP       │
-                    │    dnat_and_snat: public ↔ private   │
-                    │                                      │
-                    │  Routes:                             │
-                    │    0.0.0.0/0 → WAN gateway           │
-                    │                                      │
-                    ├────────────┬──────────────┤          │
-                    │            │              │          │
-              ┌─────┴────┐ ┌─────┴────┐         │          │
-              │ subnet-a │ │ subnet-b │         │          │
-              │ (public) │ │ (private)│         │          │
-              │10.0.1/24 │ │10.0.2/24 │         │          │
-              │  eni-1   │ │  eni-3   │         │          │
-              │  eni-2   │ │  eni-4   │         │          │
-              └──────────┘ └──────────┘         │          │
-                                                └──────────┘
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-network-path.svg" alt="VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs" width="900">
+</p>
 
 Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay
 NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int`
@@ -86,9 +57,9 @@ Instances get a private IP only. They can communicate with other instances in th
 same VPC (even across subnets and hosts via the overlay). They cannot reach the
 internet or be reached from the WAN.
 
-```
-Instance (10.0.2.5) ──→ VPC Router ──→ [no default route] ──→ dropped
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
+</p>
 
 If the VPC has an IGW attached, private subnet instances CAN reach the internet
 via the VPC router's SNAT rule (outbound only — they share the gateway IP). They
@@ -99,10 +70,9 @@ still cannot be reached from the WAN because they have no public IP.
 Instances get both a private IP and a public IP. The public IP is a 1:1 NAT
 managed by OVN — the instance OS only sees its private IP.
 
-```
-Outbound: Instance (10.0.1.5) ──→ SNAT ──→ exits as 203.0.113.10
-Inbound:  203.0.113.10 ──→ DNAT ──→ arrives at 10.0.1.5
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-public-subnet-flow.svg" alt="Public subnet — outbound SNAT and inbound DNAT between private and public IPs" width="900">
+</p>
 
 **Requirements for a public subnet:**
 
@@ -191,25 +161,9 @@ dns_servers = ["192.168.1.1", "8.8.8.8"]
 Regardless of whether IPs come from a static range or DHCP, the OVN behavior
 is identical:
 
-```
-                          Two DHCP conversations (completely independent):
-
- ┌─────────────────────────────────────────────────────────────────────┐
- │ HOST ↔ ROUTER DHCP (on br-ext, via veth pair → br-wan)              │
- │   Spinifex asks router: "give me a public IP for this VM"           │
- │   Router responds: 192.168.1.75                                     │
- │   Spinifex stores IP, tells OVN: dnat_and_snat(192.168.1.75 ↔       │
- │                                                 172.31.0.4)         │
- │   (Only used in DHCP source mode. Static mode picks from range.)    │
- └─────────────────────────────────────────────────────────────────────┘
-
- ┌─────────────────────────────────────────────────────────────────────┐
- │ VM ↔ OVN DHCP (on tap → br-int → OVN logical switch)                │
- │   cloud-init asks: "what's my IP?"                                  │
- │   OVN DHCP responds: "you are 172.31.0.4"                           │
- │   VM never sees 192.168.1.x or the router                           │
- └─────────────────────────────────────────────────────────────────────┘
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-dhcp-conversations.svg" alt="Two independent DHCP conversations — host-to-router and VM-to-OVN" width="900">
+</p>
 
 ### Choosing Static vs DHCP
 
@@ -297,9 +251,9 @@ a WAN bridge is configured (auto-detected or via `--wan-bridge`, required for pu
 The connection between them is logical, not physical — OVN's `localnet` port type
 maps the logical external switch to `br-wan` via `ovn-bridge-mappings`.
 
-```
-VM TAP ──→ br-int ──→ OVN logical pipeline ──→ localnet ──→ br-wan ──→ WAN
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-datapath-localnet.svg" alt="Data path — VM TAP through br-int, OVN pipeline, localnet, br-wan, WAN" width="900">
+</p>
 
 ## Bridge Modes
 
@@ -312,10 +266,9 @@ is the fallback for single-NIC hosts where the WAN NIC is also the SSH NIC.
 The WAN NIC is added directly to `br-wan` as an OVS port. OVS sees all
 traffic on the wire — any MAC, any protocol. No filtering, no workarounds.
 
-```
-Management NIC (host IP, SSH)
-WAN NIC ──→ br-wan (OVS port) ──→ OVN localnet
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-direct-bridge.svg" alt="Direct bridge mode — dedicated WAN NIC added directly to br-wan" width="900">
+</p>
 
 **Use when:** The host has a NIC dedicated to external/WAN traffic that is NOT
 used for SSH or management. Datacenter servers with separate management +
@@ -339,17 +292,9 @@ A macvlan sub-interface is created in bridge mode off the WAN NIC, and that
 macvlan is added to br-wan. The host keeps its IP on the original NIC —
 SSH stays up.
 
-```
-WAN NIC (host IP — unchanged)
-  │
-  ├── host traffic (SSH, management, overlay)
-  │
-  └── spx-ext-{nic} (macvlan, bridge mode, no IP)
-        │
-   br-wan
-        │
-   localnet → OVN ext switch
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-macvlan.svg" alt="Macvlan mode — WAN NIC shared between host traffic and macvlan child on br-wan" width="900">
+</p>
 
 **Use when:** The WAN NIC is the same NIC used for SSH. Single-NIC homelabs,
 edge deployments. Adding the NIC directly to OVS would break host connectivity.
@@ -451,24 +396,9 @@ ip link show spx-ext-eth0
 
 ## Network Flow Diagram
 
-```
-┌───────────────────────────────────────────────────┐
-│ Bare-Metal Host                                   │
-│                                                   │
-│  ┌──────────┐         ┌──────────┐                │
-│  │ br-int   │         │br-wan.   │                │
-│  │ (overlay)│         │  (WAN)   │                │
-│  │          │         │          │                │
-│  │ tap-eni1 │  OVN    │spx-ext-  │                │
-│  │ tap-eni2 ├──NAT───→│{nic}     │                │
-│  │ tap-eni3 │pipeline │(macvlan) │                │
-│  └──────────┘         └────┬─────┘                │
-│                            │ macvlan bridge mode  │
-│                        WAN NIC (host IP unchanged)│
-└───────────────────────┼───────────────────────────┘
-                        │
-                   Physical WAN
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-host-flow.svg" alt="Bare-metal host — br-int overlay, br-wan WAN bridge with macvlan, OVN NAT pipeline" width="900">
+</p>
 
 ## Configuration Reference
 
@@ -477,19 +407,9 @@ levels: cluster-wide mode, IP pool definitions, and per-node NIC settings.
 
 ## Configuration Levels
 
-```
-spinifex.toml
-├── [network]                        # Cluster-wide: which mode?
-│   └── external_mode = "pool"
-│
-├── [[network.external_pools]]       # IP ranges: what IPs to use?
-│   ├── name, range_start, range_end
-│   ├── gateway, prefix_len
-│   └── region, az (optional)
-│
-└── [nodes.NAME.vpcd]                # Per-node: which NIC?
-    └── external_interface
-```
+<p align="center">
+  <img src="../../.github/assets/diagrams/vpc-config-layers.svg" alt="spinifex.toml configuration layers — cluster mode, IP pools, per-node NIC" width="900">
+</p>
 
 ## Cluster-Wide: external_mode
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.1
+go 1.26.2
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
@@ -8,8 +8,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.1.4
-	github.com/mulgadc/viperblock v1.1.5
+	github.com/mulgadc/predastore v1.1.5
+	github.com/mulgadc/viperblock v1.1.6
 	github.com/nats-io/nats-server/v2 v2.12.6
 	github.com/nats-io/nats.go v1.50.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1

go.sum

@@ -29,34 +29,34 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
-github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
 github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk=
 github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ=
@@ -65,8 +65,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCd
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso=
-github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
-github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -255,10 +255,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mulgadc/predastore v1.1.4 h1:czeGQWWqmmZuGnUuvyrORavySN/SGPTwjUxBd/9swuU=
-github.com/mulgadc/predastore v1.1.4/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
-github.com/mulgadc/viperblock v1.1.5 h1:ojmoJQrY7VkDGzdQj07w2F7ICRJX9RlGAKyn87Wa55o=
-github.com/mulgadc/viperblock v1.1.5/go.mod h1:utoIQLwCmPujOBaT0mL5NhW3m2fwjQPQiQRJH6+Wtk8=
+github.com/mulgadc/predastore v1.1.5 h1:BjTEP2xbzIh0FswuCvnVcq9uy6l235lcIOwkGG/Z+f4=
+github.com/mulgadc/predastore v1.1.5/go.mod h1:la4jzK8lSLfUyUR5wNdVrkXXq8JQMGiKFvp/eXT0wjM=
+github.com/mulgadc/viperblock v1.1.6 h1:SzvpZIFWZVpWWdUSPD9k7fgmVAOsyXj9Ra686MyFsRc=
+github.com/mulgadc/viperblock v1.1.6/go.mod h1:KoJH5WFaZdIEYQ/uxugFKhQRLcMa9gmNOtyZjRHf/wI=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

scripts/build-alb-image.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-# build-alb-image.sh — Thin wrapper for backward compatibility
-# Delegates to the generic build-system-image.sh with the ALB manifest.
-exec "$(dirname "$0")/build-system-image.sh" "$(dirname "$0")/images/alb.conf" "$@"

scripts/build-system-image.sh

@@ -34,9 +34,13 @@ if [[ ! -f "$MANIFEST" ]]; then
 fi
 
 DO_IMPORT=false
-if [[ "${1:-}" == "--import" ]]; then
-    DO_IMPORT=true
-fi
+QUIET=false
+for arg in "$@"; do
+    case "$arg" in
+        --import) DO_IMPORT=true ;;
+        --quiet)  QUIET=true ;;
+    esac
+done
 
 # Source the manifest
 # shellcheck source=/dev/null
@@ -63,10 +67,17 @@ cleanup() {
     echo "Cleaning up..."
     sudo umount "${MOUNT_DIR}" 2>/dev/null || true
     sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
+    exec 9>&- 2>/dev/null || true  # release nbd lock if held
     echo "Done."
 }
 trap cleanup EXIT
 
+# In quiet mode, redirect build output to /dev/null (import output still shown)
+if [[ "$QUIET" == true ]]; then
+    exec 3>&1         # save original stdout
+    exec 1>/dev/null  # suppress build output
+fi
+
 echo "=== System Image Builder ==="
 echo "Image:   ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
 echo "Alpine:  ${ALPINE_VERSION}"
@@ -112,6 +123,39 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
     done
 fi
 
+# Serialize the entire image build with flock — concurrent builds on the same
+# host (e.g. CI single-node + multi-node jobs) share /dev/nbd0 and BUILD_DIR.
+NBD_LOCK="/tmp/build-system-image.lock"
+echo "Acquiring build lock..."
+exec 9>"$NBD_LOCK"
+flock 9
+echo "Lock acquired"
+
+# If the raw image was built recently (< 10 min), skip the entire build.
+# This avoids duplicate work when concurrent CI jobs build the same image.
+if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") )) -lt 600 ]]; then
+    echo "=== Skipping build — $OUTPUT_RAW is fresh (< 10 min old) ==="
+
+    # Restore stdout if suppressed, then jump to import
+    if [[ "$QUIET" == true ]]; then
+        exec 1>&3 3>&-
+    fi
+
+    echo ""
+    echo "=== Build complete (cached) ==="
+    echo "  raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
+
+    if [[ "$DO_IMPORT" == true ]]; then
+        echo "Importing as AMI..."
+        IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+        if [[ -n "${SYSTEM_TAG:-}" ]]; then
+            IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
+        fi
+        (cd "$PROJECT_DIR" && ./bin/spx admin images import "${IMPORT_ARGS[@]}")
+    fi
+    exit 0
+fi
+
 mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
 
 # Step 1: Download Alpine cloud image
@@ -133,6 +177,7 @@ else
 fi
 
 # Step 2: Copy image for customization
+rm -f "$OUTPUT_IMAGE"
 echo "Copying image for customization..."
 cp "${BUILD_DIR}/${ALPINE_IMAGE}" "$OUTPUT_IMAGE"
 
@@ -148,6 +193,14 @@ if [[ ! -e "${NBD_DEV}" ]]; then
 fi
 sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
 sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
+
+# Wait for the nbd device to be ready (kernel needs time to scan the block device)
+for i in $(seq 1 10); do
+    if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
+        break
+    fi
+    sleep 1
+done
 sleep 1
 
 # Alpine cloud images have ext4 directly on the block device (no partition table).
@@ -251,6 +304,11 @@ sudo qemu-nbd --disconnect "${NBD_DEV}"
 echo "Converting to raw format..."
 qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
 
+# Restore stdout if suppressed
+if [[ "$QUIET" == true ]]; then
+    exec 1>&3 3>&-
+fi
+
 echo ""
 echo "=== Build complete ==="
 echo "  Image: ${IMAGE_NAME}"
@@ -260,16 +318,19 @@ echo ""
 
 if [[ "$DO_IMPORT" == true ]]; then
     echo "Importing as AMI..."
-    (cd "$PROJECT_DIR" && ./bin/spx admin images import \
-        --file "$OUTPUT_RAW" \
-        --distro alpine \
-        --version "${ALPINE_VERSION}-${IMAGE_NAME}" \
-        --arch x86_64 \
-        --config "$HOME/spinifex/config/spinifex.toml")
+    IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro alpine --version "${ALPINE_VERSION}" --arch x86_64)
+    if [[ -n "${SYSTEM_TAG:-}" ]]; then
+        IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
+    fi
+    (cd "$PROJECT_DIR" && sudo -u spinifex-storage ./bin/spx admin images import "${IMPORT_ARGS[@]}")
 else
     echo "To import as AMI, run:"
     echo "  cd $PROJECT_DIR && ./bin/spx admin images import \\"
     echo "    --file $OUTPUT_RAW \\"
-    echo "    --distro alpine --version ${ALPINE_VERSION}-${IMAGE_NAME} --arch x86_64 \\"
-    echo "    --config \$HOME/spinifex/config/spinifex.toml"
+    if [[ -n "${SYSTEM_TAG:-}" ]]; then
+        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \\"
+        echo "    --tag ${SYSTEM_TAG}"
+    else
+        echo "    --distro alpine --version ${ALPINE_VERSION} --arch x86_64"
+    fi
 fi

scripts/create-multi-node.sh

@@ -92,7 +92,7 @@ sleep 5
 
 echo "Daemon health:"
 for IP in 127.0.0.1 127.0.0.2 127.0.0.3; do
-  echo "  $IP: $(curl -s http://$IP:4432/health | head -c 100)"
+  echo "  $IP: $(curl -sk https://$IP:4432/health | head -c 100)"
 done
 
 echo ""

scripts/dev-install.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# dev-install.sh — Full local development setup via production installer.
+# Builds from source, assembles a tarball, runs setup.sh for scaffolding
+# (user creation, directories, systemd units), initializes the cluster,
+# and starts services via systemd.
+#
+# Usage: ./scripts/dev-install.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
+
+echo "=== Building ==="
+cd "$PROJECT_ROOT" && make build
+cd "$MULGA_ROOT/viperblock" && make go_build_nbd
+
+echo "=== Assembling tarball for setup.sh ==="
+STAGING=$(mktemp -d)
+trap 'rm -rf "$STAGING"' EXIT
+cp "$PROJECT_ROOT/bin/spx" "$STAGING/spx"
+cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
+cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
+mkdir -p "$STAGING/systemd"
+cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
+mkdir -p "$STAGING/scripts"
+cp "$PROJECT_ROOT/build/scripts/"* "$STAGING/scripts/"
+cp "$PROJECT_ROOT/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
+tar czf /tmp/spinifex-local.tar.gz -C "$STAGING" .
+
+echo "=== Cleaning stale state from previous installs ==="
+# Stop any running services before modifying files
+sudo systemctl stop spinifex.target 2>/dev/null || true
+sudo systemctl reset-failed 2>/dev/null || true
+
+# Remove stale files owned by the dev user in production paths.
+# Previous dev-mode installs (admin init without sudo, start-dev.sh) leave
+# files owned by tf-user that service users (spinifex-nats, etc.) can't read
+# under systemd's ProtectSystem=strict sandboxing.
+for dir in /var/lib/spinifex /var/log/spinifex /etc/spinifex; do
+    if [ -d "$dir" ]; then
+        # Remove PID files, stale logs, and the legacy ~/spinifex/config symlink
+        sudo find "$dir" -name '*.pid' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log' -delete 2>/dev/null || true
+        sudo find "$dir" -name '*.log.*' -delete 2>/dev/null || true
+    fi
+done
+# Remove legacy data dir contents that conflict with production layout
+if [ -d /var/lib/spinifex/config ]; then
+    sudo rm -rf /var/lib/spinifex/config
+fi
+
+echo "=== Running setup.sh (creates users, dirs, systemd units) ==="
+sudo INSTALL_SPINIFEX_TARBALL=/tmp/spinifex-local.tar.gz bash "$PROJECT_ROOT/scripts/setup.sh"
+rm -f /tmp/spinifex-local.tar.gz
+
+echo "=== Setting up OVN ==="
+sudo /usr/local/share/spinifex/setup-ovn.sh --management
+
+echo "=== Initializing ==="
+sudo spx admin init --force --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1
+
+echo "=== Installing CA certificate ==="
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
+sudo update-ca-certificates
+
+echo "=== Starting services ==="
+sudo systemctl start spinifex.target
+
+# Check if user needs to re-login for spinifex group membership
+if ! id -Gn 2>/dev/null | grep -qw spinifex; then
+    newgrp spinifex
+    # echo ""
+    # echo "NOTE: You were added to the 'spinifex' group but your current"
+    # echo "session doesn't reflect it yet. Log out and back in, or run:"
+    # echo "  newgrp spinifex"
+fi
+
+echo "=== Building and importing LB image ==="
+cd "$PROJECT_ROOT" && make build-lb-agent
+"$PROJECT_ROOT/scripts/build-system-image.sh" "$PROJECT_ROOT/scripts/images/lb.conf" --import --quiet
+
+echo "=== Done ==="
+echo "Services: sudo systemctl status spinifex.target"
+echo "Logs:     journalctl -u 'spinifex-*' -f"
+echo "Test:     AWS_PROFILE=spinifex aws ec2 describe-instances"
+echo "Iterate:  make deploy (rebuild + restart all)"

scripts/dev-setup.sh

@@ -165,24 +165,16 @@ else
 fi
 
 echo ""
-echo "🎉 Development environment setup complete!"
+echo "Development environment setup complete!"
 echo ""
-echo "📋 Next steps:"
+echo "Recommended: Use scripts/dev-install.sh for a production-parity setup"
+echo "  (systemd, privilege separation, production paths)."
 echo ""
-echo "   1. Setup OVN networking:"
-echo "      ./scripts/setup-ovn.sh --management"
+echo "  scripts/dev-install.sh  # One-time: build, install, init, start via systemd"
+echo "  make deploy             # Daily: rebuild + copy binary + restart all"
+echo "  make reinstall          # Changed a .service file? Re-run full setup"
 echo ""
-echo "   2. Initialize Spinifex (creates ~/spinifex/ directories, config, certs, and installs CA):"
-echo "      sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
-echo ""
-echo "   3. Start services:"
-echo "      ./scripts/start-dev.sh"
-echo ""
-echo "🔧 Development endpoints:"
-echo "   - Spinifex Gateway:  https://localhost:$SPINIFEX_GATEWAY_PORT"
-echo "   - Predastore S3: https://localhost:$PREDASTORE_PORT"
-echo "   - NATS:          nats://localhost:$NATS_PORT"
-echo ""
-echo "🧪 Test with AWS CLI:"
-echo "   export AWS_PROFILE=spinifex"
-echo "   aws ec2 describe-instances"
+echo "Legacy workflow (deprecated, retained for pseudo-multinode CI only):"
+echo "  ./scripts/setup-ovn.sh --management"
+echo "  sudo ./bin/spx admin init --region ap-southeast-2 --az ap-southeast-2a --node node1 --nodes 1"
+echo "  ./scripts/start-dev.sh"

scripts/iac/configure-cluster.sh

@@ -176,7 +176,7 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do
     printf "    node%d (%s): " "$((i + 1))" "$ip"
     attempts=0
     while true; do
-        health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null || echo "")
+        health=$(curl -sk --connect-timeout 3 "https://$ip:4432/health" 2>/dev/null || echo "")
         if [ -n "$health" ]; then
             echo "healthy"
             break

scripts/iac/proxmox-template/README.md

@@ -7,7 +7,7 @@ Builds a Proxmox VM template with all Spinifex dependencies pre-installed. Cloni
 | Component | Details |
 |---|---|
 | System packages | QEMU, nbdkit, libvirt, gcc, jq, curl, etc. (`make install-system`) |
-| Go | 1.26.1 (`make install-go`) |
+| Go | 1.26.2 (`make install-go`) |
 | AWS CLI | v2 (`make install-aws`) |
 | Repositories | spinifex, viperblock, predastore (with `go.work` configured) |
 | Module cache | `go mod download` completed for all repos |

scripts/iac/spinifex-test.sh

@@ -185,7 +185,7 @@ cmd_status() {
         printf "  %-20s mgmt=%-16s data=%-16s " "$name" "$ip" "$data_ip"
 
         local health
-        if health=$(curl -s --connect-timeout 3 "http://$ip:4432/health" 2>/dev/null); then
+        if health=$(curl -sk --connect-timeout 3 "https://$ip:4432/health" 2>/dev/null); then
             echo "health=$health"
         else
             echo "health=unreachable"

scripts/images/alb-setup.sh

@@ -1,46 +0,0 @@
-#!/bin/sh
-# alb-setup.sh — ALB-specific chroot setup
-#
-# Runs inside the Alpine image chroot after packages are installed.
-# Sets up HAProxy placeholder config and alb-agent OpenRC init script.
-
-set -e
-
-# Create haproxy config directory and placeholder config
-mkdir -p /etc/haproxy
-cat > /etc/haproxy/haproxy.cfg <<'EOF'
-# Placeholder config — replaced by alb-agent on first config push
-global
-    daemon
-    maxconn 256
-
-defaults
-    mode http
-    timeout connect 5s
-    timeout client 30s
-    timeout server 30s
-
-frontend health
-    bind *:8405
-    http-request return status 200 content-type text/plain string "ok"
-EOF
-
-# Create alb-agent OpenRC init script
-mkdir -p /etc/init.d
-cat > /etc/init.d/alb-agent <<'INITSCRIPT'
-#!/sbin/openrc-run
-
-description="ALB NATS Config Agent"
-command="/usr/local/bin/alb-agent"
-command_args="--lb-id=${ALB_LB_ID:-unknown} --nats=${ALB_NATS_URL:-nats://10.0.2.2:4222}"
-command_background=true
-pidfile="/run/alb-agent.pid"
-output_log="/var/log/alb-agent.log"
-error_log="/var/log/alb-agent.log"
-
-depend() {
-    need net
-    after firewall
-}
-INITSCRIPT
-chmod 755 /etc/init.d/alb-agent

scripts/images/alb.conf

@@ -1,22 +0,0 @@
-# ALB system image manifest
-# Used by build-system-image.sh to create an Alpine AMI with HAProxy + alb-agent
-
-IMAGE_NAME="alb"
-IMAGE_DESCRIPTION="ALB VM with HAProxy and alb-agent"
-ALPINE_VERSION="3.21.6"
-IMAGE_SIZE="512M"
-
-# Packages to install via apk
-APK_PACKAGES="haproxy curl"
-
-# Binaries to copy into the image (source:destination)
-INSTALL_BINARIES="bin/alb-agent:/usr/local/bin/alb-agent"
-
-# Build command for binaries (run before image build)
-BUILD_COMMANDS="make build-alb-agent"
-
-# OpenRC services to enable at boot
-ENABLE_SERVICES="haproxy"
-
-# Custom setup script (runs inside chroot after packages are installed)
-SETUP_SCRIPT="scripts/images/alb-setup.sh"

scripts/images/lb-setup.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+# lb-setup.sh — LB-specific chroot setup
+#
+# Runs inside the Alpine image chroot after packages are installed.
+# Sets up HAProxy placeholder config and lb-agent OpenRC init script.
+# The lb-agent polls the AWS gateway for config updates and reports health
+# via SigV4-signed HTTP requests. Credentials come from cloud-init env vars.
+
+set -e
+
+# Create haproxy config directory and placeholder config.
+mkdir -p /etc/haproxy
+cat > /etc/haproxy/haproxy.cfg <<'EOF'
+# Placeholder config — replaced by lb-agent on first config fetch
+global
+    daemon
+    maxconn 256
+
+defaults
+    mode http
+    timeout connect 5s
+    timeout client 30s
+    timeout server 30s
+EOF
+
+# Create lb-agent OpenRC init script
+mkdir -p /etc/init.d
+cat > /etc/init.d/lb-agent <<'INITSCRIPT'
+#!/sbin/openrc-run
+
+description="LB Gateway Config Agent"
+command="/usr/local/bin/lb-agent"
+command_args="--lb-id=${LB_LB_ID:-unknown} --gateway=${LB_GATEWAY_URL} --access-key=${LB_ACCESS_KEY} --secret-key=${LB_SECRET_KEY}"
+command_background=true
+pidfile="/run/lb-agent.pid"
+output_log="/var/log/lb-agent.log"
+error_log="/var/log/lb-agent.log"
+
+depend() {
+    need net
+    after firewall
+}
+INITSCRIPT
+chmod 755 /etc/init.d/lb-agent
+
+# Move networking from boot to default runlevel so cloud-init can write
+# the network config before the networking service starts. Without this,
+# Alpine's networking service races cloud-init and starts with no config.
+rc-update del networking boot 2>/dev/null || true
+rc-update add networking default
+
+# Do NOT enable lb-agent at boot — cloud-init must write
+# /etc/conf.d/lb-agent (with LB_LB_ID, LB_GATEWAY_URL, LB_ACCESS_KEY,
+# LB_SECRET_KEY) before the service starts.
+# Cloud-init runcmd starts the service after write_files.

scripts/images/lb.conf

@@ -0,0 +1,26 @@
+# LB system image manifest
+# Used by build-system-image.sh to create an Alpine AMI with HAProxy + lb-agent
+
+IMAGE_NAME="lb"
+IMAGE_DESCRIPTION="LB VM with HAProxy and lb-agent"
+ALPINE_VERSION="3.21.6"
+IMAGE_SIZE="256M"
+
+# Tag applied to the imported AMI so the daemon can discover this as the
+# system image for ELBv2 load balancers. Must match tags.ManagedByELBv2.
+SYSTEM_TAG="spinifex:managed-by=elbv2"
+
+# Packages to install via apk
+APK_PACKAGES="haproxy curl ca-certificates"
+
+# Binaries to copy into the image (source:destination)
+INSTALL_BINARIES="bin/lb-agent:/usr/local/bin/lb-agent"
+
+# Build command for binaries (run before image build)
+BUILD_COMMANDS="make build-lb-agent"
+
+# OpenRC services to enable at boot
+ENABLE_SERVICES="haproxy"
+
+# Custom setup script (runs inside chroot after packages are installed)
+SETUP_SCRIPT="scripts/images/lb-setup.sh"

scripts/setup-ovn.sh

@@ -481,7 +481,19 @@ echo "  ovn-encap-type: geneve"
 echo ""
 echo "Step 5: Starting ovn-controller..."
 
-sudo systemctl start ovn-controller
+# Set ovn-controller file log level to WARN so it doesn't spam the log with
+# connection-retry INFO messages ("OVNSB commit failed") when the SB DB
+# isn't running. Uses a systemd ExecStartPost so it persists across restarts.
+OVN_CTRL_OVERRIDE="/etc/systemd/system/ovn-controller.service.d/log-level.conf"
+sudo mkdir -p "$(dirname "$OVN_CTRL_OVERRIDE")"
+sudo tee "$OVN_CTRL_OVERRIDE" >/dev/null <<'OVERRIDE'
+[Service]
+ExecStartPost=/bin/sh -c 'OVS_RUNDIR=/var/run/ovn exec /usr/bin/ovs-appctl -t ovn-controller vlog/set file:warn'
+OVERRIDE
+sudo systemctl daemon-reload
+echo "  ovn-controller log level: file:warn (via systemd drop-in)"
+
+sudo systemctl restart ovn-controller
 echo "  ovn-controller: started"
 
 # --- Step 6: Sysctl tuning ---
@@ -565,59 +577,43 @@ else
     echo "  systemd override: already exists"
 fi
 
-# Create sudoers rule for network commands that always need root
-# (ip tuntap, ip link set — NET_ADMIN operations)
+# Sudoers rules for spinifex-daemon and spinifex-vpcd are managed by setup.sh
+# (install_sudoers). Skip writing here to avoid conflicts.
 SUDOERS_FILE="/etc/sudoers.d/spinifex-network"
-if [ ! -f "$SUDOERS_FILE" ]; then
-    CURRENT_USER=$(whoami)
-    sudo tee "$SUDOERS_FILE" >/dev/null <<EOF
-# Spinifex VPC networking: allow non-root daemon to manage tap devices and OVS
-$CURRENT_USER ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-$CURRENT_USER ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
-EOF
-    sudo chmod 0440 "$SUDOERS_FILE"
-    echo "  sudoers rule: created ($SUDOERS_FILE)"
+if [ -f "$SUDOERS_FILE" ]; then
+    echo "  sudoers rule: already exists ($SUDOERS_FILE, managed by setup.sh)"
 else
-    echo "  sudoers rule: already exists"
+    echo "  sudoers rule: not found — run setup.sh first, or install manually"
 fi
 
 # --- Step 9: Configure OVN log rotation ---
-# OVN has built-in rotation that renames foo.log → foo.log.log. A previous
-# logrotate config used a *.log glob that caught those .log.log files too,
-# creating .log.log.1 files that accumulated to 27GB. Fix: use explicit
-# filenames so logrotate only touches the primary logs, not OVN's backups.
+# The ovn-common package provides /etc/logrotate.d/ovn-common which handles
+# rotation and vlog/reopen. We just add maxsize + rotate to cap disk usage.
 echo ""
 echo "Step 9: Configuring OVN log rotation..."
 
-# Clean up stale .log.log files from the old double-rotation bug
-if ls /var/log/ovn/*.log.log* 1>/dev/null 2>&1; then
-    sudo rm -f /var/log/ovn/*.log.log*
-    echo "  cleaned up stale .log.log files"
+OVN_LOGROTATE="/etc/logrotate.d/ovn-common"
+if [ -f "$OVN_LOGROTATE" ]; then
+    if ! grep -q 'maxsize' "$OVN_LOGROTATE"; then
+        sudo sed -i '/^\/var\/log\/ovn\/\*\.log {/a\    rotate 5\n    maxsize 100M' "$OVN_LOGROTATE"
+        echo "  added maxsize 100M + rotate 5 to $OVN_LOGROTATE"
+    else
+        echo "  $OVN_LOGROTATE already has maxsize configured"
+    fi
+else
+    echo "  WARNING: $OVN_LOGROTATE not found — install ovn-common package"
 fi
 
-LOGROTATE_FILE="/etc/logrotate.d/ovn-spinifex"
-sudo tee "$LOGROTATE_FILE" >/dev/null <<'LOGROTATE'
-/var/log/ovn/ovn-controller.log
-/var/log/ovn/ovn-northd.log
-/var/log/ovn/ovsdb-server-nb.log
-/var/log/ovn/ovsdb-server-sb.log
-{
-    daily
-    rotate 3
-    maxsize 100M
-    compress
-    missingok
-    notifempty
-    copytruncate
-}
-LOGROTATE
-echo "  logrotate config: $LOGROTATE_FILE (daily, 100M max, 3 rotations, explicit filenames)"
+# Remove our old custom config if present (superseded by patching ovn-common)
+if [ -f /etc/logrotate.d/ovn-spinifex ]; then
+    sudo rm -f /etc/logrotate.d/ovn-spinifex
+    echo "  removed obsolete /etc/logrotate.d/ovn-spinifex"
+fi
 
 # --- Step 10: Enable auto-start on boot ---
-# OVN services should start with the system in production. ovn-controller uses
-# exponential backoff if the SB DB isn't ready yet, so CPU impact is negligible.
-# In dev, stop-dev.sh explicitly stops OVN when not needed.
+# OVN services should start with the system in production. ovn-controller
+# retries when the SB DB isn't ready; file log level is set to WARN (Step 5)
+# to prevent log spam during those retries.
 echo ""
 echo "Step 10: Enabling OVN auto-start on boot..."
 sudo systemctl enable openvswitch-switch 2>/dev/null || true

scripts/setup.sh

@@ -89,27 +89,66 @@ detect_arch() {
     info "Detected architecture: $MACHINE ($ARCH)"
 }
 
-# --- Detect service user ---
-detect_service_user() {
-    # Use the real user — either the current user, or SUDO_USER if running via sudo
-    SPINIFEX_USER="${SUDO_USER:-$(whoami)}"
-    SPINIFEX_GROUP="$(id -gn "$SPINIFEX_USER")"
+# --- Create per-service system users ---
+create_service_users() {
+    SPINIFEX_GROUP="spinifex"
 
-    if [ "$SPINIFEX_USER" = "root" ]; then
-        fatal "Cannot determine service user. Do not run as root directly — use: curl ... | bash (as a normal user)"
+    # Create shared group
+    if ! getent group "$SPINIFEX_GROUP" > /dev/null 2>&1; then
+        $SUDO groupadd --system "$SPINIFEX_GROUP"
     fi
 
-    info "Services will run as: $SPINIFEX_USER:$SPINIFEX_GROUP"
+    # Create per-service users with correct home directories
+    declare -A SERVICE_HOMES=(
+        [nats]="/var/lib/spinifex/nats"
+        [gw]="/var/lib/spinifex/awsgw"
+        [daemon]="/var/lib/spinifex/spinifex"
+        [storage]="/var/lib/spinifex/predastore"
+        [viperblock]="/var/lib/spinifex/viperblock"
+        [vpcd]="/var/lib/spinifex"
+        [ui]="/var/lib/spinifex"
+    )
+    for svc in nats gw daemon storage viperblock vpcd ui; do
+        local user="spinifex-${svc}"
+        if ! id "$user" > /dev/null 2>&1; then
+            $SUDO useradd --system --no-create-home \
+                --home-dir "${SERVICE_HOMES[$svc]}" \
+                --gid "$SPINIFEX_GROUP" \
+                --shell /usr/sbin/nologin \
+                "$user"
+        fi
+    done
 
-    # Allow the service user to manage OVN/OVS and tap devices
-    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << SUDOERS
-# Spinifex VPC networking: allow service user to manage tap devices, OVS, and OVN
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
-${SPINIFEX_USER} ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+    # Add invoking user to spinifex group for admin CLI access
+    ADMIN_USER="${SUDO_USER:-$(whoami)}"
+    if [ "$ADMIN_USER" != "root" ]; then
+        $SUDO usermod -aG "$SPINIFEX_GROUP" "$ADMIN_USER"
+    fi
+
+    # KVM access for daemon
+    if getent group kvm > /dev/null 2>&1; then
+        $SUDO usermod -aG kvm spinifex-daemon
+    fi
+
+    info "Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})"
+}
+
+# --- Install scoped sudoers rules ---
+install_sudoers() {
+    $SUDO tee /etc/sudoers.d/spinifex-network > /dev/null << 'SUDOERS'
+# Spinifex daemon: tap devices, OVS bridge management, and DHCP for external IPs
+spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip
+spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd
+
+# Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl, /usr/bin/ovn-sbctl
+spinifex-vpcd ALL=(root) NOPASSWD: /usr/bin/systemctl is-active --quiet ovn-controller
 SUDOERS
     $SUDO chmod 0440 /etc/sudoers.d/spinifex-network
-    info "Sudoers rules installed for $SPINIFEX_USER"
+    $SUDO visudo -cf /etc/sudoers.d/spinifex-network || fatal "Invalid sudoers syntax in spinifex-network"
+    info "Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd"
 }
 
 # --- Install apt dependencies ---
@@ -234,18 +273,21 @@ install_files() {
         $SUDO install -m 0755 "$EXTRACT_DIR/setup-ovn.sh" /usr/local/share/spinifex/setup-ovn.sh
         info "  /usr/local/share/spinifex/setup-ovn.sh"
     fi
+
 }
 
 # --- Create directories ---
 create_directories() {
     info "Creating directories..."
 
+    # Top-level directories (root-owned, group-readable by spinifex)
     $SUDO mkdir -p /etc/spinifex
-    $SUDO chmod 0700 /etc/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex
+    $SUDO chmod 0750 /etc/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /etc/spinifex
 
     $SUDO mkdir -p /var/lib/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex
+    $SUDO chmod 0750 /var/lib/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/lib/spinifex
 
     # Symlink so services that expect BaseDir/config/ can find /etc/spinifex/
     if [ ! -e /var/lib/spinifex/config ]; then
@@ -258,21 +300,163 @@ create_directories() {
     fi
 
     $SUDO mkdir -p /var/log/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/log/spinifex
+    $SUDO chmod 0775 /var/log/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /var/log/spinifex
 
     $SUDO mkdir -p /run/spinifex
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /run/spinifex
+    $SUDO chmod 0775 /run/spinifex
+    $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex
+
+    $SUDO mkdir -p /run/spinifex/nbd
+    $SUDO chmod 0770 /run/spinifex/nbd
+    $SUDO chown "root:$SPINIFEX_GROUP" /run/spinifex/nbd
+
+    # Per-service config directories
+    $SUDO mkdir -p /etc/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
+    $SUDO chmod 0750 /etc/spinifex/nats
+
+    $SUDO mkdir -p /etc/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
+    $SUDO chmod 0750 /etc/spinifex/predastore
+
+    # Per-service data directories
+    $SUDO mkdir -p /var/lib/spinifex/nats
+    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
+    $SUDO chmod 0700 /var/lib/spinifex/nats
+
+    $SUDO mkdir -p /var/lib/spinifex/spinifex
+    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
+    $SUDO chmod 0700 /var/lib/spinifex/spinifex
+
+    $SUDO mkdir -p /var/lib/spinifex/predastore
+    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /var/lib/spinifex/predastore
+    $SUDO chmod 0700 /var/lib/spinifex/predastore
 
     $SUDO mkdir -p /var/lib/spinifex/viperblock
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /var/lib/spinifex/viperblock
+    $SUDO chmod 0700 /var/lib/spinifex/viperblock
 
+    $SUDO mkdir -p /var/lib/spinifex/vpcd
+    $SUDO chown "spinifex-vpcd:$SPINIFEX_GROUP" /var/lib/spinifex/vpcd
+    $SUDO chmod 0700 /var/lib/spinifex/vpcd
+
+    $SUDO mkdir -p /var/lib/spinifex/awsgw
+    $SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /var/lib/spinifex/awsgw
+    $SUDO chmod 0700 /var/lib/spinifex/awsgw
+
+    # Symlink so awsgw's {BaseDir}/config/ paths resolve to /etc/spinifex/
+    if [ ! -e /var/lib/spinifex/awsgw/config ]; then
+        $SUDO ln -s /etc/spinifex /var/lib/spinifex/awsgw/config
+    fi
+
+    # Top-level config files (root-owned, group-readable)
     # Generate environment file with install-specific values (e.g. arch-dependent paths)
     $SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
 # Generated by setup.sh — install-specific environment variables
 SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${PLUGINDIR}/nbdkit-viperblock-plugin.so
 EOF
-    $SUDO chown "$SPINIFEX_USER:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chown "spinifex-viperblock:$SPINIFEX_GROUP" /etc/spinifex/systemd.env
+    $SUDO chmod 0640 /etc/spinifex/systemd.env
     info "Generated /etc/spinifex/systemd.env"
+
+    # Service helper scripts (root-owned, group-executable by all service users)
+    if [ -d "$EXTRACT_DIR/scripts" ]; then
+        for script in "$EXTRACT_DIR"/scripts/*.sh; do
+            $SUDO install -o root -g "$SPINIFEX_GROUP" -m 0755 \
+                "$script" "/var/lib/spinifex/$(basename "$script")"
+            info "  /var/lib/spinifex/$(basename "$script")"
+        done
+    fi
+}
+
+# --- Fix file ownership for upgrades from v1 ---
+fix_file_ownership() {
+    info "Fixing file ownership for privilege separation..."
+
+    # Per-service data dirs — recursive chown so existing files are accessible
+    for entry in \
+        nats:spinifex-nats \
+        predastore:spinifex-storage \
+        spinifex:spinifex-daemon \
+        viperblock:spinifex-viperblock \
+        vpcd:spinifex-vpcd \
+        awsgw:spinifex-gw; do
+        IFS=: read -r dir svc_user <<< "$entry"
+        if [ -d "/var/lib/spinifex/$dir" ]; then
+            $SUDO chown -R "$svc_user:$SPINIFEX_GROUP" "/var/lib/spinifex/$dir" \
+                || fatal "Failed to set ownership on /var/lib/spinifex/$dir"
+        fi
+    done
+
+    # Per-service config dirs — recursive chown
+    if [ -d /etc/spinifex/nats ]; then
+        $SUDO chown -R "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats \
+            || fatal "Failed to set ownership on /etc/spinifex/nats"
+    fi
+    if [ -d /etc/spinifex/predastore ]; then
+        $SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
+            || fatal "Failed to set ownership on /etc/spinifex/predastore"
+    fi
+
+    # Shared config files — root:spinifex with per-file modes
+    for f in spinifex.toml master.key server.key; do
+        if [ -f "/etc/spinifex/$f" ]; then
+            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
+                || fatal "Failed to set ownership on /etc/spinifex/$f"
+            $SUDO chmod 0640 "/etc/spinifex/$f" \
+                || fatal "Failed to set permissions on /etc/spinifex/$f"
+        fi
+    done
+    for f in server.pem ca.pem; do
+        if [ -f "/etc/spinifex/$f" ]; then
+            $SUDO chown "root:$SPINIFEX_GROUP" "/etc/spinifex/$f" \
+                || fatal "Failed to set ownership on /etc/spinifex/$f"
+            $SUDO chmod 0644 "/etc/spinifex/$f" \
+                || fatal "Failed to set permissions on /etc/spinifex/$f"
+        fi
+    done
+
+    # CA private key — root-only
+    if [ -f /etc/spinifex/ca.key ]; then
+        $SUDO chown root:root /etc/spinifex/ca.key \
+            || fatal "Failed to set ownership on /etc/spinifex/ca.key"
+        $SUDO chmod 0600 /etc/spinifex/ca.key \
+            || fatal "Failed to set permissions on /etc/spinifex/ca.key"
+    fi
+
+    # Shared data dirs — root:spinifex 0770 (daemon + admin CLI write, services read).
+    # chmod must be recursive so pre-existing files (e.g. imported images originally
+    # written as 0600 by another user) become group-readable.
+    for d in images amis volumes state; do
+        if [ -d "/var/lib/spinifex/$d" ]; then
+            $SUDO chown -R "root:$SPINIFEX_GROUP" "/var/lib/spinifex/$d" \
+                || fatal "Failed to set ownership on /var/lib/spinifex/$d"
+            $SUDO chmod -R u+rwX,g+rwX,o-rwx "/var/lib/spinifex/$d" \
+                || fatal "Failed to set permissions on /var/lib/spinifex/$d"
+        fi
+    done
+
+    info "File ownership updated"
+}
+
+# --- Run config migrations ---
+run_migrations() {
+    # Only run if spinifex is already initialized (config exists)
+    if [ ! -f /etc/spinifex/spinifex.toml ]; then
+        info "Fresh install detected, skipping migrations"
+        return
+    fi
+
+    if [ "${INSTALL_SPINIFEX_SKIP_MIGRATE:-0}" = "1" ]; then
+        info "INSTALL_SPINIFEX_SKIP_MIGRATE=1, skipping migrations"
+        info "Run 'spx admin upgrade' manually to apply pending migrations"
+        return
+    fi
+
+    info "Running config migrations..."
+    $SUDO /usr/local/bin/spx admin upgrade --yes \
+        || fatal "Config migration failed. See errors above."
 }
 
 # --- Install systemd units ---
@@ -284,15 +468,13 @@ install_systemd() {
     fi
 
     for unit in "$EXTRACT_DIR"/systemd/*; do
-        # Substitute User= and Group= with the detected service user
-        sed "s/^User=spinifex$/User=$SPINIFEX_USER/;s/^Group=spinifex$/Group=$SPINIFEX_GROUP/" \
-            "$unit" | $SUDO tee "/etc/systemd/system/$(basename "$unit")" > /dev/null
-        $SUDO chmod 0644 "/etc/systemd/system/$(basename "$unit")"
+        $SUDO install -m 0644 "$unit" "/etc/systemd/system/$(basename "$unit")"
         info "  /etc/systemd/system/$(basename "$unit")"
     done
 
     $SUDO systemctl daemon-reload
-    info "Systemd units installed (running as $SPINIFEX_USER:$SPINIFEX_GROUP)"
+    $SUDO systemctl enable spinifex.target
+    info "Systemd units installed and enabled (per-service users)"
 }
 
 # --- Install logrotate ---
@@ -333,7 +515,7 @@ print_summary() {
     echo ""
     echo "  Version:      $INSTALLED_VERSION"
     echo "  Architecture: $ARCH"
-    echo "  Service user: $SPINIFEX_USER"
+    echo "  Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}"
     echo "  Binary:       /usr/local/bin/spx"
     echo "  Config:       /etc/spinifex/"
     echo "  Data:         /var/lib/spinifex/"
@@ -348,8 +530,6 @@ print_summary() {
     echo "     If your WAN is a physical NIC:"
     echo "       # Dedicated WAN NIC (not your SSH connection):"
     echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1"
-    echo "       # Single-NIC host (SSH-safe macvlan):"
-    echo "       sudo /usr/local/share/spinifex/setup-ovn.sh --management --macvlan --wan-iface=enp0s3"
     echo ""
     echo "  2. Initialize:"
     echo "     sudo spx admin init --node node1 --nodes 1"
@@ -374,15 +554,30 @@ main() {
     handle_upgrade
     install_apt_deps
     install_aws_cli
-    detect_service_user
+    create_service_users
+    install_sudoers
     download_spinifex
     install_files
     create_directories
+    fix_file_ownership
     install_systemd
+    run_migrations
     install_logrotate
     rm -rf "$SPINIFEX_TMPDIR"
     restart_if_needed
     print_summary
+
+    # Activate spinifex group membership in the invoking shell. Under curl|bash
+    # stdin is the drained pipe, so redirect from /dev/tty and exec so the new
+    # shell becomes the foreground process. Skip when we can't actually open
+    # /dev/tty (CI, cloud-init, ssh -T — stat passes but open fails with ENXIO).
+    if ! id -Gn 2>/dev/null | grep -qw "$SPINIFEX_GROUP" \
+        && ( : </dev/tty ) 2>/dev/null; then
+        echo ""
+        echo "  Activating '$SPINIFEX_GROUP' group in a subshell — type 'exit' when done."
+        echo ""
+        exec newgrp "$SPINIFEX_GROUP" < /dev/tty
+    fi
 }
 
 main "$@"

scripts/start-dev.sh

@@ -11,6 +11,11 @@
 
 set -euo pipefail
 
+echo "WARNING: start-dev.sh is deprecated for local development." >&2
+echo "  Use 'make deploy' for daily iteration or 'scripts/dev-install.sh' for first-time setup." >&2
+echo "  start-dev.sh is retained only for pseudo-multinode CI (e2e_pseudo_multi)." >&2
+echo "" >&2
+
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
@@ -33,6 +38,8 @@ echo "Using configuration directory: $CONFIG_DIR"
 LOGS_DIR="$DATA_DIR/logs"
 WAL_DIR="$DATA_DIR/spinifex"
 
+echo "⚠️  Dev mode: all services run as $(whoami) without privilege separation"
+echo ""
 echo "🚀 Starting Spinifex development environment..."
 echo "Project root: $PROJECT_ROOT"
 echo "Data directory: $DATA_DIR"
@@ -457,7 +464,7 @@ if is_multinode; then
         attempts=0
         max_attempts=5
         while [ $attempts -lt $max_attempts ]; do
-            if curl -s --connect-timeout 3 "http://$peer/health" > /dev/null 2>&1; then
+            if curl -sk --connect-timeout 3 "https://$peer/health" > /dev/null 2>&1; then
                 echo "   $peer: healthy"
                 break
             fi

scripts/workload-performance.sh

@@ -5,8 +5,8 @@
 sudo apt update
 sudo apt install -y make gcc git
 
-wget https://go.dev/dl/go1.26.1.linux-amd64.tar.gz
-sudo tar -C /usr/local -xzf go1.26.1.linux-amd64.tar.gz
+wget https://go.dev/dl/go1.26.2.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.26.2.linux-amd64.tar.gz
 
 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc
 source $HOME/.bashrc

spinifex/admin/admin.go

@@ -40,6 +40,7 @@ type ConfigSettings struct {
 	Region    string
 	NatsToken string
 	DataDir   string
+	LogDir    string
 	ConfigDir string
 
 	// Add more fields as needed
@@ -212,7 +213,9 @@ func CreateServiceDirectories(spxRoot string) {
 		filepath.Join(spxRoot, "nats"),
 		filepath.Join(spxRoot, "predastore"),
 		filepath.Join(spxRoot, "viperblock"),
+		filepath.Join(spxRoot, "vpcd"),
 		filepath.Join(spxRoot, "spinifex"),
+		filepath.Join(spxRoot, "awsgw"),
 	}
 
 	fmt.Println("\n📁 Creating directory structure...")
@@ -241,18 +244,30 @@ func FileExists(path string) bool {
 func ChownRecursive(path, username string) {
 	u, err := user.Lookup(username)
 	if err != nil {
+		slog.Warn("ChownRecursive: user lookup failed, skipping", "user", username, "path", path, "err", err)
+		return
+	}
+	uid, err := strconv.Atoi(u.Uid)
+	if err != nil {
+		slog.Warn("ChownRecursive: invalid UID, skipping", "user", username, "uid", u.Uid, "err", err)
+		return
+	}
+	gid, err := strconv.Atoi(u.Gid)
+	if err != nil {
+		slog.Warn("ChownRecursive: invalid GID, skipping", "user", username, "gid", u.Gid, "err", err)
 		return
 	}
-	uid, _ := strconv.Atoi(u.Uid)
-	gid, _ := strconv.Atoi(u.Gid)
 
 	// Use os.Root to scope filesystem operations and avoid symlink TOCTOU races.
 	// Falls back to direct chown if os.Root is not available.
 	root, rootErr := os.OpenRoot(path)
 	if rootErr != nil {
-		// Fallback: direct chown on the top-level path only
-		if chownErr := os.Chown(path, uid, gid); chownErr != nil { // #nosec G122
-			slog.Debug("chown failed", "path", path, "err", chownErr)
+		// Fallback: direct chown on the top-level path only — subdirectory contents
+		// will retain their original ownership.
+		slog.Warn("ChownRecursive: OpenRoot not available, only top-level directory ownership changed",
+			"path", path, "err", rootErr)
+		if chownErr := os.Lchown(path, uid, gid); chownErr != nil {
+			slog.Warn("chown failed", "path", path, "err", chownErr)
 		}
 		return
 	}
@@ -271,6 +286,81 @@ func ChownRecursive(path, username string) {
 	})
 }
 
+// SetServiceOwnership sets per-service ownership on data/config directories
+// and shared config files to root:spinifex with correct modes.
+// Keep in sync with setup.sh create_directories() and plan doc section 2.
+func SetServiceOwnership() {
+	grp, err := user.LookupGroup("spinifex")
+	if err != nil {
+		slog.Error("SetServiceOwnership: spinifex group not found, skipping all ownership changes", "err", err)
+		fmt.Fprintln(os.Stderr, "WARNING: spinifex group not found — service ownership not set. Run setup.sh first or create the group manually.")
+		return
+	}
+	gid, err := strconv.Atoi(grp.Gid)
+	if err != nil {
+		slog.Error("SetServiceOwnership: invalid spinifex group GID", "gid", grp.Gid, "err", err)
+		fmt.Fprintln(os.Stderr, "WARNING: invalid spinifex group GID — service ownership not set.")
+		return
+	}
+
+	// Per-service directory trees
+	for path, u := range map[string]string{
+		"/etc/spinifex/nats":           "spinifex-nats",
+		"/var/lib/spinifex/nats":       "spinifex-nats",
+		"/etc/spinifex/predastore":     "spinifex-storage",
+		"/var/lib/spinifex/predastore": "spinifex-storage",
+		"/var/lib/spinifex/spinifex":   "spinifex-daemon",
+		"/var/lib/spinifex/viperblock": "spinifex-viperblock",
+		"/var/lib/spinifex/vpcd":       "spinifex-vpcd",
+		"/var/lib/spinifex/awsgw":      "spinifex-gw",
+	} {
+		if _, err := os.Stat(path); err != nil {
+			continue
+		}
+		ChownRecursive(path, u)
+	}
+
+	// Shared data directories — root:spinifex 0770 so daemon + admin CLI can write
+	for _, dir := range []string{
+		"/var/lib/spinifex/images",
+		"/var/lib/spinifex/amis",
+		"/var/lib/spinifex/volumes",
+		"/var/lib/spinifex/state",
+		"/run/spinifex/nbd",
+	} {
+		if _, err := os.Stat(dir); err != nil {
+			continue
+		}
+		if err := os.Lchown(dir, 0, gid); err != nil {
+			slog.Warn("SetServiceOwnership: chown failed", "path", dir, "err", err)
+		}
+		if err := os.Chmod(dir, 0770); err != nil { //nolint:gosec // directories need group-write for daemon + admin CLI
+			slog.Warn("SetServiceOwnership: chmod failed", "path", dir, "err", err)
+		}
+	}
+
+	// Shared config files — root:spinifex, ca.key stays root:root 0600
+	// bootstrap.json lives in the awsgw data dir (not /etc/spinifex),
+	// so /etc/spinifex stays at 0750 (no group-write needed).
+	for path, mode := range map[string]os.FileMode{
+		"/etc/spinifex/spinifex.toml": 0640,
+		"/etc/spinifex/master.key":    0640,
+		"/etc/spinifex/server.pem":    0644,
+		"/etc/spinifex/server.key":    0640,
+		"/etc/spinifex/ca.pem":        0644,
+	} {
+		if _, err := os.Stat(path); err != nil {
+			continue
+		}
+		if err := os.Lchown(path, 0, gid); err != nil {
+			slog.Warn("SetServiceOwnership: chown failed", "path", path, "err", err)
+		}
+		if err := os.Chmod(path, mode); err != nil {
+			slog.Warn("SetServiceOwnership: chmod failed", "path", path, "err", err)
+		}
+	}
+}
+
 // updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
 func UpdateAWSINIFile(path, section string, values map[string]string) error {
 	var cfg *ini.File
@@ -753,7 +843,7 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 
 	if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
 		"region":       region,
-		"endpoint_url": fmt.Sprintf("https://%s:9999", endpointHost),
+		"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
 		"ca_bundle":    certPath,
 		"output":       "json",
 	}); err != nil {

spinifex/admin/telemetry.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
@@ -93,7 +93,7 @@ func ReadMachineID() string {
 	hostname, _ := os.Hostname()
 	mac := firstMAC()
 	hash := sha256.Sum256([]byte(hostname + mac))
-	return fmt.Sprintf("%x", hash[:16])
+	return hex.EncodeToString(hash[:16])
 }
 
 // firstMAC returns the MAC address of the first non-loopback, up interface.

spinifex/albagent/agent.go

@@ -1,305 +0,0 @@
-// Package albagent implements the NATS config agent that runs inside ALB VMs.
-// It subscribes to config updates, writes HAProxy configuration, and reloads
-// the HAProxy process.
-package albagent
-
-import (
-	"encoding/json"
-	"fmt"
-	"log/slog"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/nats-io/nats.go"
-)
-
-const (
-	// Default paths for HAProxy config and PID file.
-	DefaultConfigPath = "/etc/haproxy/haproxy.cfg"
-	DefaultPIDPath    = "/run/haproxy.pid"
-
-	// NATS topic patterns.
-	configTopicPrefix = "elbv2.alb."
-	configTopicSuffix = ".config"
-	pingTopicSuffix   = ".ping"
-)
-
-// Agent manages HAProxy configuration inside an ALB VM.
-type Agent struct {
-	lbID       string
-	natsURL    string
-	configPath string
-	pidPath    string
-
-	nc   *nats.Conn
-	subs []*nats.Subscription
-	mu   sync.Mutex
-
-	// For testing: override the reload function.
-	reloadFn func(configPath, pidPath string) error
-}
-
-// New creates a new ALB agent for the given load balancer.
-func New(lbID, natsURL string) (*Agent, error) {
-	if lbID == "" {
-		return nil, fmt.Errorf("lbID is required")
-	}
-	if natsURL == "" {
-		natsURL = "nats://127.0.0.1:4222"
-	}
-
-	return &Agent{
-		lbID:       lbID,
-		natsURL:    natsURL,
-		configPath: DefaultConfigPath,
-		pidPath:    DefaultPIDPath,
-		reloadFn:   reloadHAProxy,
-	}, nil
-}
-
-// ConfigTopic returns the NATS topic for config updates.
-func ConfigTopic(lbID string) string {
-	return configTopicPrefix + lbID + configTopicSuffix
-}
-
-// PingTopic returns the NATS topic for health pings.
-func PingTopic(lbID string) string {
-	return configTopicPrefix + lbID + pingTopicSuffix
-}
-
-// Start connects to NATS and subscribes to config and ping topics.
-func (a *Agent) Start() error {
-	nc, err := nats.Connect(a.natsURL,
-		nats.Name("alb-agent-"+a.lbID),
-		nats.MaxReconnects(-1),
-		nats.ReconnectWait(2*time.Second),
-		nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
-			slog.Warn("NATS disconnected", "err", err)
-		}),
-		nats.ReconnectHandler(func(_ *nats.Conn) {
-			slog.Info("NATS reconnected")
-		}),
-	)
-	if err != nil {
-		return fmt.Errorf("connect to NATS at %s: %w", a.natsURL, err)
-	}
-	a.nc = nc
-
-	// Subscribe to config updates
-	configSub, err := nc.Subscribe(ConfigTopic(a.lbID), a.handleConfig)
-	if err != nil {
-		nc.Close()
-		return fmt.Errorf("subscribe to config topic: %w", err)
-	}
-
-	// Subscribe to health pings (request/reply)
-	pingSub, err := nc.Subscribe(PingTopic(a.lbID), a.handlePing)
-	if err != nil {
-		nc.Close()
-		return fmt.Errorf("subscribe to ping topic: %w", err)
-	}
-
-	a.mu.Lock()
-	a.subs = append(a.subs, configSub, pingSub)
-	a.mu.Unlock()
-
-	// Flush to ensure the server has processed our subscriptions before
-	// we report that the agent is started and ready.
-	if err := nc.Flush(); err != nil {
-		nc.Close()
-		return fmt.Errorf("flush subscriptions: %w", err)
-	}
-
-	slog.Info("Agent started",
-		"lbId", a.lbID,
-		"configTopic", ConfigTopic(a.lbID),
-		"pingTopic", PingTopic(a.lbID),
-	)
-	return nil
-}
-
-// Stop gracefully shuts down the agent.
-func (a *Agent) Stop() {
-	a.mu.Lock()
-	defer a.mu.Unlock()
-
-	for _, sub := range a.subs {
-		if err := sub.Unsubscribe(); err != nil {
-			slog.Warn("Failed to unsubscribe", "err", err)
-		}
-	}
-	a.subs = nil
-
-	if a.nc != nil {
-		a.nc.Close()
-		a.nc = nil
-	}
-
-	slog.Info("Agent stopped", "lbId", a.lbID)
-}
-
-// handleConfig processes a config update message. The message payload is the
-// raw HAProxy config text. It writes the config to disk and reloads HAProxy.
-func (a *Agent) handleConfig(msg *nats.Msg) {
-	config := string(msg.Data)
-	if config == "" {
-		slog.Warn("Received empty config, ignoring")
-		a.respond(msg, "error", "empty config")
-		return
-	}
-
-	slog.Info("Received config update", "lbId", a.lbID, "size", len(config))
-
-	// Write config to disk
-	if err := WriteConfig(a.configPath, config); err != nil {
-		slog.Error("Failed to write config", "path", a.configPath, "err", err)
-		a.respond(msg, "error", err.Error())
-		return
-	}
-
-	// Reload HAProxy
-	if err := a.reloadFn(a.configPath, a.pidPath); err != nil {
-		slog.Error("Failed to reload HAProxy", "err", err)
-		a.respond(msg, "error", err.Error())
-		return
-	}
-
-	slog.Info("Config applied and HAProxy reloaded", "lbId", a.lbID)
-	a.respond(msg, "ok", "")
-}
-
-// PingResponse is the JSON response to a health ping.
-type PingResponse struct {
-	Status    string `json:"status"`
-	LBID      string `json:"lb_id"`
-	HAProxy   bool   `json:"haproxy_running"`
-	ConfigAge int64  `json:"config_age_seconds"`
-}
-
-// handlePing responds to health check requests.
-func (a *Agent) handlePing(msg *nats.Msg) {
-	haproxyRunning := isHAProxyRunning(a.pidPath)
-	configAge := configAgeSeconds(a.configPath)
-
-	resp := PingResponse{
-		Status:    "ok",
-		LBID:      a.lbID,
-		HAProxy:   haproxyRunning,
-		ConfigAge: configAge,
-	}
-
-	data, err := json.Marshal(resp)
-	if err != nil {
-		slog.Error("Failed to marshal ping response", "err", err)
-		return
-	}
-	if err := msg.Respond(data); err != nil {
-		slog.Warn("Failed to respond to ping", "err", err)
-	}
-}
-
-// respond sends a JSON reply to a NATS message if it has a reply subject.
-func (a *Agent) respond(msg *nats.Msg, status, errMsg string) {
-	if msg.Reply == "" {
-		return
-	}
-	resp := map[string]string{"status": status}
-	if errMsg != "" {
-		resp["error"] = errMsg
-	}
-	data, err := json.Marshal(resp)
-	if err != nil {
-		slog.Error("Failed to marshal response", "err", err)
-		return
-	}
-	if err := msg.Respond(data); err != nil {
-		slog.Warn("Failed to respond", "err", err)
-	}
-}
-
-// WriteConfig atomically writes an HAProxy config file.
-// It writes to a temp file first, then renames for atomicity.
-func WriteConfig(path, content string) error {
-	dir := filepath.Dir(path)
-	if err := os.MkdirAll(dir, 0o750); err != nil {
-		return fmt.Errorf("create config dir: %w", err)
-	}
-
-	tmp := path + ".tmp"
-	if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
-		return fmt.Errorf("write temp config: %w", err)
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("rename config: %w", err)
-	}
-	return nil
-}
-
-// reloadHAProxy starts or reloads the HAProxy process.
-// If HAProxy is running (PID file exists and process alive), it does a
-// graceful reload with -sf. Otherwise it starts a fresh instance.
-func reloadHAProxy(configPath, pidPath string) error {
-	// Ensure the stats socket directory exists (the config may reference
-	// /tmp/spinifex-haproxy/ which doesn't exist on fresh Alpine VMs).
-	_ = os.MkdirAll("/tmp/spinifex-haproxy", 0o750)
-
-	oldPID := readPID(pidPath)
-
-	var cmd *exec.Cmd
-	if oldPID > 0 {
-		// Graceful reload: new worker starts, old workers finish in-flight requests
-		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D", "-sf", strconv.Itoa(oldPID))
-	} else {
-		// Fresh start
-		cmd = exec.Command("haproxy", "-f", configPath, "-p", pidPath, "-D")
-	}
-
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("haproxy: %s: %w", strings.TrimSpace(string(out)), err)
-	}
-	return nil
-}
-
-// readPID reads the HAProxy PID from the PID file. Returns 0 if unavailable.
-func readPID(pidPath string) int {
-	data, err := os.ReadFile(pidPath)
-	if err != nil {
-		return 0
-	}
-	pid, err := strconv.Atoi(strings.TrimSpace(string(data)))
-	if err != nil {
-		return 0
-	}
-	// Check if process is still alive
-	proc, err := os.FindProcess(pid)
-	if err != nil {
-		return 0
-	}
-	if err := proc.Signal(syscall.Signal(0)); err != nil {
-		return 0
-	}
-	return pid
-}
-
-// isHAProxyRunning checks if HAProxy is running via its PID file.
-func isHAProxyRunning(pidPath string) bool {
-	return readPID(pidPath) > 0
-}
-
-// configAgeSeconds returns how many seconds since the config file was last modified.
-// Returns -1 if the file doesn't exist.
-func configAgeSeconds(configPath string) int64 {
-	info, err := os.Stat(configPath)
-	if err != nil {
-		return -1
-	}
-	return int64(time.Since(info.ModTime()).Seconds())
-}

spinifex/albagent/agent_test.go

@@ -1,342 +0,0 @@
-package albagent
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	natsserver "github.com/nats-io/nats-server/v2/server"
-	natstest "github.com/nats-io/nats-server/v2/test"
-	"github.com/nats-io/nats.go"
-)
-
-func startTestNATS(t *testing.T) *natsserver.Server {
-	t.Helper()
-	opts := natstest.DefaultTestOptions
-	opts.Port = -1 // random port
-	ns := natstest.RunServer(&opts)
-	t.Cleanup(ns.Shutdown)
-	return ns
-}
-
-func TestNew(t *testing.T) {
-	agent, err := New("lb-test123", "nats://localhost:4222")
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	if agent.lbID != "lb-test123" {
-		t.Errorf("lbID = %q, want %q", agent.lbID, "lb-test123")
-	}
-}
-
-func TestNew_EmptyLBID(t *testing.T) {
-	_, err := New("", "nats://localhost:4222")
-	if err == nil {
-		t.Fatal("expected error for empty lbID")
-	}
-}
-
-func TestNew_DefaultNATSURL(t *testing.T) {
-	agent, err := New("lb-test", "")
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	if agent.natsURL != "nats://127.0.0.1:4222" {
-		t.Errorf("natsURL = %q, want default", agent.natsURL)
-	}
-}
-
-func TestConfigTopic(t *testing.T) {
-	topic := ConfigTopic("lb-abc123")
-	if topic != "elbv2.alb.lb-abc123.config" {
-		t.Errorf("ConfigTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.config")
-	}
-}
-
-func TestPingTopic(t *testing.T) {
-	topic := PingTopic("lb-abc123")
-	if topic != "elbv2.alb.lb-abc123.ping" {
-		t.Errorf("PingTopic = %q, want %q", topic, "elbv2.alb.lb-abc123.ping")
-	}
-}
-
-func TestWriteConfig(t *testing.T) {
-	dir := t.TempDir()
-	path := filepath.Join(dir, "haproxy.cfg")
-
-	content := "global\n  log stdout\n"
-	if err := WriteConfig(path, content); err != nil {
-		t.Fatalf("WriteConfig: %v", err)
-	}
-
-	data, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatalf("ReadFile: %v", err)
-	}
-	if string(data) != content {
-		t.Errorf("config = %q, want %q", string(data), content)
-	}
-}
-
-func TestWriteConfig_CreatesDir(t *testing.T) {
-	dir := t.TempDir()
-	path := filepath.Join(dir, "subdir", "haproxy.cfg")
-
-	if err := WriteConfig(path, "test"); err != nil {
-		t.Fatalf("WriteConfig: %v", err)
-	}
-
-	if _, err := os.Stat(path); err != nil {
-		t.Fatalf("config file not created: %v", err)
-	}
-}
-
-func TestWriteConfig_Atomic(t *testing.T) {
-	dir := t.TempDir()
-	path := filepath.Join(dir, "haproxy.cfg")
-
-	// Write initial config
-	if err := WriteConfig(path, "initial"); err != nil {
-		t.Fatalf("WriteConfig initial: %v", err)
-	}
-
-	// Overwrite with new config
-	if err := WriteConfig(path, "updated"); err != nil {
-		t.Fatalf("WriteConfig updated: %v", err)
-	}
-
-	data, _ := os.ReadFile(path)
-	if string(data) != "updated" {
-		t.Errorf("config = %q, want %q", string(data), "updated")
-	}
-
-	// No temp file left behind
-	if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) {
-		t.Error("temp file should not exist after successful write")
-	}
-}
-
-func TestConfigAgeSeconds(t *testing.T) {
-	dir := t.TempDir()
-	path := filepath.Join(dir, "test.cfg")
-
-	// Non-existent file
-	if age := configAgeSeconds(path); age != -1 {
-		t.Errorf("age of non-existent file = %d, want -1", age)
-	}
-
-	// Write file and check age
-	os.WriteFile(path, []byte("test"), 0o644)
-	age := configAgeSeconds(path)
-	if age < 0 || age > 2 {
-		t.Errorf("age = %d, expected 0-2 seconds", age)
-	}
-}
-
-func TestIsHAProxyRunning_NoPIDFile(t *testing.T) {
-	if isHAProxyRunning("/nonexistent/haproxy.pid") {
-		t.Error("expected false for non-existent PID file")
-	}
-}
-
-func TestReadPID_InvalidContent(t *testing.T) {
-	dir := t.TempDir()
-	pidFile := filepath.Join(dir, "haproxy.pid")
-
-	os.WriteFile(pidFile, []byte("not-a-number\n"), 0o644)
-	if pid := readPID(pidFile); pid != 0 {
-		t.Errorf("readPID = %d, want 0 for invalid content", pid)
-	}
-}
-
-func TestReadPID_DeadProcess(t *testing.T) {
-	dir := t.TempDir()
-	pidFile := filepath.Join(dir, "haproxy.pid")
-
-	// Use a very high PID that's unlikely to be running
-	os.WriteFile(pidFile, []byte("999999999\n"), 0o644)
-	if pid := readPID(pidFile); pid != 0 {
-		t.Errorf("readPID = %d, want 0 for dead process", pid)
-	}
-}
-
-func TestAgent_StartStop(t *testing.T) {
-	ns := startTestNATS(t)
-
-	agent, err := New("lb-test123", ns.ClientURL())
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-
-	if err := agent.Start(); err != nil {
-		t.Fatalf("Start: %v", err)
-	}
-
-	// Verify agent is connected
-	if agent.nc == nil || !agent.nc.IsConnected() {
-		t.Fatal("agent not connected to NATS")
-	}
-
-	agent.Stop()
-
-	if agent.nc != nil {
-		t.Error("nc should be nil after Stop")
-	}
-}
-
-func TestAgent_HandleConfig(t *testing.T) {
-	ns := startTestNATS(t)
-	dir := t.TempDir()
-
-	agent, _ := New("lb-cfg123", ns.ClientURL())
-	agent.configPath = filepath.Join(dir, "haproxy.cfg")
-	agent.pidPath = filepath.Join(dir, "haproxy.pid")
-
-	// Mock reload to avoid needing real HAProxy
-	reloadCalled := false
-	agent.reloadFn = func(configPath, pidPath string) error {
-		reloadCalled = true
-		return nil
-	}
-
-	if err := agent.Start(); err != nil {
-		t.Fatalf("Start: %v", err)
-	}
-	defer agent.Stop()
-
-	// Send a config update via a separate NATS connection
-	nc, err := nats.Connect(ns.ClientURL())
-	if err != nil {
-		t.Fatalf("connect: %v", err)
-	}
-	defer nc.Close()
-
-	haproxyConfig := "global\n  log stdout\ndefaults\n  mode http\n"
-	reply, err := nc.Request(ConfigTopic("lb-cfg123"), []byte(haproxyConfig), 2*time.Second)
-	if err != nil {
-		t.Fatalf("Request: %v", err)
-	}
-
-	// Check response
-	var resp map[string]string
-	json.Unmarshal(reply.Data, &resp)
-	if resp["status"] != "ok" {
-		t.Errorf("response status = %q, want %q", resp["status"], "ok")
-	}
-
-	// Verify config was written
-	data, err := os.ReadFile(agent.configPath)
-	if err != nil {
-		t.Fatalf("ReadFile: %v", err)
-	}
-	if string(data) != haproxyConfig {
-		t.Errorf("written config = %q, want %q", string(data), haproxyConfig)
-	}
-
-	if !reloadCalled {
-		t.Error("reload function was not called")
-	}
-}
-
-func TestAgent_HandleConfig_Empty(t *testing.T) {
-	ns := startTestNATS(t)
-
-	agent, _ := New("lb-empty", ns.ClientURL())
-	agent.reloadFn = func(_, _ string) error { return nil }
-
-	if err := agent.Start(); err != nil {
-		t.Fatalf("Start: %v", err)
-	}
-	defer agent.Stop()
-
-	nc, _ := nats.Connect(ns.ClientURL())
-	defer nc.Close()
-
-	reply, err := nc.Request(ConfigTopic("lb-empty"), []byte(""), 2*time.Second)
-	if err != nil {
-		t.Fatalf("Request: %v", err)
-	}
-
-	var resp map[string]string
-	json.Unmarshal(reply.Data, &resp)
-	if resp["status"] != "error" {
-		t.Errorf("response status = %q, want %q for empty config", resp["status"], "error")
-	}
-}
-
-func TestAgent_HandlePing(t *testing.T) {
-	ns := startTestNATS(t)
-	dir := t.TempDir()
-
-	agent, _ := New("lb-ping123", ns.ClientURL())
-	agent.configPath = filepath.Join(dir, "haproxy.cfg")
-	agent.pidPath = filepath.Join(dir, "haproxy.pid")
-
-	if err := agent.Start(); err != nil {
-		t.Fatalf("Start: %v", err)
-	}
-	defer agent.Stop()
-
-	nc, _ := nats.Connect(ns.ClientURL())
-	defer nc.Close()
-
-	reply, err := nc.Request(PingTopic("lb-ping123"), nil, 2*time.Second)
-	if err != nil {
-		t.Fatalf("Request: %v", err)
-	}
-
-	var resp PingResponse
-	if err := json.Unmarshal(reply.Data, &resp); err != nil {
-		t.Fatalf("Unmarshal: %v", err)
-	}
-
-	if resp.Status != "ok" {
-		t.Errorf("status = %q, want %q", resp.Status, "ok")
-	}
-	if resp.LBID != "lb-ping123" {
-		t.Errorf("lb_id = %q, want %q", resp.LBID, "lb-ping123")
-	}
-	if resp.HAProxy {
-		t.Error("haproxy_running should be false (no PID file)")
-	}
-	if resp.ConfigAge != -1 {
-		t.Errorf("config_age = %d, want -1 (no config file)", resp.ConfigAge)
-	}
-}
-
-func TestAgent_HandleConfig_ReloadError(t *testing.T) {
-	ns := startTestNATS(t)
-	dir := t.TempDir()
-
-	agent, _ := New("lb-err", ns.ClientURL())
-	agent.configPath = filepath.Join(dir, "haproxy.cfg")
-	agent.pidPath = filepath.Join(dir, "haproxy.pid")
-	agent.reloadFn = func(_, _ string) error {
-		return fmt.Errorf("haproxy binary not found")
-	}
-
-	if err := agent.Start(); err != nil {
-		t.Fatalf("Start: %v", err)
-	}
-	defer agent.Stop()
-
-	nc, _ := nats.Connect(ns.ClientURL())
-	defer nc.Close()
-
-	reply, err := nc.Request(ConfigTopic("lb-err"), []byte("some config"), 2*time.Second)
-	if err != nil {
-		t.Fatalf("Request: %v", err)
-	}
-
-	var resp map[string]string
-	json.Unmarshal(reply.Data, &resp)
-	if resp["status"] != "error" {
-		t.Errorf("response status = %q, want %q", resp["status"], "error")
-	}
-	if resp["error"] == "" {
-		t.Error("expected error message in response")
-	}
-}

spinifex/awserrors/awserrors.go

@@ -451,6 +451,7 @@ var (
 	ErrorELBv2InvalidScheme                = "InvalidScheme"
 	ErrorELBv2SubnetNotFound               = "SubnetNotFound"
 	ErrorELBv2AvailabilityZoneNotSupported = "AvailabilityZoneNotSupported"
+	ErrorELBv2InvalidConfigurationRequest  = "InvalidConfigurationRequest"
 )
 
 // ValidErrorCode returns the error code if it exists in ErrorLookup,
@@ -908,4 +909,5 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorELBv2InvalidScheme:                {HTTPCode: 400, Message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
 	ErrorELBv2SubnetNotFound:               {HTTPCode: 400, Message: "The specified subnet does not exist."},
 	ErrorELBv2AvailabilityZoneNotSupported: {HTTPCode: 400, Message: "The specified Availability Zone is not supported."},
+	ErrorELBv2InvalidConfigurationRequest:  {HTTPCode: 400, Message: "Security groups are not supported for load balancers with type 'network'."},
 }

spinifex/awserrors/awserrors_test.go

@@ -451,6 +451,7 @@ func TestErrorLookup(t *testing.T) {
 		{code: "InvalidScheme", http: 400, message: "The specified scheme is not valid. Specify 'internet-facing' or 'internal'."},
 		{code: "SubnetNotFound", http: 400, message: "The specified subnet does not exist."},
 		{code: "AvailabilityZoneNotSupported", http: 400, message: "The specified Availability Zone is not supported."},
+		{code: "InvalidConfigurationRequest", http: 400, message: "Security groups are not supported for load balancers with type 'network'."},
 	}
 
 	if len(ErrorLookup) != len(expected) {

spinifex/config/config.go

@@ -55,80 +55,81 @@ type BootstrapConfig struct {
 // Config holds all configuration for the application
 type Config struct {
 	// Node config
-	Node     string   `mapstructure:"node"`
-	Host     string   `mapstructure:"host"` // Unique hostname or IP of this node
-	Region   string   `mapstructure:"region"`
-	AZ       string   `mapstructure:"az"`
-	DataDir  string   `mapstructure:"data_dir"`
-	Services []string `mapstructure:"services"` // Which services this node runs locally
+	Node     string   `json:"Node" mapstructure:"node"`
+	Host     string   `json:"Host" mapstructure:"host"` // Unique hostname or IP of this node
+	Region   string   `json:"Region" mapstructure:"region"`
+	AZ       string   `json:"AZ" mapstructure:"az"`
+	DataDir  string   `json:"DataDir" mapstructure:"data_dir"`
+	Services []string `json:"Services" mapstructure:"services"` // Which services this node runs locally
 
-	Daemon     DaemonConfig     `mapstructure:"daemon"`
-	NATS       NATSConfig       `mapstructure:"nats"`
-	Predastore PredastoreConfig `mapstructure:"predastore"`
-	Viperblock ViperblockConfig `mapstructure:"viperblock"`
-	AWSGW      AWSGWConfig      `mapstructure:"awsgw"`
-	VPCD       VPCDConfig       `mapstructure:"vpcd"`
+	Daemon     DaemonConfig     `json:"Daemon" mapstructure:"daemon"`
+	NATS       NATSConfig       `json:"NATS" mapstructure:"nats"`
+	Predastore PredastoreConfig `json:"Predastore" mapstructure:"predastore"`
+	Viperblock ViperblockConfig `json:"Viperblock" mapstructure:"viperblock"`
+	AWSGW      AWSGWConfig      `json:"AWSGW" mapstructure:"awsgw"`
+	VPCD       VPCDConfig       `json:"VPCD" mapstructure:"vpcd"`
 
-	BaseDir string `mapstructure:"base_dir"`
-	WalDir  string `mapstructure:"wal_dir"`
+	BaseDir string `json:"BaseDir" mapstructure:"base_dir"`
+	WalDir  string `json:"WalDir" mapstructure:"wal_dir"`
 }
 
 type AWSGWConfig struct {
-	Host    string `mapstructure:"host"`
-	TLSKey  string `mapstructure:"tlskey"`
-	TLSCert string `mapstructure:"tlscert"`
-	Config  string `mapstructure:"config"`
+	Host    string `json:"Host" mapstructure:"host"`
+	TLSKey  string `json:"TLSKey" mapstructure:"tlskey"`
+	TLSCert string `json:"TLSCert" mapstructure:"tlscert"`
+	Config  string `json:"Config" mapstructure:"config"`
 
-	Debug         bool `mapstructure:"debug"`
-	ExpectedNodes int  `mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
+	Debug         bool `json:"Debug" mapstructure:"debug"`
+	ExpectedNodes int  `json:"ExpectedNodes" mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
 }
 
 type ViperblockConfig struct {
-	ShardWAL *bool `mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
+	ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
 }
 
 // VPCDConfig holds the VPC daemon (vpcd) configuration.
 type VPCDConfig struct {
-	OVNNBAddr         string `mapstructure:"ovn_nb_addr"`        // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641")
-	OVNSBAddr         string `mapstructure:"ovn_sb_addr"`        // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642")
-	ExternalInterface string `mapstructure:"external_interface"` // WAN NIC name (e.g., "eth1", "enp0s3") — the physical NIC on the WAN bridge
-	WanBridge         string `mapstructure:"wan_bridge"`         // OVS bridge for WAN traffic (default "br-wan", maps to OVN "external" network)
-	BridgeMode        string `mapstructure:"bridge_mode"`        // "direct" or "macvlan" (auto-detected if empty)
+	OVNNBAddr         string `json:"OVNNBAddr" mapstructure:"ovn_nb_addr"`                // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641")
+	OVNSBAddr         string `json:"OVNSBAddr" mapstructure:"ovn_sb_addr"`                // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642")
+	ExternalInterface string `json:"ExternalInterface" mapstructure:"external_interface"` // WAN NIC name (e.g., "eth1", "enp0s3") — the physical NIC on the WAN bridge
+	WanBridge         string `json:"WanBridge" mapstructure:"wan_bridge"`                 // OVS bridge for WAN traffic (default "br-wan", maps to OVN "external" network)
+	BridgeMode        string `json:"BridgeMode" mapstructure:"bridge_mode"`               // "direct" or "macvlan" (auto-detected if empty)
 }
 
 type PredastoreConfig struct {
-	Host      string `mapstructure:"host"`
-	Bucket    string `mapstructure:"bucket"`
-	Region    string `mapstructure:"region"`
-	AccessKey string `mapstructure:"accesskey"`
-	SecretKey string `mapstructure:"secretkey"`
-	BaseDir   string `mapstructure:"base_dir"`
-	NodeID    int    `mapstructure:"node_id"`
+	Host      string `json:"Host" mapstructure:"host"`
+	Bucket    string `json:"Bucket" mapstructure:"bucket"`
+	Region    string `json:"Region" mapstructure:"region"`
+	AccessKey string `json:"AccessKey" mapstructure:"accesskey"`
+	SecretKey string `json:"SecretKey" mapstructure:"secretkey"`
+	BaseDir   string `json:"BaseDir" mapstructure:"base_dir"`
+	NodeID    int    `json:"NodeID" mapstructure:"node_id"`
 }
 
 // DaemonConfig holds the daemon configuration
 type DaemonConfig struct {
-	Host          string `mapstructure:"host"`
-	TLSKey        string `mapstructure:"tlskey"`
-	TLSCert       string `mapstructure:"tlscert"`
-	DevNetworking bool   `mapstructure:"dev_networking"` // VPC instances get both TAP + hostfwd for SSH dev access
+	Host          string `json:"Host" mapstructure:"host"`
+	TLSKey        string `json:"TLSKey" mapstructure:"tlskey"`
+	TLSCert       string `json:"TLSCert" mapstructure:"tlscert"`
+	DevNetworking bool   `json:"DevNetworking" mapstructure:"dev_networking"` // VPC instances get both TAP + hostfwd for SSH dev access
+	MgmtBridge    string `json:"MgmtBridge" mapstructure:"mgmt_bridge"`       // Linux bridge for system instance control plane (default "br-mgmt")
 }
 
 // NATSConfig holds the NATS configuration
 type NATSConfig struct {
-	Host string  `mapstructure:"host"`
-	ACL  NATSACL `mapstructure:"acl"`
-	Sub  NATSSub `mapstructure:"sub"`
+	Host string  `json:"Host" mapstructure:"host"`
+	ACL  NATSACL `json:"ACL" mapstructure:"acl"`
+	Sub  NATSSub `json:"Sub" mapstructure:"sub"`
 }
 
 // NATSACL holds the NATS ACL configuration
 type NATSACL struct {
-	Token string `mapstructure:"token"`
+	Token string `json:"Token" mapstructure:"token"`
 }
 
 // NATSSub holds the NATS subscription configuration
 type NATSSub struct {
-	Subject string `mapstructure:"subject"`
+	Subject string `json:"Subject" mapstructure:"subject"`
 }
 
 // NodeBaseDir returns the BaseDir for the current node, or "" if the config

spinifex/daemon/daemon.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -49,6 +50,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
+	"github.com/mulgadc/spinifex/spinifex/tags"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -133,6 +135,10 @@ type Daemon struct {
 	startTime     time.Time
 	configPath    string
 
+	// System credentials for ALB agent SigV4 auth (loaded from system-credentials.json)
+	systemAccessKey string
+	systemSecretKey string
+
 	// JetStream manager for KV state storage (nil if JetStream disabled)
 	jsManager *JetStreamManager
 
@@ -146,6 +152,14 @@ type Daemon struct {
 	// NetworkPlumber handles tap device lifecycle for VPC networking
 	networkPlumber NetworkPlumber
 
+	// Management NIC infrastructure: bridge IP + IP allocator for system instances.
+	// Populated at startup when br-mgmt is detected; nil/empty otherwise.
+	mgmtBridgeIP    string
+	mgmtIPAllocator *MgmtIPAllocator
+	// mgmtRouteVia is the AWSGW bind IP that system instances must route via the
+	// management NIC. Set when AWSGW binds to a specific IP (multi-node).
+	mgmtRouteVia string
+
 	// shuttingDown is set to true during coordinated cluster shutdown (GATE phase)
 	// or during SIGTERM-based shutdown. When true, the daemon rejects new work,
 	// crash handlers bail out, and setupShutdown skips redundant VM stops.
@@ -256,7 +270,10 @@ func (rm *ResourceManager) GetInstanceTypeInfos() []*ec2.InstanceTypeInfo {
 	defer rm.mu.RUnlock()
 
 	var infos []*ec2.InstanceTypeInfo
-	for _, it := range rm.instanceTypes {
+	for name, it := range rm.instanceTypes {
+		if instancetypes.IsSystemType(name) {
+			continue
+		}
 		infos = append(infos, it)
 	}
 	return infos
@@ -271,7 +288,11 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 
 	var infos []*ec2.InstanceTypeInfo
 
-	for _, it := range rm.instanceTypes {
+	for name, it := range rm.instanceTypes {
+		if instancetypes.IsSystemType(name) {
+			continue
+		}
+
 		vCPUs := instanceTypeVCPUs(it)
 		memMiB := instanceTypeMemoryMiB(it)
 
@@ -314,7 +335,10 @@ func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64
 	remainingVCPU := rm.availableVCPU - rm.allocatedVCPU
 	remainingMem := rm.availableMem - rm.allocatedMem
 
-	for _, it := range rm.instanceTypes {
+	for name, it := range rm.instanceTypes {
+		if instancetypes.IsSystemType(name) {
+			continue
+		}
 		typeCap := resourceStatsForType(remainingVCPU, remainingMem, it)
 		if typeCap.VCPU == 0 || typeCap.MemoryGB == 0 {
 			continue
@@ -468,6 +492,13 @@ func (d *Daemon) subscribeAll() error {
 		{"elbv2.CreateListener", d.handleELBv2CreateListener, "spinifex-workers"},
 		{"elbv2.DeleteListener", d.handleELBv2DeleteListener, "spinifex-workers"},
 		{"elbv2.DescribeListeners", d.handleELBv2DescribeListeners, "spinifex-workers"},
+		{"elbv2.DescribeTags", d.handleELBv2DescribeTags, "spinifex-workers"},
+		{"elbv2.LBAgentHeartbeat", d.handleELBv2LBAgentHeartbeat, "spinifex-workers"},
+		{"elbv2.GetLBConfig", d.handleELBv2GetLBConfig, "spinifex-workers"},
+		{"elbv2.ModifyTargetGroupAttributes", d.handleELBv2ModifyTargetGroupAttributes, "spinifex-workers"},
+		{"elbv2.DescribeTargetGroupAttributes", d.handleELBv2DescribeTargetGroupAttributes, "spinifex-workers"},
+		{"elbv2.ModifyLoadBalancerAttributes", d.handleELBv2ModifyLoadBalancerAttributes, "spinifex-workers"},
+		{"elbv2.DescribeLoadBalancerAttributes", d.handleELBv2DescribeLoadBalancerAttributes, "spinifex-workers"},
 		{fmt.Sprintf("spinifex.admin.%s.health", d.node), d.handleHealthCheck, ""},
 		{"spinifex.nodes.discover", d.handleNodeDiscover, ""},
 		{"spinifex.node.status", d.handleNodeStatus, ""},
@@ -650,6 +681,20 @@ func (d *Daemon) Start() error {
 			d.eipService = eipSvc
 			slog.Info("EIP service initialized")
 		}
+
+		// Inject external IPAM + EIP KV into VPC service so DeleteNetworkInterface
+		// can release auto-assigned public IPs and NAT rules.
+		eipJS, eipJSErr := d.natsConn.JetStream()
+		if eipJSErr != nil {
+			slog.Warn("Failed to get JetStream for VPC external IPAM injection", "err", eipJSErr)
+		} else {
+			eipKV, eipKVErr := utils.GetOrCreateKVBucket(eipJS, handlers_ec2_eip.KVBucketEIPs, 10)
+			if eipKVErr != nil {
+				slog.Warn("Failed to get EIP KV bucket for VPC service", "err", eipKVErr)
+			} else {
+				d.vpcService.SetExternalIPAM(d.externalIPAM, eipKV)
+			}
+		}
 	}
 
 	d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) {
@@ -666,38 +711,65 @@ func (d *Daemon) Start() error {
 		return fmt.Errorf("failed to initialize ELBv2 service: %w", err)
 	}
 	if d.vpcService != nil {
-		d.elbv2Service.SetVPCService(d.vpcService)
+		d.elbv2Service.VPCService = d.vpcService
 	}
 
-	// Wire ALB VM lifecycle: instance launcher + NATS URL for cloud-init.
-	// In dev mode (user-mode networking), VMs reach the host at 10.0.2.2
-	// (QEMU's default gateway). Replace 0.0.0.0 with this address.
-	d.elbv2Service.SetInstanceLauncher(d)
-	if d.config != nil && d.config.NATS.Host != "" {
-		natsHost := d.config.NATS.Host
-		if d.config.Daemon.DevNetworking && strings.HasPrefix(natsHost, "0.0.0.0") {
-			natsHost = strings.Replace(natsHost, "0.0.0.0", "10.0.2.2", 1)
-		}
-		d.elbv2Service.SetNATSURL("nats://" + natsHost)
+	// Wire LB VM lifecycle: instance launcher for system VMs.
+	d.elbv2Service.InstanceLauncher = d
+
+	// Detect management bridge for system instance control plane NICs.
+	// Must run before wireLBAgentConfig so the gateway URL uses br-mgmt IP.
+	mgmtBridge := "br-mgmt"
+	if d.config.Daemon.MgmtBridge != "" {
+		mgmtBridge = d.config.Daemon.MgmtBridge
 	}
-	// Discover system AMI for ALB VMs. Prefer an image with "alb" in the
-	// name (pre-baked Alpine with HAProxy), fall back to first available.
+	bridgeIP, bridgeErr := GetBridgeIPv4(mgmtBridge)
+	if bridgeErr != nil {
+		slog.Warn("Management bridge not detected, system instances will not get mgmt NIC", "bridge", mgmtBridge, "err", bridgeErr)
+	} else if bridgeIP == "" {
+		slog.Warn("Management bridge not found, system instances will not get mgmt NIC", "bridge", mgmtBridge)
+	} else {
+		d.mgmtBridgeIP = bridgeIP
+		alloc, allocErr := NewMgmtIPAllocator(bridgeIP)
+		if allocErr != nil {
+			slog.Error("Failed to create mgmt IP allocator", "bridgeIP", bridgeIP, "err", allocErr)
+		} else {
+			d.mgmtIPAllocator = alloc
+			slog.Info("Management bridge detected", "bridge", mgmtBridge, "ip", bridgeIP)
+		}
+	}
+
+	// Wire system credentials + gateway URL for LB agent SigV4 auth.
+	d.wireLBAgentConfig()
+
+	// Set up lazy system AMI discovery for LB VMs. The image may not exist
+	// at daemon startup (imported later), so we resolve it at request time.
 	if d.imageService != nil {
-		imagesOut, imgErr := d.imageService.DescribeImages(&ec2.DescribeImagesInput{}, utils.GlobalAccountID)
-		if imgErr == nil && len(imagesOut.Images) > 0 {
-			systemAMI := aws.StringValue(imagesOut.Images[0].ImageId)
-			for _, img := range imagesOut.Images {
-				name := aws.StringValue(img.Name)
-				if strings.Contains(name, "alb") {
-					systemAMI = aws.StringValue(img.ImageId)
-					break
-				}
+		imgSvc := d.imageService
+		d.elbv2Service.SetSystemAMIFunc(func() (string, error) {
+			imagesOut, imgErr := imgSvc.DescribeImages(&ec2.DescribeImagesInput{
+				Filters: []*ec2.Filter{{
+					Name:   aws.String("tag:" + tags.ManagedByKey),
+					Values: []*string{aws.String(tags.ManagedByELBv2)},
+				}},
+			}, utils.GlobalAccountID)
+			if imgErr != nil {
+				return "", fmt.Errorf("describe LB system images: %w", imgErr)
 			}
-			d.elbv2Service.SetSystemAMI(systemAMI)
-			slog.Info("System AMI set for ALB VMs", "amiId", systemAMI)
-		}
+			if len(imagesOut.Images) == 0 {
+				return "", errors.New("LB system image not imported; run: spx admin images import --name lb-alpine-3.21.6-x86_64")
+			}
+			amiID := aws.StringValue(imagesOut.Images[0].ImageId)
+			slog.Info("System AMI resolved for LB VMs", "amiId", amiID, "name", aws.StringValue(imagesOut.Images[0].Name))
+			return amiID, nil
+		})
 	}
 
+	// System VMs (LB, NAT GW) use the dedicated sys.micro instance type.
+	d.elbv2Service.SetSystemInstanceTypeFunc(func() string {
+		return "sys.micro"
+	})
+
 	// Ensure default VPC exists for system and admin accounts
 	// (matches AWS: every account has a default VPC with IGW + default SG)
 	if d.vpcService != nil {
@@ -733,6 +805,15 @@ func (d *Daemon) Start() error {
 	d.upgradeJetStreamReplicas()
 	d.restoreInstances()
 
+	// Rebuild mgmt IP allocator from restored VMs so we don't re-allocate IPs
+	// that are already in use by running system instances.
+	if d.mgmtIPAllocator != nil {
+		d.Instances.Mu.Lock()
+		d.mgmtIPAllocator.Rebuild(d.Instances.VMS)
+		d.Instances.Mu.Unlock()
+		slog.Info("Rebuilt mgmt IP allocator from restored instances", "allocated", d.mgmtIPAllocator.AllocatedCount())
+	}
+
 	if err := d.subscribeAll(); err != nil {
 		return fmt.Errorf("failed to subscribe to NATS topics: %w", err)
 	}
@@ -1544,21 +1625,47 @@ func (d *Daemon) ClusterManager() error {
 		}
 	})
 
-	// Start HTTP server in goroutine — use a channel to propagate bind errors
-	// back to the caller so Start() fails visibly instead of silently.
+	// Load TLS certificate (C-5: serve over HTTPS instead of plaintext HTTP)
+	// Resolve relative cert paths against config directory (cert lives alongside spinifex.toml).
+	// For binary installs, systemd sets absolute paths via env vars; for dev, the config
+	// stores relative paths like "config/server.pem" which need resolution.
+	tlsCert := d.config.Daemon.TLSCert
+	tlsKey := d.config.Daemon.TLSKey
+	if tlsCert == "" || tlsKey == "" {
+		return fmt.Errorf("cluster manager TLS not configured: set daemon.tlscert and daemon.tlskey in config")
+	}
+	if d.configPath != "" {
+		configDir := filepath.Dir(d.configPath)
+		if !filepath.IsAbs(tlsCert) {
+			tlsCert = filepath.Join(configDir, filepath.Base(tlsCert))
+		}
+		if !filepath.IsAbs(tlsKey) {
+			tlsKey = filepath.Join(configDir, filepath.Base(tlsKey))
+		}
+	}
+	cert, err := tls.LoadX509KeyPair(tlsCert, tlsKey)
+	if err != nil {
+		return fmt.Errorf("cluster manager load TLS cert: %w", err)
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+	}
+
 	d.clusterServer = &http.Server{
 		Addr:              daemonHost,
 		Handler:           r,
 		ReadHeaderTimeout: 10 * time.Second,
+		TLSConfig:         tlsConfig,
 	}
 
-	ln, err := net.Listen("tcp", daemonHost)
+	ln, err := tls.Listen("tcp", daemonHost, tlsConfig)
 	if err != nil {
 		return fmt.Errorf("cluster manager listen on %s: %w", daemonHost, err)
 	}
 
 	go func() {
-		slog.Info("Starting cluster manager", "host", daemonHost)
+		slog.Info("Starting cluster manager (TLS)", "host", daemonHost)
 		if err := d.clusterServer.Serve(ln); err != nil && err != http.ErrServerClosed {
 			slog.Error("Cluster manager failed", "error", err)
 		}
@@ -1759,6 +1866,18 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 				if err := d.networkPlumber.CleanupTapDevice(instance.ENIId); err != nil {
 					slog.Warn("Failed to clean up tap device", "eni", instance.ENIId, "err", err)
 				}
+				// Clean up any extra ENI tap devices (multi-subnet ALB VMs).
+				d.cleanupExtraENITaps(instance)
+			}
+
+			// Clean up management TAP and release IP
+			if instance.MgmtTap != "" {
+				if err := CleanupMgmtTapDevice(instance.MgmtTap); err != nil {
+					slog.Warn("Failed to clean up mgmt tap device", "tap", instance.MgmtTap, "instanceId", instance.ID, "err", err)
+				}
+				if d.mgmtIPAllocator != nil {
+					d.mgmtIPAllocator.Release(instance.ID)
+				}
 			}
 
 			// Release public IP before deleting ENI
@@ -1785,17 +1904,30 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
 				}
 			}
 
-			// On termination, delete the auto-created ENI (releases IP back to IPAM,
-			// publishes vpc.delete-port for vpcd). On stop, ENI persists (AWS behavior).
+			// On termination, detach and delete the auto-created ENI (releases IP
+			// back to IPAM, publishes vpc.delete-port for vpcd). On stop, ENI
+			// persists (AWS behavior). Must detach first to clear in-use status.
+			// Tolerate NotFound — ENI may have been cleaned up already.
+			// Other errors (KV failures, permission issues, in-use) are real
+			// failures that could leak IPAM addresses.
 			if deleteVolume && instance.ENIId != "" && d.vpcService != nil {
-				_, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
+				if detachErr := d.vpcService.DetachENI(instance.AccountID, instance.ENIId); detachErr != nil {
+					slog.Warn("Failed to detach ENI on termination", "eni", instance.ENIId, "instanceId", instance.ID, "err", detachErr)
+				}
+				if _, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
 					NetworkInterfaceId: &instance.ENIId,
-				}, instance.AccountID)
-				if eniErr != nil {
-					slog.Error("Failed to delete ENI on termination", "eni", instance.ENIId, "err", eniErr)
+				}, instance.AccountID); eniErr != nil {
+					if strings.Contains(eniErr.Error(), awserrors.ErrorInvalidNetworkInterfaceIDNotFound) {
+						slog.Debug("ENI already cleaned up on termination", "eni", instance.ENIId)
+					} else {
+						slog.Error("Failed to delete ENI on termination", "eni", instance.ENIId, "instanceId", instance.ID, "err", eniErr)
+					}
 				} else {
 					slog.Info("Deleted ENI on termination", "eni", instance.ENIId, "instanceId", instance.ID)
 				}
+				// Extra ENIs (multi-subnet ALB VMs) are cleaned up by
+				// DeleteLoadBalancer via its own lb.ENIs loop — the daemon
+				// doesn't duplicate that teardown here.
 			}
 
 			// Deallocate resources
@@ -2246,6 +2378,12 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 
 		slog.Info("VPC networking configured", "tap", tapName, "eni", instance.ENIId, "mac", instance.ENIMac)
 
+		// Additional VPC NICs for multi-subnet system VMs (e.g. ALBs with
+		// subnets across multiple AZs).
+		if err := d.setupExtraENINICs(instance); err != nil {
+			return err
+		}
+
 		// DEV_NETWORKING: add a second NIC with hostfwd for SSH dev access
 		if d.config.Daemon.DevNetworking {
 			sshDebugAddr, err := viperblock.FindFreePort()
@@ -2321,6 +2459,17 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 		})
 	}
 
+	// Management NIC: system instances get a TAP on br-mgmt for control plane traffic.
+	if instance.MgmtMAC != "" && instance.MgmtTap != "" {
+		instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
+			Value: fmt.Sprintf("tap,id=mgmt0,ifname=%s,script=no,downscript=no", instance.MgmtTap),
+		})
+		instance.Config.Devices = append(instance.Config.Devices, vm.Device{
+			Value: fmt.Sprintf("virtio-net-pci,netdev=mgmt0,mac=%s", instance.MgmtMAC),
+		})
+		slog.Info("Management NIC configured", "tap", instance.MgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID)
+	}
+
 	instance.Config.Devices = append(instance.Config.Devices, vm.Device{
 		Value: "virtio-rng-pci",
 	})
@@ -2759,6 +2908,10 @@ func (rm *ResourceManager) updateInstanceSubscriptions() {
 	defer rm.subsMu.Unlock()
 
 	for typeName, typeInfo := range rm.instanceTypes {
+		// System types (sys.micro, etc.) are internal-only — not routable via customer API.
+		if instancetypes.IsSystemType(typeName) {
+			continue
+		}
 		queueTopic := fmt.Sprintf("ec2.RunInstances.%s", typeName)
 		canFit := rm.canAllocate(typeInfo, 1) >= 1
 
@@ -2802,3 +2955,74 @@ func (rm *ResourceManager) updateInstanceSubscriptions() {
 		}
 	}
 }
+
+// wireLBAgentConfig loads system credentials, resolves the gateway URL,
+// reads the CA certificate, and wires them into the ELBv2 service so LB
+// VMs get SigV4 credentials and gateway URL injected via cloud-init.
+func (d *Daemon) wireLBAgentConfig() {
+	// Use system credentials from spinifex.toml (predastore section).
+	// These are the same service-to-service credentials written by admin init
+	// into both spinifex.toml and system-credentials.json. Reading from the
+	// config avoids file permission issues with the separate JSON file.
+	if d.config.Predastore.AccessKey != "" && d.config.Predastore.SecretKey != "" {
+		d.systemAccessKey = d.config.Predastore.AccessKey
+		d.systemSecretKey = d.config.Predastore.SecretKey
+		d.elbv2Service.SystemAccessKey = d.config.Predastore.AccessKey
+		d.elbv2Service.SystemSecretKey = d.config.Predastore.SecretKey
+		slog.Info("System credentials loaded for LB agent auth")
+	} else {
+		slog.Warn("System credentials missing from spinifex.toml predastore section — LB VMs will not have SigV4 credentials for agent auth")
+	}
+
+	// Resolve gateway URL — the address LB VMs use to reach the AWS gateway.
+	// Internet-facing LB VMs reach it via VPC networking; internal LB VMs
+	// reach it via the management NIC (br-mgmt) with a host route added
+	// to their cloud-init when AWSGW binds to a specific WAN IP.
+	var gatewayHost string
+	awsgwBindIP := ""
+	if d.config.AWSGW.Host != "" {
+		if h, _, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil {
+			awsgwBindIP = h
+		}
+	}
+
+	if d.mgmtBridgeIP != "" {
+		if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" {
+			// Multi-node: AWSGW listens on a specific WAN IP. Use that IP
+			// as the gateway URL. Internal LBs will get a host route via
+			// br-mgmt to reach it (see LaunchSystemInstance).
+			gatewayHost = awsgwBindIP
+			d.mgmtRouteVia = awsgwBindIP
+		} else {
+			// Single-node: AWSGW listens on all interfaces — br-mgmt IP works.
+			gatewayHost = d.mgmtBridgeIP
+		}
+	} else if d.config.Daemon.DevNetworking {
+		gatewayHost = "10.0.2.2"
+	} else if awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" {
+		gatewayHost = awsgwBindIP
+	}
+
+	// Extract port from AWSGW host config (e.g. "0.0.0.0:9999" → "9999").
+	gatewayPort := "9999"
+	if d.config.AWSGW.Host != "" {
+		if _, port, splitErr := net.SplitHostPort(d.config.AWSGW.Host); splitErr == nil && port != "" {
+			gatewayPort = port
+		}
+	}
+
+	if gatewayHost != "" {
+		gatewayURL := "https://" + net.JoinHostPort(gatewayHost, gatewayPort)
+		d.elbv2Service.GatewayURL = gatewayURL
+		slog.Info("LB agent gateway URL configured", "url", gatewayURL)
+	} else {
+		slog.Error("LB agent gateway URL not configured: no reachable host found — all CreateLoadBalancer calls will fail")
+	}
+
+	// Pass mgmt route info so lbVMUserData can add a bootcmd route for
+	// internal LBs that reach the AWSGW via the management NIC.
+	if d.mgmtRouteVia != "" {
+		d.elbv2Service.MgmtRouteGateway = d.mgmtBridgeIP
+		d.elbv2Service.MgmtRouteTarget = d.mgmtRouteVia
+	}
+}

spinifex/daemon/daemon_handlers.go

@@ -1,12 +1,12 @@
 package daemon
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 
@@ -140,10 +140,15 @@ func (d *Daemon) handleNodeDiscover(msg *nats.Msg) {
 }
 
 // daemonIP extracts the IP portion from the daemon host (host:port format).
+// Returns 127.0.0.1 when the host is 0.0.0.0 since that bind address is not
+// a valid connect address and is excluded from cert SANs.
 func (d *Daemon) daemonIP() string {
 	host := d.config.Daemon.Host
 	if h, _, err := net.SplitHostPort(host); err == nil {
-		return h
+		host = h
+	}
+	if host == "0.0.0.0" {
+		return "127.0.0.1"
 	}
 	return host
 }
@@ -202,7 +207,7 @@ func (d *Daemon) queryNATSRole() string {
 	if !d.config.HasService("nats") {
 		return ""
 	}
-	url := fmt.Sprintf("http://%s:%d/varz", d.daemonIP(), natsMonitorPort)
+	url := "http://" + net.JoinHostPort(d.daemonIP(), strconv.Itoa(natsMonitorPort)) + "/varz"
 	return fetchNATSRole(url, roleHTTPClient)
 }
 
@@ -212,7 +217,7 @@ func (d *Daemon) queryPredastoreRole() string {
 	if !d.config.HasService("predastore") {
 		return ""
 	}
-	url := fmt.Sprintf("https://%s:%d/status", d.daemonIP(), predastoreDBPort)
+	url := "https://" + net.JoinHostPort(d.daemonIP(), strconv.Itoa(predastoreDBPort)) + "/status"
 	return fetchPredastoreRole(url, roleTLSHTTPClient)
 }
 
@@ -220,9 +225,6 @@ var roleHTTPClient = &http.Client{Timeout: 500 * time.Millisecond}
 
 var roleTLSHTTPClient = &http.Client{
 	Timeout: 500 * time.Millisecond,
-	Transport: &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // local self-signed cert
-	},
 }
 
 // fetchNATSRole queries a NATS /varz endpoint and returns "leader", "follower", or "".
@@ -291,6 +293,7 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) {
 			InstanceID:   v.ID,
 			Status:       string(v.Status),
 			InstanceType: v.InstanceType,
+			ManagedBy:    v.ManagedBy,
 		}
 		// Get vCPU/memory from the resource manager's instance type info
 		if it, ok := d.resourceMgr.instanceTypes[v.InstanceType]; ok {

spinifex/daemon/daemon_handlers_elbv2.go

@@ -49,3 +49,31 @@ func (d *Daemon) handleELBv2DeleteListener(msg *nats.Msg) {
 func (d *Daemon) handleELBv2DescribeListeners(msg *nats.Msg) {
 	handleNATSRequest(msg, d.elbv2Service.DescribeListeners)
 }
+
+func (d *Daemon) handleELBv2DescribeTags(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTags)
+}
+
+func (d *Daemon) handleELBv2LBAgentHeartbeat(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.LBAgentHeartbeat)
+}
+
+func (d *Daemon) handleELBv2GetLBConfig(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.GetLBConfig)
+}
+
+func (d *Daemon) handleELBv2ModifyTargetGroupAttributes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.ModifyTargetGroupAttributes)
+}
+
+func (d *Daemon) handleELBv2DescribeTargetGroupAttributes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeTargetGroupAttributes)
+}
+
+func (d *Daemon) handleELBv2ModifyLoadBalancerAttributes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.ModifyLoadBalancerAttributes)
+}
+
+func (d *Daemon) handleELBv2DescribeLoadBalancerAttributes(msg *nats.Msg) {
+	handleNATSRequest(msg, d.elbv2Service.DescribeLoadBalancerAttributes)
+}

spinifex/daemon/daemon_handlers_instance.go

@@ -238,7 +238,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 						region = d.config.Region
 						az = d.config.AZ
 					}
-					publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", *eni.NetworkInterfaceId, instance.ID)
+					publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", "", *eni.NetworkInterfaceId, instance.ID)
 					if allocErr != nil {
 						slog.Warn("Failed to allocate public IP for instance", "instanceId", instance.ID, "err", allocErr)
 					} else {
@@ -1481,11 +1481,11 @@ func (d *Daemon) handleEC2DescribeInstanceAttribute(msg *nats.Msg) {
 	respondWithJSON(msg, output)
 }
 
-// publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
+// publishNATEvent sends a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
+// For vpc.add-nat, it uses request-reply to ensure the OVN NAT rule is committed
+// before returning, preventing ARP propagation races. For vpc.delete-nat, it
+// uses fire-and-forget since the caller doesn't need to wait.
 func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) {
-	if d.natsConn == nil {
-		return
-	}
 	evt := struct {
 		VpcId      string `json:"vpc_id"`
 		ExternalIP string `json:"external_ip"`
@@ -1493,12 +1493,13 @@ func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName,
 		PortName   string `json:"port_name"`
 		MAC        string `json:"mac"`
 	}{VpcId: vpcId, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac}
-	data, err := json.Marshal(evt)
-	if err != nil {
-		slog.Error("Failed to marshal NAT event", "topic", topic, "err", err)
+
+	if topic == "vpc.add-nat" {
+		if err := utils.RequestEvent(d.natsConn, topic, evt, 10*time.Second); err != nil {
+			slog.Warn("publishNATEvent: failed to add NAT rule (will retry via reconciliation)",
+				"topic", topic, "externalIP", externalIP, "logicalIP", logicalIP, "err", err)
+		}
 		return
 	}
-	if err := d.natsConn.Publish(topic, data); err != nil {
-		slog.Error("Failed to publish NAT event", "topic", topic, "err", err)
-	}
+	utils.PublishEvent(d.natsConn, topic, evt)
 }

spinifex/daemon/daemon_handlers_test.go

@@ -2435,7 +2435,7 @@ func TestDaemonIP(t *testing.T) {
 		{"HostOnly", "myhost", "myhost"},
 		{"IPv6", "[::1]:4432", "::1"},
 		{"EmptyString", "", ""},
-		{"HostPortZero", "0.0.0.0:0", "0.0.0.0"},
+		{"HostPortZero", "0.0.0.0:0", "127.0.0.1"},
 	}
 
 	for _, tt := range tests {

spinifex/daemon/daemon_handlers_volume.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -145,7 +146,7 @@ func (d *Daemon) handleAttachVolume(msg *nats.Msg, command types.EC2InstanceComm
 	if serverType == "unix" {
 		serverArg = map[string]any{"type": "unix", "path": socketPath}
 	} else {
-		serverArg = map[string]any{"type": "inet", "host": nbdHost, "port": fmt.Sprintf("%d", nbdPort)}
+		serverArg = map[string]any{"type": "inet", "host": nbdHost, "port": strconv.Itoa(nbdPort)}
 	}
 
 	nodeName := fmt.Sprintf("nbd-%s", volumeID)

spinifex/daemon/daemon_mgmt_ip.go

@@ -0,0 +1,111 @@
+package daemon
+
+import (
+	"fmt"
+	"log/slog"
+	"net"
+	"sync"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+// MgmtIPAllocator manages static IP allocation for management NICs on system instances.
+// IPs are derived from the host's management bridge subnet (.10–.249 range).
+// Thread-safe via its own mutex — LaunchSystemInstance does not hold a global lock.
+type MgmtIPAllocator struct {
+	mu        sync.Mutex
+	allocated map[string]string // instanceID → IP
+	baseIP    net.IP            // network base (e.g. 10.15.8.0)
+	rangeMin  byte              // first host byte (10)
+	rangeMax  byte              // last host byte (249)
+}
+
+// NewMgmtIPAllocator creates an allocator for the /24 subnet of bridgeIP.
+// Allocates from .10 to .249 (240 addresses).
+func NewMgmtIPAllocator(bridgeIP string) (*MgmtIPAllocator, error) {
+	ip := net.ParseIP(bridgeIP)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid bridge IP: %s", bridgeIP)
+	}
+	ip4 := ip.To4()
+	if ip4 == nil {
+		return nil, fmt.Errorf("not an IPv4 address: %s", bridgeIP)
+	}
+
+	// Base is the /24 network: zero out the last octet
+	base := make(net.IP, 4)
+	copy(base, ip4)
+	base[3] = 0
+
+	return &MgmtIPAllocator{
+		allocated: make(map[string]string),
+		baseIP:    base,
+		rangeMin:  10,
+		rangeMax:  249,
+	}, nil
+}
+
+// Allocate assigns a management IP to the given instance.
+// Returns the allocated IP string (e.g. "10.15.8.10").
+func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Check if already allocated
+	if ip, ok := a.allocated[instanceID]; ok {
+		return ip, nil
+	}
+
+	// Build set of used IPs for fast lookup
+	used := make(map[string]struct{}, len(a.allocated))
+	for _, ip := range a.allocated {
+		used[ip] = struct{}{}
+	}
+
+	// Find first free IP in range
+	for i := a.rangeMin; i <= a.rangeMax; i++ {
+		candidate := fmt.Sprintf("%d.%d.%d.%d", a.baseIP[0], a.baseIP[1], a.baseIP[2], i)
+		if _, taken := used[candidate]; !taken {
+			a.allocated[instanceID] = candidate
+			slog.Debug("Management IP allocated", "instance", instanceID, "ip", candidate)
+			return candidate, nil
+		}
+	}
+
+	return "", fmt.Errorf("management IP range exhausted (%d.%d.%d.%d–%d.%d.%d.%d)",
+		a.baseIP[0], a.baseIP[1], a.baseIP[2], a.rangeMin,
+		a.baseIP[0], a.baseIP[1], a.baseIP[2], a.rangeMax)
+}
+
+// Release frees the management IP for the given instance.
+func (a *MgmtIPAllocator) Release(instanceID string) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if ip, ok := a.allocated[instanceID]; ok {
+		delete(a.allocated, instanceID)
+		slog.Debug("Management IP released", "instance", instanceID, "ip", ip)
+	}
+}
+
+// Rebuild reconstructs the allocated set from existing VMs.
+// Called on daemon startup after loading VMs from KV store.
+func (a *MgmtIPAllocator) Rebuild(vms map[string]*vm.VM) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	for id, v := range vms {
+		if v.MgmtIP != "" {
+			a.allocated[id] = v.MgmtIP
+		}
+	}
+
+	slog.Info("Management IP allocator rebuilt", "count", len(a.allocated))
+}
+
+// AllocatedCount returns the number of currently allocated IPs.
+func (a *MgmtIPAllocator) AllocatedCount() int {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return len(a.allocated)
+}

spinifex/daemon/daemon_mgmt_ip_test.go

@@ -0,0 +1,232 @@
+package daemon
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+func TestNewMgmtIPAllocator(t *testing.T) {
+	tests := []struct {
+		name      string
+		bridgeIP  string
+		wantErr   bool
+		wantBase3 byte // expected 4th octet of base (always 0)
+	}{
+		{"valid", "10.15.8.1", false, 0},
+		{"different subnet", "192.168.1.33", false, 0},
+		{"invalid", "not-an-ip", true, 0},
+		{"ipv6", "::1", true, 0},
+		{"empty", "", true, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a, err := NewMgmtIPAllocator(tt.bridgeIP)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatal("expected error")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if a.baseIP[3] != tt.wantBase3 {
+				t.Errorf("base[3] = %d, want %d", a.baseIP[3], tt.wantBase3)
+			}
+		})
+	}
+}
+
+func TestMgmtIPAllocator_Allocate(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// First allocation gets .10
+	ip, err := a.Allocate("i-first")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "10.15.8.10" {
+		t.Errorf("first IP = %q, want 10.15.8.10", ip)
+	}
+
+	// Second allocation gets .11
+	ip, err = a.Allocate("i-second")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "10.15.8.11" {
+		t.Errorf("second IP = %q, want 10.15.8.11", ip)
+	}
+
+	// Re-allocating same instance returns same IP
+	ip, err = a.Allocate("i-first")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "10.15.8.10" {
+		t.Errorf("re-allocate = %q, want 10.15.8.10", ip)
+	}
+
+	if a.AllocatedCount() != 2 {
+		t.Errorf("count = %d, want 2", a.AllocatedCount())
+	}
+}
+
+func TestMgmtIPAllocator_Release(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a.Allocate("i-one")
+	a.Allocate("i-two")
+	a.Release("i-one")
+
+	if a.AllocatedCount() != 1 {
+		t.Errorf("count after release = %d, want 1", a.AllocatedCount())
+	}
+
+	// Released IP should be reused
+	ip, err := a.Allocate("i-three")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "10.15.8.10" {
+		t.Errorf("reused IP = %q, want 10.15.8.10", ip)
+	}
+}
+
+func TestMgmtIPAllocator_ReleaseNonexistent(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Should not panic
+	a.Release("i-nonexistent")
+}
+
+func TestMgmtIPAllocator_Exhaustion(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Fill all 240 slots (.10–.249)
+	for i := range 240 {
+		_, err := a.Allocate(fmt.Sprintf("i-%d", i))
+		if err != nil {
+			t.Fatalf("allocation %d failed: %v", i, err)
+		}
+	}
+
+	// Next should fail
+	_, err = a.Allocate("i-overflow")
+	if err == nil {
+		t.Fatal("expected exhaustion error")
+	}
+
+	// Release one, then it should work
+	a.Release("i-0")
+	ip, err := a.Allocate("i-overflow")
+	if err != nil {
+		t.Fatalf("after release: %v", err)
+	}
+	if ip != "10.15.8.10" {
+		t.Errorf("reused IP = %q, want 10.15.8.10", ip)
+	}
+}
+
+func TestMgmtIPAllocator_Rebuild(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	vms := map[string]*vm.VM{
+		"i-a": {MgmtIP: "10.15.8.10"},
+		"i-b": {MgmtIP: "10.15.8.15"},
+		"i-c": {MgmtIP: ""}, // no mgmt NIC
+		"i-d": {MgmtIP: "10.15.8.20"},
+	}
+
+	a.Rebuild(vms)
+
+	if a.AllocatedCount() != 3 {
+		t.Errorf("count after rebuild = %d, want 3", a.AllocatedCount())
+	}
+
+	// Next allocation should skip the already-used IPs
+	ip, err := a.Allocate("i-new")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "10.15.8.11" {
+		t.Errorf("next IP after rebuild = %q, want 10.15.8.11", ip)
+	}
+}
+
+func TestMgmtIPAllocator_DifferentSubnet(t *testing.T) {
+	a, err := NewMgmtIPAllocator("192.168.1.33")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ip, err := a.Allocate("i-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if ip != "192.168.1.10" {
+		t.Errorf("IP = %q, want 192.168.1.10", ip)
+	}
+}
+
+func TestMgmtIPAllocator_Concurrent(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	results := make(map[string]string)
+	var mu sync.Mutex
+	errs := make([]error, 0)
+
+	for i := range 50 {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			id := fmt.Sprintf("i-%d", n)
+			ip, err := a.Allocate(id)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs = append(errs, err)
+				return
+			}
+			results[id] = ip
+		}(i)
+	}
+	wg.Wait()
+
+	if len(errs) > 0 {
+		t.Fatalf("allocation errors: %v", errs)
+	}
+	if len(results) != 50 {
+		t.Errorf("got %d results, want 50", len(results))
+	}
+
+	// All IPs should be unique
+	seen := make(map[string]bool)
+	for id, ip := range results {
+		if seen[ip] {
+			t.Errorf("duplicate IP %s for %s", ip, id)
+		}
+		seen[ip] = true
+	}
+}

spinifex/daemon/daemon_system_instance.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
+	"github.com/mulgadc/spinifex/spinifex/tags"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 )
@@ -25,6 +26,12 @@ var _ handlers_elbv2.SystemInstanceLauncher = (*Daemon)(nil)
 // path, but skips the NATS request/response envelope and key pair validation.
 func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) {
 	accountID := utils.GlobalAccountID
+	// ENI account may differ from system account — the ENI is created under
+	// the caller's account, so lookups/updates must use that account ID.
+	eniAccountID := input.AccountID
+	if eniAccountID == "" {
+		eniAccountID = accountID
+	}
 
 	// Validate instance type
 	instanceType, exists := d.resourceMgr.instanceTypes[input.InstanceType]
@@ -45,12 +52,22 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		return nil, fmt.Errorf("insufficient capacity for %s: %w", input.InstanceType, err)
 	}
 
-	// Build a RunInstancesInput for the instance service
+	// Build a RunInstancesInput for the instance service.
+	// Tag the instance as ELBv2-managed so the UI hides it from
+	// customer-facing listings (Nodes page).
 	runInput := &ec2.RunInstancesInput{
 		InstanceType: aws.String(input.InstanceType),
 		ImageId:      aws.String(input.ImageID),
 		MinCount:     aws.Int64(1),
 		MaxCount:     aws.Int64(1),
+		TagSpecifications: []*ec2.TagSpecification{
+			{
+				ResourceType: aws.String("instance"),
+				Tags: []*ec2.Tag{
+					{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByELBv2)},
+				},
+			},
+		},
 	}
 	if input.SubnetID != "" {
 		runInput.SubnetId = aws.String(input.SubnetID)
@@ -68,6 +85,10 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 	}
 
 	instance.AccountID = accountID
+	instance.ManagedBy = tags.ManagedByELBv2
+	ec2Instance.Tags = []*ec2.Tag{
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByELBv2)},
+	}
 	instance.Reservation = &ec2.Reservation{}
 	instance.Reservation.SetReservationId(utils.GenerateResourceID("r"))
 	instance.Reservation.SetOwnerId(accountID)
@@ -86,10 +107,29 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		}
 		// Mark ENI as attached to this instance
 		if d.vpcService != nil {
-			if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+			if _, attachErr := d.vpcService.AttachENI(eniAccountID, instance.ENIId, instance.ID, 0); attachErr != nil {
 				slog.Warn("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
 			}
 		}
+		// Attach any additional pre-created ENIs (multi-subnet ALB VMs).
+		// Use the launcher input slice as the source of truth so the VM
+		// record carries every NIC the ALB spans even if the attach fails
+		// for one of them (we clean up on failure below).
+		for idx, extra := range input.ExtraENIs {
+			if d.vpcService != nil {
+				if _, attachErr := d.vpcService.AttachENI(eniAccountID, extra.ENIID, instance.ID, int64(idx+1)); attachErr != nil {
+					slog.Error("LaunchSystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", instance.ID, "err", attachErr)
+					d.cleanupFailedSystemInstance(instance, instanceType)
+					return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
+				}
+			}
+			instance.ExtraENIs = append(instance.ExtraENIs, vm.ExtraENI{
+				ENIID:    extra.ENIID,
+				ENIMac:   extra.ENIMac,
+				ENIIP:    extra.ENIIP,
+				SubnetID: extra.SubnetID,
+			})
+		}
 	} else if input.SubnetID != "" && d.vpcService != nil {
 		// Auto-create ENI in subnet
 		eniOut, eniErr := d.vpcService.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{
@@ -113,6 +153,45 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		}
 	}
 
+	// Allocate public IP for internet-facing ALBs
+	publicIP := ""
+	if input.Scheme == handlers_elbv2.SchemeInternetFacing && d.externalIPAM != nil && d.vpcService != nil && instance.ENIId != "" {
+		region := ""
+		az := ""
+		if d.config != nil {
+			region = d.config.Region
+			az = d.config.AZ
+		}
+		allocatedIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", "", instance.ENIId, instance.ID)
+		if allocErr != nil {
+			slog.Error("LaunchSystemInstance: failed to allocate public IP for internet-facing ALB", "instanceId", instance.ID, "err", allocErr)
+			d.cleanupFailedSystemInstance(instance, instanceType)
+			return nil, fmt.Errorf("allocate public IP for internet-facing ALB: %w", allocErr)
+		} else {
+			publicIP = allocatedIP
+			if updateErr := d.vpcService.UpdateENIPublicIP(eniAccountID, instance.ENIId, publicIP, poolName); updateErr != nil {
+				slog.Warn("LaunchSystemInstance: failed to update ENI with public IP", "eniId", instance.ENIId, "err", updateErr)
+			}
+			// Look up VpcId from the ENI for the NAT event
+			vpcID := ""
+			result, descErr := d.vpcService.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
+				NetworkInterfaceIds: []*string{aws.String(instance.ENIId)},
+			}, eniAccountID)
+			if descErr == nil && len(result.NetworkInterfaces) > 0 && result.NetworkInterfaces[0].VpcId != nil {
+				vpcID = *result.NetworkInterfaces[0].VpcId
+			}
+			portName := "port-" + instance.ENIId
+			d.publishNATEvent("vpc.add-nat", vpcID, publicIP, privateIP, portName, instance.ENIMac)
+			instance.PublicIP = publicIP
+			instance.PublicIPPool = poolName
+			slog.Info("LaunchSystemInstance: allocated public IP",
+				"instanceId", instance.ID,
+				"publicIp", publicIP,
+				"pool", poolName,
+			)
+		}
+	}
+
 	// Add to daemon state so LaunchInstance can find it
 	d.Instances.Mu.Lock()
 	d.Instances.VMS[instance.ID] = instance
@@ -137,6 +216,42 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		instance.DevMAC = generateDevMAC(instance.ID)
 	}
 
+	// Management NIC: allocate IP, generate MAC, create TAP on br-mgmt
+	if d.mgmtIPAllocator != nil && d.mgmtBridgeIP != "" {
+		mgmtBridge := "br-mgmt"
+		if d.config.Daemon.MgmtBridge != "" {
+			mgmtBridge = d.config.Daemon.MgmtBridge
+		}
+
+		mgmtIP, allocErr := d.mgmtIPAllocator.Allocate(instance.ID)
+		if allocErr != nil {
+			if input.Scheme == handlers_elbv2.SchemeInternal {
+				d.cleanupFailedSystemInstance(instance, instanceType)
+				return nil, fmt.Errorf("allocate mgmt IP for internal-scheme ALB: %w", allocErr)
+			}
+			slog.Warn("LaunchSystemInstance: failed to allocate mgmt IP, skipping mgmt NIC", "instanceId", instance.ID, "err", allocErr)
+		} else {
+			instance.MgmtMAC = generateMgmtMAC(instance.ID)
+			instance.MgmtIP = mgmtIP
+
+			tapName, tapErr := SetupMgmtTapDevice(instance.ID, instance.MgmtMAC, mgmtBridge)
+			if tapErr != nil {
+				d.mgmtIPAllocator.Release(instance.ID)
+				instance.MgmtMAC = ""
+				instance.MgmtIP = ""
+				if input.Scheme == handlers_elbv2.SchemeInternal {
+					d.cleanupFailedSystemInstance(instance, instanceType)
+					return nil, fmt.Errorf("setup mgmt tap for internal-scheme ALB: %w", tapErr)
+				}
+				slog.Error("LaunchSystemInstance: failed to setup mgmt tap", "instanceId", instance.ID, "err", tapErr)
+			} else {
+				instance.MgmtTap = tapName
+				slog.Info("LaunchSystemInstance: mgmt NIC configured",
+					"instanceId", instance.ID, "mgmtIP", mgmtIP, "mgmtMAC", instance.MgmtMAC, "mgmtTap", tapName)
+			}
+		}
+	}
+
 	// Store extra hostfwd ports (filled in by StartInstance with actual host ports)
 	if len(input.HostfwdPorts) > 0 {
 		instance.ExtraHostfwd = make(map[int]int, len(input.HostfwdPorts))
@@ -174,11 +289,14 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
 		"instanceId", instance.ID,
 		"instanceType", input.InstanceType,
 		"privateIp", privateIP,
+		"publicIp", publicIP,
+		"scheme", input.Scheme,
 	)
 
 	return &handlers_elbv2.SystemInstanceOutput{
 		InstanceID: instance.ID,
 		PrivateIP:  privateIP,
+		PublicIP:   publicIP,
 		HostfwdMap: instance.ExtraHostfwd,
 	}, nil
 }
@@ -245,6 +363,16 @@ func (d *Daemon) cleanupFailedSystemInstance(instance *vm.VM, instanceType *ec2.
 	d.markInstanceFailed(instance, "system_instance_launch_failed")
 	d.resourceMgr.deallocate(instanceType)
 
+	// Clean up management TAP and release IP
+	if instance.MgmtTap != "" {
+		if err := CleanupMgmtTapDevice(instance.MgmtTap); err != nil {
+			slog.Warn("Failed to cleanup mgmt tap on failed launch", "tap", instance.MgmtTap, "err", err)
+		}
+		if d.mgmtIPAllocator != nil {
+			d.mgmtIPAllocator.Release(instance.ID)
+		}
+	}
+
 	// Clean up ENI if we auto-created one (not pre-created)
 	// Pre-created ENIs are managed by the caller (e.g. ELBv2 service)
 	if instance.ENIId != "" && d.vpcService != nil {

spinifex/daemon/daemon_test.go

@@ -1,9 +1,19 @@
 package daemon
 
 import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
+	"math/big"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -18,6 +28,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
+	"github.com/mulgadc/spinifex/spinifex/instancetypes"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/qmp"
 	"github.com/mulgadc/spinifex/spinifex/types"
@@ -806,7 +817,10 @@ func TestHandleEC2DescribeInstanceTypes(t *testing.T) {
 		// With 2 vCPUs and 16GB, every type with 2 vCPUs and <=16GB memory should have 1 slot.
 		// Calculate expected by counting fitting types directly.
 		expectedSlots := 0
-		for _, it := range daemon.resourceMgr.instanceTypes {
+		for name, it := range daemon.resourceMgr.instanceTypes {
+			if instancetypes.IsSystemType(name) {
+				continue
+			}
 			vcpus := *it.VCpuInfo.DefaultVCpus
 			memGB := float64(*it.MemoryInfo.SizeInMiB) / 1024.0
 			if vcpus <= 2 && memGB <= 16.0 {
@@ -1396,9 +1410,12 @@ func TestInstanceTypeSubscriptions(t *testing.T) {
 		handler := func(msg *nats.Msg) {}
 		rm.initSubscriptions(nc, handler, "test-node")
 
-		// Count how many types actually fit on this machine
+		// Count how many types actually fit on this machine (excluding system types)
 		fittableTypes := 0
-		for _, typeInfo := range rm.instanceTypes {
+		for name, typeInfo := range rm.instanceTypes {
+			if instancetypes.IsSystemType(name) {
+				continue
+			}
 			if rm.canAllocate(typeInfo, 1) >= 1 {
 				fittableTypes++
 			}
@@ -4026,3 +4043,137 @@ func TestBuildDrives_CloudInitVolume(t *testing.T) {
 	assert.Equal(t, "cdrom", d.Media)
 	assert.Equal(t, "cloudinit", d.ID)
 }
+
+// --- ClusterManager TLS ---
+
+func TestClusterManager_TLSServesHTTPS(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	// Generate self-signed cert for testing
+	certPEM, keyPEM := generateTestCert(t)
+	certPath := filepath.Join(tmpDir, "server.pem")
+	keyPath := filepath.Join(tmpDir, "server.key")
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0600))
+
+	// Find a free port
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr := ln.Addr().String()
+	ln.Close()
+
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				BaseDir: tmpDir,
+				Daemon: config.DaemonConfig{
+					Host:    addr,
+					TLSCert: certPath,
+					TLSKey:  keyPath,
+				},
+			},
+		},
+	}
+
+	daemon, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	daemon.configPath = filepath.Join(tmpDir, "spinifex.toml")
+
+	err = daemon.ClusterManager()
+	require.NoError(t, err)
+	defer daemon.clusterServer.Close()
+
+	// Plain HTTP should fail
+	_, err = net.DialTimeout("tcp", addr, time.Second)
+	require.NoError(t, err) // TCP connects, but HTTP won't work
+
+	// HTTPS with InsecureSkipVerify should succeed
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Timeout: 2 * time.Second,
+	}
+	resp, err := client.Get(fmt.Sprintf("https://%s/health", addr))
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func TestClusterManager_EmptyCertReturnsError(t *testing.T) {
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				Daemon: config.DaemonConfig{
+					Host: "127.0.0.1:0",
+				},
+			},
+		},
+	}
+
+	daemon, err := NewDaemon(cfg)
+	require.NoError(t, err)
+
+	err = daemon.ClusterManager()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "TLS not configured")
+}
+
+func TestClusterManager_InvalidCertReturnsError(t *testing.T) {
+	tmpDir := t.TempDir()
+	certPath := filepath.Join(tmpDir, "bad.pem")
+	keyPath := filepath.Join(tmpDir, "bad.key")
+	require.NoError(t, os.WriteFile(certPath, []byte("not a cert"), 0600))
+	require.NoError(t, os.WriteFile(keyPath, []byte("not a key"), 0600))
+
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {
+				Daemon: config.DaemonConfig{
+					Host:    "127.0.0.1:0",
+					TLSCert: certPath,
+					TLSKey:  keyPath,
+				},
+			},
+		},
+	}
+
+	daemon, err := NewDaemon(cfg)
+	require.NoError(t, err)
+
+	err = daemon.ClusterManager()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cluster manager load TLS cert")
+}
+
+// generateTestCert creates a self-signed certificate for testing.
+func generateTestCert(t *testing.T) (certPEM, keyPEM []byte) {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "test"},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().Add(time.Hour),
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+	}
+
+	certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	certBuf := &bytes.Buffer{}
+	require.NoError(t, pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: certDER}))
+
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	keyBuf := &bytes.Buffer{}
+	require.NoError(t, pem.Encode(keyBuf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}))
+
+	return certBuf.Bytes(), keyBuf.Bytes()
+}

spinifex/daemon/daemon_wire_lb_test.go

@@ -0,0 +1,174 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/config"
+	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newWireLBTestDaemon creates a minimal Daemon with an ELBv2 service for
+// wireLBAgentConfig tests. Only the fields read by wireLBAgentConfig are set.
+func newWireLBTestDaemon(t *testing.T, cfg *config.Config) *Daemon {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc)
+	require.NoError(t, err)
+	t.Cleanup(func() { svc.Close() })
+
+	return &Daemon{
+		config:       cfg,
+		elbv2Service: svc,
+	}
+}
+
+func TestWireLBAgentConfig_Credentials(t *testing.T) {
+	cfg := &config.Config{
+		Predastore: config.PredastoreConfig{
+			AccessKey: "AKID-test",
+			SecretKey: "SK-test",
+		},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+
+	d.wireLBAgentConfig()
+
+	assert.Equal(t, "AKID-test", d.systemAccessKey)
+	assert.Equal(t, "SK-test", d.systemSecretKey)
+}
+
+func TestWireLBAgentConfig_NoCredentials(t *testing.T) {
+	cfg := &config.Config{}
+	d := newWireLBTestDaemon(t, cfg)
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.systemAccessKey)
+	assert.Empty(t, d.systemSecretKey)
+}
+
+func TestWireLBAgentConfig_GatewayURL_MultiNode(t *testing.T) {
+	// Multi-node: mgmtBridgeIP set, AWSGW binds to a specific WAN IP.
+	// Gateway should use the AWSGW bind IP, not br-mgmt.
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "192.168.1.10:9999"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Equal(t, "192.168.1.10", d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_GatewayURL_SingleNode(t *testing.T) {
+	// Single-node: mgmtBridgeIP set, AWSGW on 0.0.0.0 → use br-mgmt IP.
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_GatewayURL_MgmtNoAWSGW(t *testing.T) {
+	// mgmtBridgeIP set, no AWSGW Host → use br-mgmt IP.
+	cfg := &config.Config{}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_GatewayURL_DevNetworking(t *testing.T) {
+	cfg := &config.Config{
+		Daemon: config.DaemonConfig{DevNetworking: true},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+
+	d.wireLBAgentConfig()
+
+	// DevNetworking fallback uses host gateway 10.0.2.2
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_GatewayURL_AWSGWBindOnly(t *testing.T) {
+	// No mgmtBridgeIP, no DevNetworking, AWSGW binds to specific IP.
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "192.168.1.10:8443"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_GatewayURL_NoHost(t *testing.T) {
+	// No mgmtBridgeIP, no DevNetworking, no AWSGW → no gateway URL.
+	cfg := &config.Config{}
+	d := newWireLBTestDaemon(t, cfg)
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_PortExtraction(t *testing.T) {
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "192.168.1.10:8443"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	// Port should be extracted from AWSGW Host, not default 9999
+	assert.Equal(t, "192.168.1.10", d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_DefaultPort(t *testing.T) {
+	// AWSGW Host without port → default 9999.
+	// Use mgmtBridgeIP with no AWSGW bind IP so gateway uses br-mgmt.
+	cfg := &config.Config{}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_MgmtRoute(t *testing.T) {
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "192.168.1.10:9999"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Equal(t, "192.168.1.10", d.mgmtRouteVia)
+}
+
+func TestWireLBAgentConfig_NoMgmtRoute(t *testing.T) {
+	// Single-node: AWSGW on 0.0.0.0 → no mgmtRouteVia set.
+	cfg := &config.Config{
+		AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"},
+	}
+	d := newWireLBTestDaemon(t, cfg)
+	d.mgmtBridgeIP = "10.15.8.1"
+
+	d.wireLBAgentConfig()
+
+	assert.Empty(t, d.mgmtRouteVia)
+}

spinifex/daemon/jetstream.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/migrate"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
@@ -83,8 +84,8 @@ func (m *JetStreamManager) InitKVBucket() error {
 	}
 
 	m.kv = kv
-	if err := utils.WriteVersion(kv, InstanceStateBucketVersion); err != nil {
-		return fmt.Errorf("write version to %s: %w", InstanceStateBucket, err)
+	if err := migrate.DefaultRegistry.RunKV(InstanceStateBucket, kv, InstanceStateBucketVersion); err != nil {
+		return fmt.Errorf("migrate %s: %w", InstanceStateBucket, err)
 	}
 	return nil
 }
@@ -113,8 +114,8 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
 	}
 
 	m.clusterKV = kv
-	if err := utils.WriteVersion(kv, ClusterStateBucketVersion); err != nil {
-		return fmt.Errorf("write version to %s: %w", ClusterStateBucket, err)
+	if err := migrate.DefaultRegistry.RunKV(ClusterStateBucket, kv, ClusterStateBucketVersion); err != nil {
+		return fmt.Errorf("migrate %s: %w", ClusterStateBucket, err)
 	}
 	return nil
 }
@@ -144,8 +145,8 @@ func (m *JetStreamManager) InitTerminatedInstanceBucket() error {
 	}
 
 	m.terminatedKV = kv
-	if err := utils.WriteVersion(kv, TerminatedInstanceBucketVersion); err != nil {
-		return fmt.Errorf("write version to %s: %w", TerminatedInstanceBucket, err)
+	if err := migrate.DefaultRegistry.RunKV(TerminatedInstanceBucket, kv, TerminatedInstanceBucketVersion); err != nil {
+		return fmt.Errorf("migrate %s: %w", TerminatedInstanceBucket, err)
 	}
 	return nil
 }
@@ -199,8 +200,9 @@ func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.K
 		return nil, err
 	}
 
-	if err := utils.WriteVersion(kv, version); err != nil {
-		slog.Error("Failed to write version to recreated bucket", "bucket", cfg.Bucket, "err", err)
+	if err := migrate.DefaultRegistry.RunKV(cfg.Bucket, kv, version); err != nil {
+		slog.Error("Failed to run migrations on recreated bucket", "bucket", cfg.Bucket, "err", err)
+		return nil, fmt.Errorf("migrate recreated bucket %s: %w", cfg.Bucket, err)
 	}
 
 	*field = kv

spinifex/daemon/network.go

@@ -7,6 +7,9 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/mulgadc/spinifex/spinifex/vm"
 )
 
 // sudoCommand wraps exec.Command with sudo when running as non-root.
@@ -95,6 +98,42 @@ func (p *OVSNetworkPlumber) CleanupTapDevice(eniId string) error {
 	return nil
 }
 
+// setupExtraENINICs creates tap devices on br-int and appends matching QEMU
+// virtio-net device entries to instance.Config for each additional ENI a
+// system VM spans. The primary ENI (instance.ENIId) is handled separately by
+// the LaunchInstance caller. Cloud-init brings the guest interfaces up via
+// per-MAC DHCP blocks written by generateNetworkConfig.
+func (d *Daemon) setupExtraENINICs(instance *vm.VM) error {
+	for idx, extra := range instance.ExtraENIs {
+		if err := d.networkPlumber.SetupTapDevice(extra.ENIID, extra.ENIMac); err != nil {
+			slog.Error("Failed to set up tap device for extra ENI", "eni", extra.ENIID, "err", err)
+			return fmt.Errorf("setup tap device for extra ENI %s: %w", extra.ENIID, err)
+		}
+		extraTapName := TapDeviceName(extra.ENIID)
+		netID := fmt.Sprintf("net%d", idx+1)
+		instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
+			Value: fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", netID, extraTapName),
+		})
+		instance.Config.Devices = append(instance.Config.Devices, vm.Device{
+			Value: fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s", netID, extra.ENIMac),
+		})
+		slog.Info("Extra VPC NIC configured",
+			"tap", extraTapName, "eni", extra.ENIID, "mac", extra.ENIMac, "subnet", extra.SubnetID)
+	}
+	return nil
+}
+
+// cleanupExtraENITaps removes tap devices for every extra ENI attached to a
+// system VM. Errors are logged but not returned so a partial cleanup still
+// frees as many resources as possible.
+func (d *Daemon) cleanupExtraENITaps(instance *vm.VM) {
+	for _, extra := range instance.ExtraENIs {
+		if err := d.networkPlumber.CleanupTapDevice(extra.ENIID); err != nil {
+			slog.Warn("Failed to clean up extra ENI tap device", "eni", extra.ENIID, "err", err)
+		}
+	}
+}
+
 // TapDeviceName returns the Linux tap device name for an ENI.
 // Linux IFNAMSIZ limits interface names to 15 characters.
 // ENI IDs like "eni-abc123def456789" are too long, so we truncate.
@@ -115,13 +154,106 @@ func OVSIfaceID(eniId string) string {
 
 // generateDevMAC creates a locally-administered unicast MAC for the dev/hostfwd NIC.
 // Uses prefix 02:de:00 to distinguish from ENI MACs (02:00:00).
-// All octets must be valid hex for QEMU's virtio-net-pci mac property.
 func generateDevMAC(instanceId string) string {
-	h := uint32(0)
-	for _, c := range instanceId {
-		h = h*31 + uint32(c) // #nosec G115 -- intentional overflow for hashing
+	return utils.HashMAC("02:de:00", instanceId)
+}
+
+// generateMgmtMAC creates a locally-administered unicast MAC for the management NIC.
+// Uses prefix 02:a0:00 to distinguish from ENI MACs (02:00:00) and dev MACs (02:de:00).
+func generateMgmtMAC(instanceId string) string {
+	return utils.HashMAC("02:a0:00", instanceId)
+}
+
+// MgmtTapName returns the Linux TAP device name for a management NIC.
+// Uses "mg" prefix + truncated instance ID to stay within 15-char IFNAMSIZ.
+func MgmtTapName(instanceID string) string {
+	id := strings.TrimPrefix(instanceID, "i-")
+	name := "mg" + id
+	if len(name) > 15 {
+		name = name[:15]
 	}
-	return fmt.Sprintf("02:de:00:%02x:%02x:%02x", (h>>16)&0xff, (h>>8)&0xff, h&0xff)
+	return name
+}
+
+// GetBridgeIPv4 returns the first IPv4 address on the named Linux bridge.
+// Returns "", nil if the bridge does not exist.
+func GetBridgeIPv4(bridgeName string) (string, error) {
+	iface, err := net.InterfaceByName(bridgeName)
+	if err != nil {
+		// "no such network interface" means the bridge doesn't exist yet — expected.
+		// Other errors (permission denied on /sys/class/net/, etc.) are real failures.
+		if strings.Contains(err.Error(), "no such network interface") {
+			return "", nil
+		}
+		return "", fmt.Errorf("lookup %s: %w", bridgeName, err)
+	}
+	addrs, err := iface.Addrs()
+	if err != nil {
+		return "", fmt.Errorf("list addrs on %s: %w", bridgeName, err)
+	}
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok {
+			continue
+		}
+		if ipNet.IP.To4() != nil {
+			return ipNet.IP.String(), nil
+		}
+	}
+	return "", fmt.Errorf("no IPv4 address on %s", bridgeName)
+}
+
+// SetupMgmtTapDevice creates a TAP device and adds it to the management Linux bridge.
+// Unlike VPC TAPs (OVS/OVN), this is a plain Linux bridge TAP.
+func SetupMgmtTapDevice(instanceID, mac, bridge string) (string, error) {
+	tapName := MgmtTapName(instanceID)
+
+	// Clean up stale TAP if present
+	if _, err := os.Stat("/sys/class/net/" + tapName); err == nil {
+		slog.Warn("Stale mgmt tap found, cleaning up", "tap", tapName)
+		if err := sudoCommand("ip", "link", "set", tapName, "nomaster").Run(); err != nil {
+			slog.Warn("Failed to remove stale mgmt tap from bridge", "tap", tapName, "err", err)
+		}
+		if err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run(); err != nil {
+			slog.Warn("Failed to delete stale mgmt tap", "tap", tapName, "err", err)
+		}
+	}
+
+	// Create TAP
+	if out, err := sudoCommand("ip", "tuntap", "add", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
+		return "", fmt.Errorf("create mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+	}
+
+	// Bring up
+	if out, err := sudoCommand("ip", "link", "set", tapName, "up").CombinedOutput(); err != nil {
+		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
+		return "", fmt.Errorf("bring up mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+	}
+
+	// Add to Linux bridge
+	if out, err := sudoCommand("ip", "link", "set", tapName, "master", bridge).CombinedOutput(); err != nil {
+		_ = sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").Run()
+		return "", fmt.Errorf("add mgmt tap to %s: %s: %w", bridge, strings.TrimSpace(string(out)), err)
+	}
+
+	slog.Info("Management TAP created", "tap", tapName, "bridge", bridge, "mac", mac)
+	return tapName, nil
+}
+
+// CleanupMgmtTapDevice removes a management TAP device from its bridge and deletes it.
+func CleanupMgmtTapDevice(tapName string) error {
+	// Remove from bridge
+	if out, err := sudoCommand("ip", "link", "set", tapName, "nomaster").CombinedOutput(); err != nil {
+		slog.Warn("Failed to remove mgmt tap from bridge", "tap", tapName, "err", err, "out", strings.TrimSpace(string(out)))
+	}
+
+	// Delete TAP
+	if out, err := sudoCommand("ip", "tuntap", "del", "dev", tapName, "mode", "tap").CombinedOutput(); err != nil {
+		return fmt.Errorf("delete mgmt tap %s: %s: %w", tapName, strings.TrimSpace(string(out)), err)
+	}
+
+	slog.Info("Management TAP cleaned up", "tap", tapName)
+	return nil
 }
 
 // OVNHealthStatus reports the readiness of OVN networking on this compute node.

spinifex/daemon/network_test.go

@@ -238,6 +238,139 @@ func TestNetworkPlumber_InterfaceCompliance(t *testing.T) {
 	var _ NetworkPlumber = &MockNetworkPlumber{}
 }
 
+func TestSetupExtraENINICs_AppendsOnePerExtra(t *testing.T) {
+	mock := &MockNetworkPlumber{}
+	d := &Daemon{networkPlumber: mock}
+	instance := &vm.VM{
+		ID: "i-multi",
+		ExtraENIs: []vm.ExtraENI{
+			{ENIID: "eni-aaa", ENIMac: "02:00:00:aa:aa:aa", ENIIP: "10.0.1.4", SubnetID: "subnet-a"},
+			{ENIID: "eni-bbb", ENIMac: "02:00:00:bb:bb:bb", ENIIP: "10.0.2.4", SubnetID: "subnet-b"},
+		},
+	}
+
+	if err := d.setupExtraENINICs(instance); err != nil {
+		t.Fatalf("setupExtraENINICs failed: %v", err)
+	}
+
+	// One SetupTapDevice call per extra ENI.
+	if len(mock.SetupCalls) != 2 {
+		t.Fatalf("expected 2 SetupTapDevice calls, got %d", len(mock.SetupCalls))
+	}
+	if mock.SetupCalls[0].ENIId != "eni-aaa" || mock.SetupCalls[0].MAC != "02:00:00:aa:aa:aa" {
+		t.Errorf("first setup call = %+v, want eni-aaa/02:00:00:aa:aa:aa", mock.SetupCalls[0])
+	}
+	if mock.SetupCalls[1].ENIId != "eni-bbb" || mock.SetupCalls[1].MAC != "02:00:00:bb:bb:bb" {
+		t.Errorf("second setup call = %+v, want eni-bbb/02:00:00:bb:bb:bb", mock.SetupCalls[1])
+	}
+
+	// NetDevs and Devices each get one entry per extra ENI, named net1/net2.
+	if len(instance.Config.NetDevs) != 2 || len(instance.Config.Devices) != 2 {
+		t.Fatalf("expected 2 netdevs + 2 devices, got %d + %d",
+			len(instance.Config.NetDevs), len(instance.Config.Devices))
+	}
+	if !strings.Contains(instance.Config.NetDevs[0].Value, "id=net1") {
+		t.Errorf("netdev[0] = %q, want id=net1", instance.Config.NetDevs[0].Value)
+	}
+	if !strings.Contains(instance.Config.NetDevs[1].Value, "id=net2") {
+		t.Errorf("netdev[1] = %q, want id=net2", instance.Config.NetDevs[1].Value)
+	}
+	if !strings.Contains(instance.Config.Devices[0].Value, "mac=02:00:00:aa:aa:aa") {
+		t.Errorf("device[0] = %q, missing primary MAC", instance.Config.Devices[0].Value)
+	}
+	if !strings.Contains(instance.Config.Devices[1].Value, "mac=02:00:00:bb:bb:bb") {
+		t.Errorf("device[1] = %q, missing second MAC", instance.Config.Devices[1].Value)
+	}
+}
+
+func TestSetupExtraENINICs_NoExtras_NoOp(t *testing.T) {
+	mock := &MockNetworkPlumber{}
+	d := &Daemon{networkPlumber: mock}
+	instance := &vm.VM{ID: "i-single"}
+
+	if err := d.setupExtraENINICs(instance); err != nil {
+		t.Fatalf("setupExtraENINICs failed: %v", err)
+	}
+	if len(mock.SetupCalls) != 0 {
+		t.Errorf("expected zero setup calls for no extras, got %d", len(mock.SetupCalls))
+	}
+	if len(instance.Config.NetDevs) != 0 || len(instance.Config.Devices) != 0 {
+		t.Errorf("expected no netdevs/devices, got %d/%d",
+			len(instance.Config.NetDevs), len(instance.Config.Devices))
+	}
+}
+
+func TestSetupExtraENINICs_TapSetupErrorReturns(t *testing.T) {
+	mock := &MockNetworkPlumber{SetupErr: fmt.Errorf("simulated tap failure")}
+	d := &Daemon{networkPlumber: mock}
+	instance := &vm.VM{
+		ID: "i-multi-err",
+		ExtraENIs: []vm.ExtraENI{
+			{ENIID: "eni-aaa", ENIMac: "02:00:00:aa:aa:aa"},
+			{ENIID: "eni-bbb", ENIMac: "02:00:00:bb:bb:bb"},
+		},
+	}
+
+	err := d.setupExtraENINICs(instance)
+	if err == nil {
+		t.Fatal("expected error from failing tap setup, got nil")
+	}
+	if !strings.Contains(err.Error(), "eni-aaa") {
+		t.Errorf("error = %v, want it to mention the failing ENI", err)
+	}
+	// Must bail on first failure — second ENI should not be touched.
+	if len(mock.SetupCalls) != 1 {
+		t.Errorf("expected 1 setup call before bailout, got %d", len(mock.SetupCalls))
+	}
+	// No NIC config appended for a failed setup.
+	if len(instance.Config.NetDevs) != 0 {
+		t.Errorf("expected no netdevs on failure, got %d", len(instance.Config.NetDevs))
+	}
+}
+
+func TestCleanupExtraENITaps_CallsCleanupPerExtra(t *testing.T) {
+	mock := &MockNetworkPlumber{}
+	d := &Daemon{networkPlumber: mock}
+	instance := &vm.VM{
+		ID: "i-multi-clean",
+		ExtraENIs: []vm.ExtraENI{
+			{ENIID: "eni-111"},
+			{ENIID: "eni-222"},
+			{ENIID: "eni-333"},
+		},
+	}
+
+	d.cleanupExtraENITaps(instance)
+
+	if len(mock.CleanupCalls) != 3 {
+		t.Fatalf("expected 3 cleanup calls, got %d", len(mock.CleanupCalls))
+	}
+	for i, want := range []string{"eni-111", "eni-222", "eni-333"} {
+		if mock.CleanupCalls[i] != want {
+			t.Errorf("cleanup[%d] = %q, want %q", i, mock.CleanupCalls[i], want)
+		}
+	}
+}
+
+func TestCleanupExtraENITaps_ErrorsAreLogged(t *testing.T) {
+	mock := &MockNetworkPlumber{CleanupErr: fmt.Errorf("simulated cleanup failure")}
+	d := &Daemon{networkPlumber: mock}
+	instance := &vm.VM{
+		ID: "i-multi-clean-err",
+		ExtraENIs: []vm.ExtraENI{
+			{ENIID: "eni-111"},
+			{ENIID: "eni-222"},
+		},
+	}
+
+	// Must not panic or return — errors are swallowed by design so partial
+	// cleanup still frees later entries.
+	d.cleanupExtraENITaps(instance)
+	if len(mock.CleanupCalls) != 2 {
+		t.Errorf("expected both extras to be attempted, got %d cleanup calls", len(mock.CleanupCalls))
+	}
+}
+
 func TestOVNHealthStatus_Fields(t *testing.T) {
 	// Verify OVNHealthStatus struct can be used for health reporting
 	status := OVNHealthStatus{
@@ -435,3 +568,85 @@ func TestOVSIfaceID_Format(t *testing.T) {
 		}
 	}
 }
+
+func TestGenerateMgmtMAC(t *testing.T) {
+	tests := []string{
+		"i-abc123",
+		"i-def456",
+		"i-ghi789",
+	}
+
+	seen := make(map[string]bool)
+	for _, id := range tests {
+		mac := generateMgmtMAC(id)
+		if !strings.HasPrefix(mac, "02:a0:00:") {
+			t.Errorf("generateMgmtMAC(%q) = %q, want prefix '02:a0:00:'", id, mac)
+		}
+		if len(mac) != 17 {
+			t.Errorf("generateMgmtMAC(%q) = %q, expected 17 chars", id, mac)
+		}
+		if seen[mac] {
+			t.Errorf("generateMgmtMAC(%q) = %q, duplicate MAC", id, mac)
+		}
+		seen[mac] = true
+	}
+
+	// Deterministic
+	mac1 := generateMgmtMAC("i-test123")
+	mac2 := generateMgmtMAC("i-test123")
+	if mac1 != mac2 {
+		t.Errorf("generateMgmtMAC not deterministic: %q != %q", mac1, mac2)
+	}
+
+	// Different from dev MAC for same instance
+	devMAC := generateDevMAC("i-test123")
+	mgmtMAC := generateMgmtMAC("i-test123")
+	if devMAC == mgmtMAC {
+		t.Errorf("dev and mgmt MACs should differ for same instance: both %q", devMAC)
+	}
+}
+
+func TestMgmtTapName(t *testing.T) {
+	tests := []struct {
+		instanceID string
+		expected   string
+	}{
+		{"i-abc123", "mgabc123"},
+		{"i-abc123def456789", "mgabc123def4567"}, // Truncated to 15 chars
+		{"i-a", "mga"},
+		{"abc123", "mgabc123"}, // No i- prefix
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.instanceID, func(t *testing.T) {
+			got := MgmtTapName(tt.instanceID)
+			if got != tt.expected {
+				t.Errorf("MgmtTapName(%q) = %q, want %q", tt.instanceID, got, tt.expected)
+			}
+			if len(got) > 15 {
+				t.Errorf("MgmtTapName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.instanceID, got, len(got))
+			}
+		})
+	}
+}
+
+func TestGetBridgeIPv4_Loopback(t *testing.T) {
+	// "lo" is always present and has 127.0.0.1
+	ip, err := GetBridgeIPv4("lo")
+	if err != nil {
+		t.Fatalf("GetBridgeIPv4(lo): %v", err)
+	}
+	if ip != "127.0.0.1" {
+		t.Errorf("GetBridgeIPv4(lo) = %q, want 127.0.0.1", ip)
+	}
+}
+
+func TestGetBridgeIPv4_NonexistentBridge(t *testing.T) {
+	ip, err := GetBridgeIPv4("br-nonexistent-test-xyz")
+	if err != nil {
+		t.Fatalf("expected nil error for absent bridge, got: %v", err)
+	}
+	if ip != "" {
+		t.Errorf("expected empty IP for absent bridge, got %q", ip)
+	}
+}

spinifex/formation/formation.go

@@ -239,6 +239,7 @@ func (fs *FormationServer) Shutdown(ctx context.Context) error {
 }
 
 func (fs *FormationServer) handleJoin(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, 1*1024*1024) // 1 MB limit
 	var req JoinRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 		writeJSON(w, http.StatusBadRequest, JoinResponse{

spinifex/gateway/auth.go

@@ -31,12 +31,6 @@ const (
 func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// Skip OPTIONS requests (CORS preflight)
-			if r.Method == http.MethodOptions {
-				next.ServeHTTP(w, r)
-				return
-			}
-
 			authHeader := r.Header.Get("Authorization")
 			if authHeader == "" {
 				gw.writeSigV4Error(w, r, awserrors.ErrorMissingAuthenticationToken)
@@ -314,10 +308,7 @@ func canonicalHeaderName(header string) string {
 
 // writeSigV4Error writes an EC2-compatible XML error response for authentication failures.
 func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request, errorCode string) {
-	requestID := r.Header.Get("X-Amz-Request-Id")
-	if requestID == "" {
-		requestID = uuid.NewString()
-	}
+	requestID := uuid.NewString()
 
 	errorMsg, exists := awserrors.ErrorLookup[errorCode]
 	if !exists {

spinifex/gateway/auth_test.go

@@ -333,20 +333,6 @@ func TestSigV4Auth_ValidSignature(t *testing.T) {
 	}
 }
 
-func TestSigV4Auth_OptionsSkipsAuth(t *testing.T) {
-	handler := setupTestApp(testAccessKey, testSecretKey)
-
-	req := httptest.NewRequest(http.MethodOptions, "/", nil)
-	req.Host = "localhost:9999"
-	// No Authorization header
-
-	resp := doRequest(handler, req)
-
-	if resp.StatusCode != http.StatusOK {
-		t.Errorf("Expected status 200 for OPTIONS, got %d", resp.StatusCode)
-	}
-}
-
 func TestSigV4Auth_ValidSignatureWithBody(t *testing.T) {
 	handler := setupTestApp(testAccessKey, testSecretKey)
 
@@ -1026,7 +1012,7 @@ func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
 	}
 }
 
-func TestWriteSigV4Error_PreservesRequestID(t *testing.T) {
+func TestWriteSigV4Error_IgnoresClientRequestID(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true, Region: testRegion}
 
 	w := httptest.NewRecorder()
@@ -1036,8 +1022,11 @@ func TestWriteSigV4Error_PreservesRequestID(t *testing.T) {
 	gw.writeSigV4Error(w, req, awserrors.ErrorIncompleteSignature)
 
 	body, _ := io.ReadAll(w.Result().Body)
-	if !strings.Contains(string(body), "<RequestID>test-request-id-123</RequestID>") {
-		t.Errorf("Expected preserved request ID, got: %s", string(body))
+	if strings.Contains(string(body), "test-request-id-123") {
+		t.Errorf("Expected server-generated request ID, but client ID was used: %s", string(body))
+	}
+	if !strings.Contains(string(body), "<RequestID>") {
+		t.Errorf("Expected RequestID in response, got: %s", string(body))
 	}
 }
 

spinifex/gateway/ec2/instance/RunInstances.go

@@ -51,6 +51,10 @@ func ValidateRunInstancesInput(input *ec2.RunInstancesInput) (err error) {
 		return errors.New(awserrors.ErrorMissingParameter)
 	}
 
+	if input.KeyName == nil || *input.KeyName == "" {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+
 	if !strings.HasPrefix(*input.ImageId, "ami-") {
 		return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 	}