Merge remote-tracking branch 'origin/feat/reboot-relaunch' into e2e/integration

RebootInstances powered the guest down under -action shutdown=pause and
then ran system_reset + cont in the same QEMU process. On some hosts the
reset left the guest silent in firmware with QMP still answering, while a
fresh QEMU on the same volumes always booted.

A reboot now shuts the guest down over ACPI (forcing it off after 60s, as
before), ends that QEMU, and starts a fresh one on the same mounted volumes,
taps, IMDS datapath and identity, the way nova's soft reboot and Proxmox's
qm reboot do. The instance stays running in Describe, no hooks fire, the
console log is appended rather than truncated, and the existing QMP client
is reconnected in place so the heartbeat carries on.

A relaunched guest that writes nothing to its console within 60s has its
run state, vCPUs, vCPU thread states and registers (sampled twice) logged,
and if it has printed before, its QEMU is replaced once more. A relaunch
that cannot start goes to crash recovery. Stop, terminate and hot-plug
wait for a relaunch in flight so volumes are never sealed under a process
being started. Guest-initiated reboots stay an in-process reset.

The daemon now logs the host kernel, QEMU version and the OVMF package
and checksum at start, and every reboot relaunch carries them.

.gitattributes

@@ -0,0 +1,3 @@
+# The committed bundle carries the AWS SDK's template strings verbatim,
+# including trailing spaces that are part of their values.
+spinifex/runtime/roles/spinifexui/frontend/dist/** -whitespace

.github/assets/diagrams/tf-nginx-alb.svg

@@ -19,8 +19,8 @@
   </defs>
   <rect width="900" height="480" fill="url(#bg)" rx="12"/>
 
-  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · VPC WITH PUBLIC AND PRIVATE SUBNETS</text>
-  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · multi-AZ</text>
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · PUBLIC SUBNETS · SPREAD ACROSS NODES</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · server_count = 3</text>
 
   <!-- Internet -->
   <rect x="350" y="54" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
@@ -41,58 +41,45 @@
   <line x1="230" y1="184" x2="230" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
   <line x1="670" y1="184" x2="670" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
 
-  <!-- public_a -->
   <g transform="translate(108, 202)">
-    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
     <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_a</text>
-    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.1.0/24</text>
     <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
-
-    <rect x="20" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="120" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
-
-    <rect x="20" y="70" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="120" y="85" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">NAT GW</text>
+    <rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+    <rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-1</text>
+    <text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
+    <rect x="22" y="110" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="122" y="126" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-3</text>
+    <text x="122" y="140" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
   </g>
 
-  <!-- public_b -->
   <g transform="translate(548, 202)">
-    <rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
     <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_b</text>
-    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.2.0/24</text>
     <line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
-
-    <rect x="20" y="56" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="120" y="71" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+    <rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
+    <rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-2</text>
+    <text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
   </g>
 
-  <!-- SNAT arrow from public_a NAT → private row -->
-  <line x1="230" y1="306" x2="230" y2="340" stroke="#F87171" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.65"/>
-  <polygon points="226,336 234,336 230,344" fill="#F87171" opacity="0.8"/>
-  <rect x="256" y="314" width="240" height="20" rx="10" fill="#0a0a0a" stroke="#F87171" stroke-width="0.6" stroke-opacity="0.55"/>
-  <text x="376" y="328" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">SNAT for private subnets (egress)</text>
-
-  <!-- private_a -->
-  <g transform="translate(108, 350)">
-    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
-    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_a</text>
-    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-a</text>
-    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
-
-    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_1</text>
-    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
-  </g>
-
-  <!-- private_b -->
-  <g transform="translate(548, 350)">
-    <rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
-    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_b</text>
-    <text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-b</text>
-    <line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
-
-    <rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
-    <text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_2</text>
-    <text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
-  </g>
+  <!-- spread placement group -->
+  <rect x="108" y="372" width="684" height="88" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="128" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">nginx-alb-spread</text>
+  <text x="772" y="394" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">placement group · strategy spread · one server per node</text>
+  <line x1="124" y1="402" x2="776" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
+  <rect x="128" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="230" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node1</text>
+  <text x="230" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
+  <rect x="348" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="450" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node2</text>
+  <text x="450" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
+  <rect x="568" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="670" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node3</text>
+  <text x="670" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
 </svg>

.github/assets/diagrams/vultr-deployment.svg

@@ -0,0 +1,110 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="580" viewBox="0 0 900 580">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="vpc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#58a6ff14"/>
+      <stop offset="100%" style="stop-color:#58a6ff05"/>
+    </linearGradient>
+    <linearGradient id="node-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="svc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#a855f71f"/>
+      <stop offset="100%" style="stop-color:#a855f708"/>
+    </linearGradient>
+    <marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0,0 L10,5 L0,10 z" fill="#71717a"/>
+    </marker>
+  </defs>
+  <rect width="900" height="580" fill="url(#bg)" rx="12"/>
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ON VULTR · ONE NODE OR THREE</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">--nodes 1 or 3 · vx1-g-16c-64g or bare metal</text>
+  <rect x="30" y="52" width="250" height="56" rx="8" fill="#e4e4e70d" stroke="#e4e4e7" stroke-width="1.1" stroke-opacity="0.6"/>
+  <text x="155.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#e4e4e7" text-anchor="middle" font-weight="700">Your machine</text>
+  <text x="155.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">deploy-vultr.sh · OpenTofu or Terraform</text>
+  <rect x="325" y="52" width="250" height="56" rx="8" fill="#3fb9500d" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
+  <text x="450.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+  <text x="450.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">customers reach your EC2 and load balancers</text>
+  <rect x="620" y="52" width="250" height="56" rx="8" fill="#58a6ff12" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
+  <text x="745.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#79c0ff" text-anchor="middle" font-weight="700">Vultr API</text>
+  <text x="745.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a5d6ff" text-anchor="middle">VPC · servers · reserved IPs</text>
+  <rect x="30" y="160" width="840" height="400" rx="10" fill="url(#vpc-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.55" stroke-dasharray="6,4"/>
+  <text x="50" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#79c0ff" font-weight="700">Vultr VPC</text>
+  <text x="850" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#58a6ff" text-anchor="end">private network 10.10.8.0/24</text>
+  <line x1="155" y1="108" x2="155" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
+  <text x="165" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">Terraform builds · SSH installs</text>
+  <line x1="450" y1="108" x2="450" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
+  <text x="460" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">reserved IPs</text>
+  <line x1="745" y1="158" x2="745" y2="110" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
+  <text x="735" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="end">Spinifex calls the API</text>
+  <rect x="60" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+  <text x="78" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node1</text>
+  <text x="292" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">Ubuntu 24.04 · Spinifex</text>
+  <line x1="74" y1="237" x2="296" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
+  <rect x="76" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="113.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
+  <rect x="160" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="190.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
+  <rect x="230" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="260.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
+  <text x="185" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
+  <rect x="325" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
+  <text x="343" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node2</text>
+  <text x="557" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
+  <line x1="339" y1="237" x2="561" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
+  <rect x="341" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="378.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
+  <rect x="425" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="455.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
+  <rect x="495" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="525.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
+  <text x="450" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
+  <rect x="590" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
+  <text x="608" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node3</text>
+  <text x="822" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
+  <line x1="604" y1="237" x2="826" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
+  <rect x="606" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="643.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
+  <rect x="690" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="720.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
+  <rect x="760" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="790.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
+  <text x="715" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
+  <line x1="310" y1="264" x2="325" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
+  <line x1="575" y1="264" x2="590" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
+  <text x="450" y="348" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#79c0ff" text-anchor="middle">nodes cluster over the private network: control plane, EBS replicas and S3 shards</text>
+  <rect x="60" y="370" width="780" height="166" rx="8" fill="url(#svc-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
+  <text x="80" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">AWS services across the fleet</text>
+  <text x="820" y="394" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">spread across the nodes; with three, moved to a survivor if one fails</text>
+  <line x1="76" y1="402" x2="824" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.6"/>
+  <rect x="76" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="119.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EC2</text>
+  <text x="119.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">instances</text>
+  <rect x="169" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="212.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ALB / NLB</text>
+  <text x="212.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">load balancers</text>
+  <rect x="262" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="305.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EKS</text>
+  <text x="305.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">Kubernetes</text>
+  <rect x="355" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="398.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ECS</text>
+  <text x="398.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">containers</text>
+  <rect x="448" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="491.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">RDS</text>
+  <text x="491.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">databases</text>
+  <rect x="541" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="584.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">S3</text>
+  <text x="584.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">object storage</text>
+  <rect x="634" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="677.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EBS</text>
+  <text x="677.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">volumes</text>
+  <rect x="727" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
+  <text x="770.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">IAM</text>
+  <text x="770.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">access</text>
+  <text x="450" y="500" font-family="'Segoe UI', system-ui, sans-serif" font-size="10.5" fill="#a1a1aa" text-anchor="middle">Use the AWS CLI, SDKs and Terraform against your own endpoint, unmodified.</text>
+  <text x="450" y="520" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Public addresses are Vultr reserved IPs, attached to the node each resource runs on.</text>
+</svg>

.github/e2e-cells.json

@@ -8,7 +8,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 2,
@@ -19,7 +20,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 3,
@@ -30,7 +32,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 8,
@@ -165,7 +168,7 @@
     "runner": "ci-multi",
     "extra": "",
     "suite": "instancerecovery",
-    "timeout_minutes": 105
+    "timeout_minutes": 125
   },
   {
     "cell": 32,
@@ -200,7 +203,8 @@
     "arch": "x86_64",
     "runner": "ci-single",
     "extra": "",
-    "suite": ""
+    "suite": "",
+    "timeout_minutes": 60
   },
   {
     "cell": 35,

.github/pull_request_template.md

@@ -16,6 +16,10 @@
 - Evidence added or rechecked:
 - ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
 
+## Mulga design
+
+<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
+
 ## Community Contribution License
 
 - [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.

.github/scripts/hypervisor-disk-preflight.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Usage: hypervisor-disk-preflight.sh <env.tfvars>
+#
+# Fails fast when a hypervisor named in the tofu env is short of disk, before
+# `tofu apply` tries to define a domain there. libvirt writes each domain's XML
+# under /etc/libvirt/qemu on the root filesystem and its disks into the pool, and
+# a full one fails the apply minutes in as "Domain Creation Failed".
+#
+# Hosts are reached the way the libvirt provider reaches them (ssh <user>@<name>,
+# resolved through ~/.ssh/config). A host that cannot be reached is reported and
+# skipped: tofu's own error for it is clearer than anything said here.
+#
+# Env: HV_SSH_USER (tf-user), HV_MIN_ROOT_FREE_MB (1024), HV_MIN_POOL_FREE_GB (20).
+set -uo pipefail
+
+tfvars="${1:?usage: $0 <env.tfvars>}"
+user="${HV_SSH_USER:-tf-user}"
+min_root_kb=$(( ${HV_MIN_ROOT_FREE_MB:-1024} * 1024 ))
+min_pool_kb=$(( ${HV_MIN_POOL_FREE_GB:-20} * 1024 * 1024 ))
+
+if [ ! -r "$tfvars" ]; then
+  echo "::error::hypervisor disk preflight: cannot read $tfvars" >&2
+  exit 1
+fi
+
+# One node per line in every env file: { name = "...", ..., storage_pool = "..." }.
+pairs=$(grep -v '^[[:space:]]*#' "$tfvars" |
+  sed -nE 's/.*name[[:space:]]*=[[:space:]]*"([^"]+)".*storage_pool[[:space:]]*=[[:space:]]*"([^"]+)".*/\1 \2/p')
+if [ -z "$pairs" ]; then
+  echo "::warning::hypervisor disk preflight: no nodes with a storage_pool in $tfvars; nothing checked"
+  exit 0
+fi
+
+# Prints "<root-avail-kb> <pool-avail-kb> <pool-path>" for the pool named in $1.
+# shellcheck disable=SC2016 # expanded on the hypervisor, not here
+remote='set -u
+root=$(df -Pk /etc/libvirt/qemu | awk "NR==2{print \$4}")
+path=$(virsh -c qemu:///system pool-dumpxml "$1" 2>/dev/null | sed -n "s:.*<path>\(.*\)</path>.*:\1:p" | head -n1)
+pool=-
+[ -n "$path" ] && pool=$(df -Pk "$path" | awk "NR==2{print \$4}")
+echo "$root $pool ${path:--}"'
+
+fail=0
+while read -r host pool; do
+  [ -n "$host" ] || continue
+  if ! out=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "${user}@${host}" bash -s -- "$pool" <<<"$remote" 2>&1); then
+    echo "::warning::hypervisor disk preflight: cannot reach $host ($out); its disk is unchecked"
+    continue
+  fi
+  read -r root_kb pool_kb pool_path <<<"$(tail -n1 <<<"$out")"
+  if ! [[ "$root_kb" =~ ^[0-9]+$ ]]; then
+    echo "::warning::hypervisor disk preflight: unexpected answer from $host: $out"
+    continue
+  fi
+  echo "$host: root fs $((root_kb / 1024)) MiB free; pool $pool ($pool_path) $([[ "$pool_kb" =~ ^[0-9]+$ ]] && echo "$((pool_kb / 1024 / 1024)) GiB free" || echo "not found")"
+  if [ "$root_kb" -lt "$min_root_kb" ]; then
+    echo "::error::hypervisor $host has $((root_kb / 1024)) MiB free on the filesystem holding /etc/libvirt/qemu (need ${HV_MIN_ROOT_FREE_MB:-1024} MiB): libvirt cannot write a domain definition there. Free space on $host before re-running." >&2
+    fail=1
+  fi
+  if [[ "$pool_kb" =~ ^[0-9]+$ ]] && [ "$pool_kb" -lt "$min_pool_kb" ]; then
+    echo "::error::hypervisor $host has $((pool_kb / 1024 / 1024)) GiB free in storage pool $pool at $pool_path (need ${HV_MIN_POOL_FREE_GB:-20} GiB): the node's disk cannot grow. Free space on $host before re-running." >&2
+    fail=1
+  fi
+done <<<"$pairs"
+
+exit "$fail"

.github/scripts/hypervisor-disk-preflight_test.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+# hypervisor-disk-preflight_test.sh — drives hypervisor-disk-preflight.sh against
+# a stub ssh that answers for each hypervisor from a canned file, so every
+# verdict is checked without a real host.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+PREFLIGHT="$SCRIPT_DIR/hypervisor-disk-preflight.sh"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# The stub answers "<root-kb> <pool-kb> <path>" from $FAKE/<host>, or exits 255
+# when that host has no file, the way ssh does for a host it cannot reach.
+mkdir -p "$TMP/bin" "$TMP/hosts"
+cat > "$TMP/bin/ssh" <<'STUB'
+#!/bin/sh
+for a in "$@"; do case $a in *@*) host=${a#*@} ;; esac; done
+cat >/dev/null
+if [ -f "$FAKE/$host" ]; then cat "$FAKE/$host"; exit 0; fi
+echo "ssh: Could not resolve hostname $host" >&2
+exit 255
+STUB
+chmod +x "$TMP/bin/ssh"
+PATH="$TMP/bin:$PATH"
+FAKE="$TMP/hosts"
+export PATH FAKE
+
+cat > "$TMP/env.tfvars" <<'TFVARS'
+nodes = [
+  { name = "wattle", wan_ip = "192.0.2.1", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
+  # { name = "retired", wan_ip = "192.0.2.9", storage_pool = "drive1-pool", base_image = "/x.qcow2" },
+  { name = "banksia", wan_ip = "192.0.2.2", storage_pool = "nvme1-pool", base_image = "/x.qcow2" },
+]
+TFVARS
+
+GiB=1048576
+plenty="$((50 * GiB)) $((200 * GiB)) /mnt/drive1/pool"
+
+# run <label> <want-rc> <want-text>
+run() {
+    rc=0
+    out=$(bash "$PREFLIGHT" "$TMP/env.tfvars" 2>&1) || rc=$?
+    if [ "$rc" -ne "$2" ]; then
+        fail "$1: exit $rc, want $2"
+        echo "$out" | sed 's/^/        /'
+        return
+    fi
+    case $out in
+        *"$3"*) pass "$1" ;;
+        *) fail "$1: output lacks '$3'"; echo "$out" | sed 's/^/        /' ;;
+    esac
+}
+
+echo "-- hypervisor-disk-preflight.sh"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$plenty" > "$FAKE/banksia"
+run "every host has room" 0 "banksia: root fs 51200 MiB free"
+
+echo "0 $((200 * GiB)) /mnt/drive1/pool" > "$FAKE/wattle"
+run "a full root filesystem fails fast" 1 "::error::hypervisor wattle has 0 MiB free on the filesystem holding /etc/libvirt/qemu"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$((50 * GiB)) $((3 * GiB)) /mnt/nvme1/pool" > "$FAKE/banksia"
+run "a nearly full pool fails fast" 1 "::error::hypervisor banksia has 3 GiB free in storage pool nvme1-pool"
+
+echo "$plenty" > "$FAKE/banksia"
+rm -f "$FAKE/wattle"
+run "an unreachable host is reported, not failed" 0 "::warning::hypervisor disk preflight: cannot reach wattle"
+
+echo "$plenty" > "$FAKE/wattle"
+echo "$((50 * GiB)) - -" > "$FAKE/banksia"
+run "a pool libvirt does not know is reported, not failed" 0 "pool nvme1-pool (-) not found"
+
+echo "0 0 /x" > "$FAKE/retired"
+echo "$plenty" > "$FAKE/banksia"
+run "a commented-out node is never checked" 0 "wattle: root fs"
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi
+echo "  hypervisor-disk-preflight ok"

.github/scripts/mulga_design_check.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
+
+import os
+import re
+
+COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
+SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
+LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
+PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
+
+
+def filled(body):
+    inside = False
+    for line in COMMENT_RE.sub("", body or "").splitlines():
+        if SECTION_RE.match(line):
+            inside = True
+        elif inside and line.lstrip().startswith("#"):
+            return False
+        elif inside:
+            m = LINE_RE.match(line)
+            if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
+                return True
+    return False
+
+
+def main():
+    if filled(os.environ.get("PR_BODY", "")):
+        print("The Mulga design line is filled in.")
+    else:
+        print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/suite-sets.sh

@@ -117,8 +117,9 @@ e2e_suite_timeout() {
     # storage case waits for a pause measured at 4m47s and then holds four
     # minutes to prove nothing moved. Three tests, each taking a node away and
     # bringing it back, and the partition case adds a lease validity, a lease
-    # TTL and a boot on top of everything the other two wait for.
-    instancerecovery) echo "90m" ;;
+    # TTL and a boot on top of everything the other two wait for. The data
+    # test lets its guest age past the pending watchdog twice, about 20 more.
+    instancerecovery) echo "110m" ;;
     # lbrecovery pays for a load balancer twice: once to stand one up with two
     # backends placed around it, and once for the recovery itself — a stale
     # owner, a lease, a boot, HAProxy's own start, and its health checks against
@@ -129,6 +130,10 @@ e2e_suite_timeout() {
     # budget on a node stopping and starting: seven units down, a raft election,
     # and the whole service stack back up with every stream caught up.
     kvquorum) echo "40m" ;;
+    # single is the broadest suite and finishes, it does not hang: passing runs
+    # took 24-29m, and an Oracle host ran out at 30m inside the last test's
+    # cleanup. The default left it no headroom at all.
+    single) echo "40m" ;;
     *) echo "30m" ;;
   esac
 }

.github/scripts/test_mulga_design_check.py

@@ -0,0 +1,23 @@
+import unittest
+
+from mulga_design_check import filled
+
+
+class FilledTest(unittest.TestCase):
+    def test_accepts_either_form(self):
+        self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
+        self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
+
+    def test_rejects_unanswered(self):
+        self.assertFalse(filled(""))
+        self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
+        self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
+        self.assertFalse(filled("## Mulga design\nUpdated:"))
+
+    def test_scoped_to_the_section(self):
+        self.assertFalse(filled("Updated: mulga#1"))
+        self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/actions.lock

@@ -18,6 +18,11 @@ workflows:
         - 'actions/create-github-app-token@v3.3.0'
         - 'actions/upload-artifact@v7.0.2'
         - 'hashicorp/setup-terraform@v4.0.1'
+    '.github/workflows/e2e-cloudvendor-vultr.yml':
+        - 'actions/checkout@v7.0.1'
+        - 'actions/create-github-app-token@v3.3.0'
+        - 'actions/upload-artifact@v7.0.2'
+        - 'hashicorp/setup-terraform@v4.0.1'
     '.github/workflows/e2e-ddil.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'
@@ -45,6 +50,8 @@ workflows:
     '.github/workflows/image-scripts.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-go@v7.0.0'
+    '.github/workflows/mulga-design.yml':
+        - 'actions/checkout@v7.0.1'
     '.github/workflows/perf-3node.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'
@@ -73,6 +80,7 @@ workflows:
     '.github/workflows/ui.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-node@v7.1.0'
+        - 'actions/upload-artifact@v7.0.2'
         - 'pnpm/action-setup@v6.1.0'
 dependencies:
     'actions/cache@v6.1.0':

.github/workflows/dev-release.yml

@@ -102,9 +102,9 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
         if: steps.version.outputs.exists != 'true'
-        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
+        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
 
       - name: Build amd64 Tarball
         if: steps.version.outputs.exists != 'true'

.github/workflows/e2e-cloudvendor-vultr.yml

@@ -0,0 +1,142 @@
+name: E2E Cloud Vendor Vultr
+
+# On demand only. Deploys a ref to Vultr with deploy-vultr.sh, runs the same
+# published Terraform workbooks as the hypervisor nightly and OCI, and destroys
+# the deployment, failing if anything is left billing.
+on:
+  workflow_dispatch:
+    inputs:
+      nodes:
+        description: "1 or 3 nodes"
+        required: false
+        default: "1"
+        type: choice
+        options: ["1", "3"]
+      plan:
+        description: "Vultr plan: vx1-g-16c-64g (VM) or vbm-8c-64gb-amd (bare metal)"
+        required: false
+        default: vx1-g-16c-64g
+      region:
+        description: "Vultr region"
+        required: false
+        default: ord
+      workbooks:
+        description: "Space-separated workbooks (blank = the driver's full list)"
+        required: false
+      source:
+        description: "tree (build this ref) or release (the published installer)"
+        required: false
+        default: tree
+        type: choice
+        options: [tree, release]
+      channel:
+        description: "With source=release: latest, or dev for the newest prerelease"
+        required: false
+        default: latest
+      keep_on_fail:
+        description: "Leave a failed deployment up to inspect. It keeps billing."
+        required: false
+        type: boolean
+        default: false
+      runner:
+        description: "Runner label (blank = the VULTR_E2E_RUNNER variable, else ubuntu-26.04)"
+        required: false
+
+permissions:
+  contents: read
+
+# Deployments are named per run, but the account's reserved-IP quota is shared,
+# so a second dispatch queues rather than competing with the first.
+concurrency:
+  group: e2e-vultr
+  cancel-in-progress: false
+
+jobs:
+  vultr:
+    # Hosted by default: the job only drives Terraform and SSH. The API key's
+    # access control must admit this runner, so a hosted runner needs a key
+    # whose list holds 0.0.0.0/0; a self-hosted runner can use a fixed address.
+    runs-on: ${{ inputs.runner || vars.VULTR_E2E_RUNNER || 'ubuntu-26.04' }}
+    timeout-minutes: 240
+    env:
+      VULTR_NAME: e2e-${{ github.run_id }}
+      VULTR_NODES: ${{ inputs.nodes }}
+      VULTR_PLAN: ${{ inputs.plan }}
+      VULTR_REGION: ${{ inputs.region }}
+      VULTR_SOURCE: ${{ inputs.source }}
+      VULTR_CHANNEL: ${{ inputs.channel }}
+      VULTR_ARTIFACT_DIR: ${{ github.workspace }}/e2e-vultr
+      SPX_REF: ${{ github.ref_name }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          path: spinifex
+          fetch-depth: 0
+
+      - uses: actions/create-github-app-token@v3.3.0
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      # build-install-artifacts.sh lives in mulga; the workbooks and their driver
+      # are in spinifex, so only the one directory is needed here.
+      - uses: actions/checkout@v7.0.1
+        with:
+          repository: mulgadc/mulga
+          path: mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+
+      - uses: hashicorp/setup-terraform@v4.0.1
+        with:
+          terraform_wrapper: false
+
+      - name: Run Vultr
+        working-directory: spinifex/scripts/terraform/vultr-spx
+        env:
+          VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
+          MULGA_ROOT: ${{ github.workspace }}/mulga
+          # Through the environment, never interpolated into the script: a
+          # dispatch input is untrusted text to the shell.
+          WORKBOOKS_INPUT: ${{ inputs.workbooks }}
+          VULTR_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
+        run: |
+          set -uo pipefail
+          if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
+          ./e2e-vultr.sh 2>&1 | tee "$RUNNER_TEMP/e2e-vultr.log"
+          exit "${PIPESTATUS[0]}"
+
+      # always(): a cancelled or timed-out job is the one that would leave
+      # servers and reserved IPs billing. destroy works from this run's state.
+      - name: Sweep anything left behind
+        if: always() && !inputs.keep_on_fail
+        working-directory: spinifex/scripts/terraform/vultr-spx
+        env:
+          VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
+        run: ./e2e-vultr.sh --sweep-only
+
+      - name: Publish the verdict
+        if: always()
+        run: |
+          {
+            echo "### Vultr: ${VULTR_NODES} x ${VULTR_PLAN} in ${VULTR_REGION}, ${VULTR_SOURCE} build of ${SPX_REF}"
+            echo '```'
+            cat "$VULTR_ARTIFACT_DIR/verdict.txt" 2>/dev/null || echo "no verdict written"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      # Logs and the verdict only. The throwaway SSH key and the Terraform state
+      # under ~/.spinifex never reach an artifact.
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: e2e-vultr-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/e2e-vultr.log
+            ${{ github.workspace }}/e2e-vultr/*.log
+            ${{ github.workspace }}/e2e-vultr/verdict.txt
+            ${{ github.workspace }}/e2e-vultr/node*/*.log
+          retention-days: 14

.github/workflows/e2e-nightly.yml

@@ -184,6 +184,14 @@ jobs:
             echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
           } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
 
+      # A hypervisor out of disk fails the apply minutes in, as "Domain Creation
+      # Failed" from libvirt writing the domain XML. Say so before starting.
+      - name: Preflight hypervisor disk
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hypervisor-disk-preflight.sh" \
+            "envs/${SPINIFEX_CI_ENV}.tfvars"
+
       - name: Provision VMs
         working-directory: mulga/scripts/tofu-cluster
         env:
@@ -241,6 +249,11 @@ jobs:
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
 
+          # The select job sized this cell's budget from these same timeouts, so
+          # resolve them here and carry the pairs into the remote loop.
+          SUITE_TMOS=""
+          for s in ${E2E_SUITES_NIGHTLY_SINGLE}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
+
           INVENTORY=$(tofu output -json inventory)
           WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
           SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -250,9 +263,10 @@ jobs:
              for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
                rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
@@ -324,16 +338,20 @@ jobs:
             NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
           done
 
+          SUITE_TMOS=""
+          for s in ${E2E_SUITES_NIGHTLY_MULTI}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
+
           ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
             "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
              for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
                echo \"::group::suite \$SUITE\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
                rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \

.github/workflows/mulga-design.yml

@@ -0,0 +1,33 @@
+name: Mulga design
+
+# Reminds the author to say whether the Mulga design records need to change.
+# It reads only the pull request body, uses no secrets, and warns without ever
+# failing: the check is a prompt, not a gate.
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [opened, edited, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  mulga_design:
+    name: Mulga design line
+    if: ${{ !endsWith(github.actor, '[bot]') }}
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Check the Mulga design line
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        working-directory: .github/scripts
+        run: |
+          python3 -m unittest -q test_mulga_design_check
+          python3 mulga_design_check.py

.github/workflows/release.yml

@@ -57,8 +57,8 @@ jobs:
           go-version-file: spinifex/go.mod
           cache: false
 
-      - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
-        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
+        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
 
       - name: Build amd64 Tarball
         env:

.github/workflows/ui.yml

@@ -7,6 +7,11 @@ on:
       - dev
     paths:
       - spinifex/runtime/roles/spinifexui/frontend/**
+      - spinifex/runtime/roles/spinifexui/*.go
+      - spinifex/operator/admin/instance_metrics*.go
+      - spinifex/operator/admin/live_streams*.go
+      - spinifex/runtime/roles/qmpcollector/**
+      - contracts/telemetry/**
       - .github/workflows/ui.yml
   pull_request:
     branches:
@@ -14,6 +19,11 @@ on:
       - dev
     paths:
       - spinifex/runtime/roles/spinifexui/frontend/**
+      - spinifex/runtime/roles/spinifexui/*.go
+      - spinifex/operator/admin/instance_metrics*.go
+      - spinifex/operator/admin/live_streams*.go
+      - spinifex/runtime/roles/qmpcollector/**
+      - contracts/telemetry/**
       - .github/workflows/ui.yml
   workflow_dispatch:
 
@@ -69,3 +79,85 @@ jobs:
             echo "$drift"
             exit 1
           fi
+
+  # The pinned image lives in one file that scripts/ui-playwright.sh also reads,
+  # so make ui-baselines and the job below draw screenshots in the same image.
+  ui_e2e_image:
+    name: UI End-to-End Image
+    runs-on: ubuntu-26.04
+    outputs:
+      image: ${{ steps.read.outputs.image }}
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: spinifex/runtime/roles/spinifexui/frontend/playwright.image
+          sparse-checkout-cone-mode: false
+
+      - name: Read Playwright Image
+        id: read
+        run: echo "image=$(tr -d '[:space:]' < spinifex/runtime/roles/spinifexui/frontend/playwright.image)" >> "$GITHUB_OUTPUT"
+
+  # The browser suite: the console against a faked API on Vite dev, plus the
+  # built dist under the production CSP. It runs in Playwright's own image, which
+  # carries the browsers, so screenshots rasterise as they did for the baselines.
+  ui_e2e:
+    name: UI End-to-End
+    needs: ui_e2e_image
+    runs-on: ubuntu-26.04
+    timeout-minutes: 20
+    container:
+      image: ${{ needs.ui_e2e_image.outputs.image }}
+      options: --ipc=host
+    defaults:
+      run:
+        working-directory: spinifex/runtime/roles/spinifexui/frontend
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6.1.0
+        with:
+          package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v7.1.0
+        with:
+          cache: "pnpm"
+          cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
+          node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
+
+      - name: Install Dependencies
+        run: pnpm install --frozen-lockfile
+
+      # The image's browsers belong to one Playwright release, so a lockfile
+      # bump without an image bump would otherwise fail on a missing executable.
+      - name: Check Playwright Matches the Image
+        env:
+          IMAGE: ${{ needs.ui_e2e_image.outputs.image }}
+        run: |
+          version=$(pnpm exec playwright --version | awk '{print $2}')
+          case "$IMAGE" in
+            *":v$version-"*) ;;
+            *) echo "::error::Lockfile has Playwright $version but the image is $IMAGE. Update playwright.image."; exit 1 ;;
+          esac
+
+      - name: Build Frontend
+        run: pnpm build
+
+      - name: Run End-to-End Suite
+        run: pnpm e2e
+
+      - name: Upload Screenshot Diffs
+        if: failure()
+        uses: actions/upload-artifact@v7.0.2
+        with:
+          name: ui-e2e-test-results
+          path: spinifex/runtime/roles/spinifexui/frontend/test-results/
+          if-no-files-found: ignore
+          retention-days: 14

.gitignore

@@ -24,6 +24,9 @@ build/microvm/qboot.LICENSE
 build/microvm/sqs-store/vmlinuz
 build/microvm/sqs-store/initramfs.cpio.gz
 build/microvm/sqs-store/.stage.*/
+build/microvm/sqs-worker/vmlinuz
+build/microvm/sqs-worker/initramfs.cpio.gz
+build/microvm/sqs-worker/.stage.*/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -56,6 +56,19 @@ build-ui:
 	@echo -e "\n....Building spinifex-ui frontend...."
 	cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
 
+# Playwright suite for the console, against Vite dev and the built dist under
+# the console's CSP. Not part of preflight; needs `pnpm exec playwright install chromium`.
+test-ui-e2e: build-ui
+	cd spinifex/runtime/roles/spinifexui/frontend && pnpm e2e
+
+# The same suite inside the pinned Playwright image CI uses (podman or docker),
+# so screenshots compare as they will on the runner. ui-baselines redraws them.
+ui-e2e-container:
+	scripts/ui-playwright.sh compare
+
+ui-baselines:
+	scripts/ui-playwright.sh baselines
+
 # GO commands
 VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
 COMMIT  ?= $(shell git rev-parse --short HEAD)
@@ -157,6 +170,21 @@ bin/nats-server: go.mod go.sum
 
 build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
 
+# The shared SQS worker microVM image: the store image's /init and kernel with
+# the worker binary instead of nats-server. Not installed or released yet.
+SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
+SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
+SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop) bin/sqs-worker
+
+$(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
+	SQS_MICROVM_PROFILE=worker ./scripts/build-sqs-store-microvm-image.sh
+
+bin/sqs-worker: go.mod go.sum $(shell find cmd/sqs-worker spinifex/agents/sqs -name '*.go' ! -name '*_test.go')
+	@mkdir -p bin
+	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
+
+build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the shared SQS worker microVM kernel and initramfs
+
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
 QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
@@ -174,6 +202,12 @@ install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS sto
 	sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
 	sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
 
+install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS worker microVM image to /usr/share/spinifex/microvm/sqs-worker/
+	sudo install -d /usr/share/spinifex/microvm/sqs-worker
+	sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
+	sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
+	sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
+
 install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
 	sudo install -d /usr/share/spinifex/microvm
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
@@ -213,6 +247,12 @@ test-build-scripts:
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
+	@echo -e "\n....Running .github/scripts/*_test.sh...."
+	@for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh
 	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@@ -475,6 +515,7 @@ distro-amd64:
 		echo "[distro-amd64]          run 'make build-microvm-image' before 'make distro-amd64'"; \
 	fi
 	./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
+	./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
 	tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
 
@@ -493,7 +534,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image install-microvm install-sqs-store-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-store-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

architecture/layering/allowlist_test.go

@@ -34,6 +34,11 @@ var allowlist = []allowance{
 	{From: "cmd/aws-model-coverage", To: "domains/route53/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 	{From: "cmd/aws-model-coverage", To: "domains/elbv2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 	{From: "cmd/aws-model-coverage", To: "domains/rds/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/ec2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/ecs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/eks/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/iam/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/sts/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 
 	{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
 
@@ -55,9 +60,6 @@ var allowlist = []allowance{
 	{From: "domains/ec2/awsapi/instance", To: "runtime/compute/cache", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
 	{From: "domains/ec2/awsapi/instance", To: "runtime/compute/vm", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
 
-	{From: "domains/ec2/awsapi/spotinstance", To: "domains/admission/quota", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
-	{From: "domains/ec2/awsapi/spotinstance", To: "domains/iam", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
-
 	{From: "domains/ec2/eip", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
 
 	{From: "domains/ec2/guestmetadata", To: "domains/dns", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
@@ -94,40 +96,14 @@ var allowlist = []allowance{
 	{From: "domains/ochre", To: "domains/ec2/systeminstance", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
 	{From: "domains/ochre", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
 	{From: "domains/ochre", To: "domains/network/systemvpc", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
-	{From: "domains/ochre", To: "ingress/aws/bodyscope", Debt: "domains/ochre (action adapters)", Reason: "Bedrock action adapters still live in the Ochre package; they move to domains/ochre/awsapi"},
 	{From: "domains/ochre", To: "runtime/compute/gpu", Debt: "Domain → runtime", Reason: "GPU runtime state read directly; needs a domain-owned projection"},
 
 	{From: "domains/ochre/vector", To: "domains/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
 
-	{From: "gateway", To: "domains/ec2/awsapi", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/account", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/capacityreservation", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/eigw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/eip", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/idem", Debt: "gateway/ec2.go (registration; ec2_idempotency.go)", Reason: "dispatch-time idempotency wrapper; moves with the EC2 dispatch map"},
-	{From: "gateway", To: "domains/ec2/awsapi/igw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/image", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/instance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/key", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/launchtemplate", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/natgw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/placementgroup", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/routetable", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/snapshot", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/spotinstance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/tags", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/volume", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/vpc", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ec2/awsapi/zone", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
-
 	{From: "domains/rds/awsapi", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
 
 	{From: "domains/sts/awsapi", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
 
-	{From: "gateway", To: "domains/eks/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/iam/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/sts/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
-	{From: "gateway", To: "domains/ecs/awsapi", Debt: "gateway (service registration)", Reason: "ECS and ELBv2 dispatch stays in gateway until the ingress registration seam"},
 	{From: "domains/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "domains/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
 	{From: "domains/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
@@ -160,6 +136,4 @@ var allowlist = []allowance{
 	{From: "domains/rds/instance", To: "runtime/compute/vm", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
 
 	{From: "domains/sts", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "STS reads IAM service, trust-policy and OIDC-provider state; needs an IAM capability"},
-
-	{From: "operator/cli", To: "gateway", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
 }

architecture/layering/layering_test.go

@@ -213,3 +213,28 @@ func TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon(t *testing.T) {
 		t.Error("gateway -> runtime/host/process is rejected, but it is not the daemon")
 	}
 }
+
+// TestADR0002_S7_RequestRecordRejectsOutsiders: the request record is readable
+// by the gateway and ingress/aws only.
+func TestADR0002_S7_RequestRecordRejectsOutsiders(t *testing.T) {
+	record, ok := classify("spinifex/ingress/aws/reqctx")
+	if !ok || !record.requestRecord {
+		t.Fatalf("ingress/aws/reqctx is not classified as the request record: %+v", record)
+	}
+	r := ruleByID("S7_RequestRecordPrivateToIngress")
+	for name, wantRejected := range map[string]bool{
+		"domains/ecs/awsapi":   true,
+		"runtime/roles/awsgw":  true,
+		"operator/admin":       true,
+		"gateway":              false,
+		"ingress/aws/dispatch": false,
+	} {
+		from, ok := classify(fullPath(name))
+		if !ok {
+			t.Fatalf("%s is unclassified", name)
+		}
+		if got := r.forbids(from, record, name); got != wantRejected {
+			t.Errorf("%s -> ingress/aws/reqctx rejected=%v, want %v", name, got, wantRejected)
+		}
+	}
+}

architecture/layering/layers_test.go

@@ -43,6 +43,8 @@ type class struct {
 	domain string // owning domain for domain-shaped and agent packages
 	kind   domainKind
 	daemon bool // runtime/daemon, the interim daemon -> gateway -> handlers root
+	// requestRecord marks ingress/aws/reqctx, the request record ADR-0002:S7 keeps private to ingress.
+	requestRecord bool
 }
 
 func (c class) domainShaped() bool { return c.kind != notDomain }
@@ -103,7 +105,7 @@ func classifyApp(app []string) (class, bool) {
 	case "foundation":
 		return class{layer: layerFoundation}, true
 	case "ingress":
-		return class{layer: layerIngress}, true
+		return class{layer: layerIngress, requestRecord: len(app) >= 3 && app[1] == "aws" && app[2] == "reqctx"}, true
 	case "runtime":
 		return class{layer: layerRuntime, daemon: len(app) >= 2 && app[1] == "daemon"}, true
 	case "operator":

architecture/layering/rules_test.go

@@ -1,6 +1,9 @@
 package layering
 
-import "slices"
+import (
+	"slices"
+	"strings"
+)
 
 // rule is one ADR-0001 dependency rule over a classified import edge.
 type rule struct {
@@ -151,6 +154,18 @@ var rules = []rule{
 				(importsDomainImplementation(t) || t.layer == layerAgents)
 		},
 	},
+	{
+		id:      "S7_RequestRecordPrivateToIngress",
+		clause:  "ADR-0002:S7: the request record is private to ingress, so no domain, runtime or operator package reads it; only the gateway and packages under ingress/aws may import ingress/aws/reqctx",
+		hint:    "receive what you need through dispatch.Invocation instead of the request record",
+		subject: func(_ class, _ string) bool { return true },
+		forbids: func(_, t class, fromName string) bool {
+			if !t.requestRecord {
+				return false
+			}
+			return fromName != "gateway" && !strings.HasPrefix(fromName, "ingress/aws/")
+		},
+	},
 	{
 		id:      "Interim_DaemonGatewayHandlersDirection",
 		clause:  "during migration the existing daemon -> gateway -> handlers rule remains the guardrail (Q-20, Q-29)",

build/microvm/sqs-store/sqs-store-lib.sh

@@ -101,6 +101,24 @@ sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
 # sqs_same <pid> <starttime>: the recorded process is still alive.
 sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
 
+# sqs_write_config <addr> <nkey> <file>: the dedicated server's config. Its
+# max_payload admits a 1 MiB SQS message with its headers (ADR-0011:S4); the
+# 1MB NATS default does not.
+sqs_write_config() {
+    cat > "$3" <<EOF
+listen: "$1:$SQS_PORT"
+server_name: "sqs-store"
+max_payload: 2MiB
+jetstream {
+  store_dir: "$SQS_DATA"
+  sync_interval: always
+}
+authorization {
+  users = [ { nkey: "$2" } ]
+}
+EOF
+}
+
 # sqs_prepare: validate the contract, check and mount the data disk, write the
 # server config and mark the store ready. Returns non-zero, unready, on any refusal.
 sqs_prepare() {
@@ -117,17 +135,7 @@ sqs_prepare() {
     # The server checks the config, including the NKey checksum, before the
     # disk is touched.
     umask 077
-    cat > "$SQS_RUN/nats.conf" <<EOF
-listen: "$addr:$SQS_PORT"
-server_name: "sqs-store"
-jetstream {
-  store_dir: "$SQS_DATA"
-  sync_interval: always
-}
-authorization {
-  users = [ { nkey: "$nkey" } ]
-}
-EOF
+    sqs_write_config "$addr" "$nkey" "$SQS_RUN/nats.conf" || return 1
     if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
         sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
         return 1
@@ -161,7 +169,8 @@ sqs_run() {
         return 2
     fi
     sqs_log "run: starting pid=$$"
-    exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
+    # ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
+    GOMEMLIMIT=512MiB exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
 }
 
 # sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run

build/microvm/sqs-worker/inittab

@@ -0,0 +1,8 @@
+# The power button loads here, not in /init: busybox init has its SIGUSR2
+# (poweroff) handler installed before sysinit runs, so a press can never reach
+# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
+::sysinit:/sbin/modprobe tiny_power_button power_signal=12
+::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
+::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
+::ctrlaltdel:/sbin/reboot
+::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop

build/microvm/sqs-worker/sqs-worker-lib.sh

@@ -0,0 +1,115 @@
+# sqs-worker-lib.sh — shared by the SQS worker guest's readiness and stop
+# steps. Sourced, never executed. Every path and command is overridable so the
+# host test can drive the same code against test-owned processes and stub tools.
+#
+# store-ready exists only while the running watcher's last check passed. A
+# check proves the store accepted the worker key, not the store server's
+# identity, and a store failure is noticed up to one interval plus call time late.
+
+SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
+SQSW_DATA=${SQSW_DATA:-/data}
+SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
+SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
+SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
+SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
+SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
+SQSW_PARK_INTERVAL=${SQSW_PARK_INTERVAL:-30}
+UMOUNT=${UMOUNT:-umount}
+
+sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
+
+# sqsw_alive <pid>: running or stopped, not exited (a zombie counts as exited).
+sqsw_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
+
+# sqsw_starttime <pid>: kernel start time (stat field 22). It survives exec, and
+# pairing it with the pid reduces PID-reuse risk; it is not an absolute guarantee.
+sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
+
+# sqsw_same <pid> <starttime>: the recorded process is still alive.
+sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
+
+# sqsw_run: the respawned readiness body. A predecessor that was killed or
+# crashed could not remove its marker, so every start removes it first. Its
+# identity is recorded before the stopping check, as the store's run step does.
+sqsw_run() {
+    mkdir -p "$SQSW_RUN"
+    echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/ready.pid"
+    if [ -e "$SQSW_RUN/store-ready" ]; then
+        rm -f "$SQSW_RUN/store-ready"
+        sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
+    fi
+    # Returning here would make init respawn and log at once, forever. Park
+    # instead, logging once: shutdown stops this recorded process, and a key
+    # that appears later lets the watcher start.
+    if [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; then
+        if [ -e "$SQSW_RUN/stopping" ]; then
+            sqsw_log "ready: shutdown in progress, not starting"
+        else
+            sqsw_log "ready: no worker key; readiness not started, waiting"
+        fi
+        while [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; do
+            sleep "$SQSW_PARK_INTERVAL"
+        done
+    fi
+    # Init respawns at once; a watcher that keeps exiting restarts at most
+    # once per gap instead of in a tight loop.
+    now=$(date +%s)
+    last=$(cat "$SQSW_RUN/ready.last" 2>/dev/null || echo 0)
+    wait=$((last + SQSW_RESPAWN_GAP - now))
+    if [ "$wait" -gt 0 ] && [ "$wait" -le "$SQSW_RESPAWN_GAP" ]; then sleep "$wait"; fi
+    date +%s > "$SQSW_RUN/ready.last"
+    sqsw_log "ready: starting watcher pid=$$"
+    exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
+}
+
+# sqsw_stop: block any respawn, drop the marker, SIGTERM the recorded watcher,
+# SIGKILL only after the budget, and unmount the worker disk only once the
+# watcher is confirmed gone. A survivor keeps the disk mounted.
+sqsw_stop() {
+    mkdir -p "$SQSW_RUN"
+    : > "$SQSW_RUN/stopping"
+    rm -f "$SQSW_RUN/store-ready"
+    rec=$(cat "$SQSW_RUN/ready.pid" 2>/dev/null || true)
+    pid=${rec%% *} st=${rec#* }
+    [ "$st" != "$rec" ] || st=""
+    if [ -n "$pid" ] && sqsw_same "$pid" "$st"; then
+        kill -TERM "$pid" 2>/dev/null || true
+        sqsw_log "stop: SIGTERM sent to readiness watcher pid=$pid"
+        i=0
+        while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
+        if sqsw_same "$pid" "$st"; then
+            sqsw_log "stop: readiness watcher pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
+            kill -KILL "$pid" 2>/dev/null || true
+            j=0
+            while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
+        fi
+        if sqsw_same "$pid" "$st"; then
+            rm -f "$SQSW_RUN/store-ready"
+            sqsw_log "stop: readiness watcher pid=$pid STILL ALIVE, leaving $SQSW_DATA mounted"
+            return 1
+        fi
+        sqsw_log "stop: readiness watcher pid=$pid exited"
+    else
+        sqsw_log "stop: no running readiness watcher"
+    fi
+    rm -f "$SQSW_RUN/store-ready" "$SQSW_RUN/ready.pid"
+    sync
+    if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
+        rc=0
+        "$UMOUNT" "$SQSW_DATA" || rc=$?
+        sqsw_log "stop: umount $SQSW_DATA rc=$rc"
+    fi
+}
+
+# sqsw_shutdown: the only shutdown action. The generic read-only remount runs
+# only after the watcher stopped; a survivor keeps every mount as it is.
+sqsw_shutdown() {
+    if ! sqsw_stop; then
+        sync
+        sqsw_log "shutdown: readiness watcher survived, skipping umount -a -r"
+        return 1
+    fi
+    rc=0
+    "$UMOUNT" -a -r || rc=$?
+    echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQSW_CONSOLE"
+}

build/microvm/sqs-worker/sqs-worker-prepare

@@ -0,0 +1,8 @@
+#!/bin/sh
+# sysinit: load the disk drivers, then select, check and mount the worker disk
+# and load or create the worker's store key on it. Any refusal is logged; the
+# key is never read or created on a disk that is not proven mounted.
+modprobe virtio_blk 2>/dev/null || true
+modprobe ext4 2>/dev/null || true
+/usr/local/bin/sqs-worker boot >> /dev/console 2>&1 || echo "[sqs-worker] boot refused" >> /dev/console
+exit 0

build/microvm/sqs-worker/sqs-worker-ready

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Respawned by init. A watcher that must not run parks here instead of exiting,
+# so init does not respawn it in a tight loop.
+. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
+sqsw_run
+exec sleep 2147483647

build/microvm/sqs-worker/sqs-worker-stop

@@ -0,0 +1,5 @@
+#!/bin/sh
+# The only shutdown action: stop the readiness watcher and unmount the worker
+# disk, then run the generic read-only remount only if the watcher is gone.
+. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
+sqsw_shutdown

build/systemd/spinifex-firewall.service

@@ -1,11 +1,10 @@
-# spinifex-unit-version: 1
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex host firewall (nftables input policy)
 Documentation=file:/etc/spinifex/firewall/spinifex.nft
 Wants=network-pre.target
 Before=network-pre.target shutdown.target
 DefaultDependencies=no
-ConditionPathExists=/etc/spinifex/firewall/peers.nft
 
 [Service]
 Type=oneshot
@@ -13,8 +12,11 @@ RemainAfterExit=yes
 StandardInput=null
 ProtectSystem=full
 ProtectHome=true
-ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
-ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+# Unconditional: connection tracking loads on every node, so a flow that opens
+# while the policy is not armed survives the arm. The policy itself still loads
+# only where the daemon has written a peer file.
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply boot
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply boot
 TimeoutStopSec=10s
 
 # No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD

cmd/aws-model-coverage/main.go

@@ -9,10 +9,15 @@ import (
 	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
+	ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
+	ecsawsapi "github.com/mulgadc/spinifex/spinifex/domains/ecs/awsapi"
+	eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
 	elbv2awsapi "github.com/mulgadc/spinifex/spinifex/domains/elbv2/awsapi"
+	iamawsapi "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
 	rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
 	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
+	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 
@@ -58,6 +63,11 @@ func compareAll() ([]awsmodel.OperationCoverage, error) {
 		route53awsapi.ServiceName: route53awsapi.OperationInventory(),
 		elbv2awsapi.InventoryName: elbv2awsapi.OperationInventory(),
 		rdsawsapi.ServiceName:     rdsawsapi.OperationInventory(),
+		ec2awsapi.ServiceName:     ec2awsapi.OperationInventory(),
+		ecsawsapi.ServiceName:     ecsawsapi.OperationInventory(),
+		eksawsapi.ServiceName:     eksawsapi.OperationInventory(),
+		iamawsapi.ServiceName:     iamawsapi.OperationInventory(),
+		stsawsapi.ServiceName:     stsawsapi.OperationInventory(),
 	})
 	// The loop below walks the embedded models, so a served service with no
 	// model would otherwise drop out of the report without a word.

cmd/eks-etcd-member/main.go

@@ -0,0 +1,50 @@
+// eks-etcd-member removes a dead peer from the embedded etcd cluster of an EKS
+// K3s control-plane VM. A member that is already gone exits 0; any other
+// failure exits non-zero.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/etcdmember"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
+}
+
+func run(args []string, stdout, stderr io.Writer) int {
+	if len(args) < 1 || args[0] != "remove" {
+		fmt.Fprintln(stderr, "usage: eks-etcd-member remove --peer-ip 10.0.0.12 [--self-ip 10.0.0.10]")
+		return 2
+	}
+	fs := flag.NewFlagSet("remove", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var cfg etcdmember.Config
+	endpoint := fs.String("endpoint", etcdmember.DefaultEndpoint, "etcd client endpoint")
+	certDir := fs.String("cert-dir", etcdmember.DefaultCertDir, "directory holding the K3s etcd client.crt, client.key and server-ca.crt")
+	timeout := fs.Duration("timeout", 30*time.Second, "overall deadline")
+	fs.StringVar(&cfg.PeerIP, "peer-ip", "", "peer URL host of the member to remove")
+	fs.StringVar(&cfg.SelfIP, "self-ip", "", "this host's own address; removal of it is refused")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
+	defer cancel()
+
+	cli, err := etcdmember.Dial(*endpoint, *certDir, *timeout)
+	if err != nil {
+		fmt.Fprintln(stderr, "etcd member remove failed:", err)
+		return 1
+	}
+	defer func() { _ = cli.Close() }()
+	return etcdmember.Run(ctx, cli, cfg, stdout)
+}

cmd/eks-etcd-member/main_test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit/fakeetcd"
+	"github.com/stretchr/testify/assert"
+)
+
+var fakeMembers = []fakeetcd.Member{
+	{ID: 0xa1, Name: "cp-a", PeerURLs: []string{"https://10.0.0.10:2380"}},
+	{ID: 0xb2, Name: "cp-b", PeerURLs: []string{"https://10.0.0.12:2380"}},
+}
+
+func TestRunRequiresTheRemoveSubcommand(t *testing.T) {
+	t.Parallel()
+	for _, args := range [][]string{nil, {"list"}} {
+		var out, errOut bytes.Buffer
+		assert.Equal(t, 2, run(args, &out, &errOut))
+		assert.Contains(t, errOut.String(), "usage: eks-etcd-member remove")
+	}
+}
+
+func TestRunRejectsUnknownFlags(t *testing.T) {
+	t.Parallel()
+	var out, errOut bytes.Buffer
+	assert.Equal(t, 2, run([]string{"remove", "--bogus"}, &out, &errOut))
+}
+
+func TestRunFailsWhenEtcdCertificatesAreMissing(t *testing.T) {
+	t.Parallel()
+	var out, errOut bytes.Buffer
+	code := run([]string{"remove", "--peer-ip", "10.0.0.12", "--cert-dir", t.TempDir()}, &out, &errOut)
+	assert.Equal(t, 1, code)
+	assert.Contains(t, errOut.String(), "etcd member remove failed")
+}
+
+func TestRunRemovesTheNamedPeer(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, fakeMembers)
+	var out, errOut bytes.Buffer
+
+	code := run([]string{"remove", "--peer-ip", "10.0.0.12", "--self-ip", "10.0.0.10",
+		"--endpoint", srv.Endpoint, "--cert-dir", srv.CertDir}, &out, &errOut)
+
+	assert.Zero(t, code, errOut.String()+out.String())
+	assert.Contains(t, out.String(), "removed etcd member 10.0.0.12")
+	assert.Equal(t, []uint64{0xb2}, srv.Removed())
+}
+
+func TestRunTreatsAnAbsentPeerAsSuccess(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, fakeMembers)
+	var out, errOut bytes.Buffer
+
+	code := run([]string{"remove", "--peer-ip", "10.0.0.99",
+		"--endpoint", srv.Endpoint, "--cert-dir", srv.CertDir}, &out, &errOut)
+
+	assert.Zero(t, code)
+	assert.Contains(t, out.String(), "member already absent")
+	assert.Empty(t, srv.Removed())
+}

cmd/sqs-worker/main.go

@@ -0,0 +1,120 @@
+// sqs-worker runs inside the shared SQS worker microVM.
+//
+// Usage:
+//
+//	sqs-worker boot
+//	sqs-worker ready
+//
+// boot selects and mounts the worker's data disk named by fw_cfg, asks the
+// gateway for its current first-store decision with its instance-role
+// credentials, then loads or creates the store key on that disk and
+// publishes only the public key. ready then keeps checking the bound store's
+// JetStream with that key until it is stopped, keeping a readiness marker only
+// while the check passes; it proves the store accepted the worker key, not
+// the store server's identity.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/sqsworkergw"
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerboot"
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+)
+
+// The cluster CA the daemon supplies; the only root the client trusts.
+const caPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw"
+
+// newClaimer signs with the IMDS instance-role credentials only; there is
+// no static-key fallback.
+func newClaimer() workerboot.NewClaimer {
+	return func(region, gatewayURL string) (workerkey.Claimer, error) {
+		return newClient(region, gatewayURL)
+	}
+}
+
+func newClient(region, gatewayURL string) (*sqsworkergw.Client, error) {
+	ca, err := os.ReadFile(caPath)
+	if err != nil {
+		return nil, fmt.Errorf("read cluster CA: %w", err)
+	}
+	return sqsworkergw.New(gatewayURL, region, ca, gwsign.NewInstanceRole())
+}
+
+type describer struct{ c *sqsworkergw.Client }
+
+func (d describer) DescribeStore(ctx context.Context) (workerboot.Store, error) {
+	st, err := d.c.DescribeStore(ctx)
+	return workerboot.Store(st), err
+}
+
+func main() {
+	if len(os.Args) != 2 || (os.Args[1] != "boot" && os.Args[1] != "ready") {
+		fmt.Fprintln(os.Stderr, "usage: sqs-worker boot|ready")
+		os.Exit(2)
+	}
+	logf := func(msg string) { fmt.Fprintf(os.Stderr, "[sqs-worker] %s\n", msg) }
+	run := boot
+	if os.Args[1] == "ready" {
+		run = ready
+	}
+	if err := run(logf); err != nil {
+		logf("FATAL: " + err.Error())
+		os.Exit(1)
+	}
+}
+
+func config(logf func(string)) workerboot.Config {
+	return workerboot.Config{
+		FwCfgDir:      "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-worker",
+		SysBlock:      "/sys/block",
+		DevDir:        "/dev",
+		MountInfo:     "/proc/self/mountinfo",
+		DataDir:       "/data",
+		RunDir:        "/run/sqs-worker",
+		Disk:          workerboot.SystemDisk{E2fsck: "/sbin/e2fsck"},
+		IsBlockDevice: workerboot.IsBlockDevice,
+		NewClaimer:    newClaimer(),
+		Attempts:      60,
+		Wait:          5 * time.Second,
+		Log:           logf,
+	}
+}
+
+func boot(logf func(string)) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	_, err := workerboot.Boot(ctx, config(logf))
+	return err
+}
+
+// ready runs until init stops it; the marker is removed on the way out.
+func ready(logf func(string)) error {
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
+	defer cancel()
+	err := workerboot.Watch(ctx, workerboot.ReadyConfig{
+		Config: config(logf),
+		NewDescriber: func(region, gatewayURL string) (workerboot.StoreDescriber, error) {
+			c, err := newClient(region, gatewayURL)
+			if err != nil {
+				return nil, err
+			}
+			return describer{c}, nil
+		},
+		Checker:     workerboot.NATSChecker{},
+		CallTimeout: 30 * time.Second,
+		Refresh:     30 * time.Second,
+		Refused:     5 * time.Minute,
+	})
+	if errors.Is(err, context.Canceled) {
+		return nil
+	}
+	return err
+}

contracts/cluster/v1/heartbeat.go

@@ -0,0 +1,29 @@
+package clusterv1
+
+import "time"
+
+// ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
+// other cluster-wide state.
+const ClusterStateBucket = "spinifex-cluster-state"
+
+// HeartbeatKey is the ClusterStateBucket key a node writes its heartbeat under.
+func HeartbeatKey(node string) string {
+	return "heartbeat." + node
+}
+
+// HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
+// decodes only the timestamp, so a reader does not depend on the capacity
+// figures the daemon writes beside it.
+type HeartbeatTimestamp struct {
+	Timestamp string `json:"timestamp"`
+}
+
+// Time returns when the heartbeat was written. A timestamp that is missing or
+// unparseable vouches for nothing, so ok is false.
+func (h HeartbeatTimestamp) Time() (written time.Time, ok bool) {
+	parsed, err := time.Parse(time.RFC3339, h.Timestamp)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return parsed.UTC(), true
+}

contracts/cluster/v1/heartbeat_test.go

@@ -0,0 +1,35 @@
+package clusterv1_test
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The bucket and key are the deployed layout: a rolling upgrade has old daemons
+// writing and new readers reading, so these strings cannot change.
+func TestHeartbeatKey_IsTheDeployedLayout(t *testing.T) {
+	assert.Equal(t, "spinifex-cluster-state", clusterv1.ClusterStateBucket)
+	assert.Equal(t, "heartbeat.node-a", clusterv1.HeartbeatKey("node-a"))
+}
+
+func TestHeartbeatTimestamp_ReadsAFullDaemonHeartbeat(t *testing.T) {
+	full := `{"node":"node-a","epoch":3,"timestamp":"2026-10-30T01:02:03Z","vm_count":4,"allocated_vcpu":8}`
+	var h clusterv1.HeartbeatTimestamp
+	require.NoError(t, json.Unmarshal([]byte(full), &h))
+
+	written, ok := h.Time()
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+}
+
+func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
+	for _, ts := range []string{"", "yesterday"} {
+		_, ok := clusterv1.HeartbeatTimestamp{Timestamp: ts}.Time()
+		assert.False(t, ok, "timestamp %q", ts)
+	}
+}

contracts/ec2/v1/volume_held.go

@@ -0,0 +1,7 @@
+package ec2v1
+
+// VolumeHeldElsewhereText is the text of a start refused because another node
+// holds one of the instance's volumes. It crosses NATS as message text, so
+// readers match on it; the provider that refuses and every reader share this
+// string so they cannot drift.
+const VolumeHeldElsewhereText = "volume is leased by another owner"

contracts/telemetry/v1/live.go

@@ -0,0 +1,71 @@
+package telemetryv1
+
+import (
+	"regexp"
+)
+
+// LiveSamplePeriodMillis is how often a watched instance is sampled for the
+// console's live view.
+const LiveSamplePeriodMillis = 500
+
+// LiveWatchLeaseMillis is how long one watch keeps the live sampler running.
+// Watchers renew well inside it, so a viewer that goes away costs at most this.
+const LiveWatchLeaseMillis = 30_000
+
+// LiveRingSamples bounds the in-memory history a live sampler keeps for a
+// reconnecting viewer: five minutes at LiveSamplePeriodMillis.
+const LiveRingSamples = 5 * 60 * 1000 / LiveSamplePeriodMillis
+
+// Live subjects sit outside metrics.> so nothing that ingests the periodic
+// CloudWatch batches ever reads a sub-second rate as a period. The account is part
+// of the subject, so only the poller for that account's instance answers.
+const (
+	liveWatchSubjectPrefix  = "telemetry.watch.ec2."
+	liveSampleSubjectPrefix = "telemetry.live.ec2."
+)
+
+var (
+	liveAccountPattern  = regexp.MustCompile(`^[0-9]{12}$`)
+	liveInstancePattern = regexp.MustCompile(`^i-[0-9a-f]{8,17}$`)
+)
+
+// ValidLiveTarget reports whether accountID and instanceID are safe to place
+// in a live subject: no NATS token separators or wildcards can get through.
+func ValidLiveTarget(accountID, instanceID string) bool {
+	return liveAccountPattern.MatchString(accountID) && liveInstancePattern.MatchString(instanceID)
+}
+
+// LiveWatchSubject is where a viewer asks the owning node to keep sampling. A
+// request on it is answered with the ring since LiveWatch.SinceMillis.
+func LiveWatchSubject(accountID, instanceID string) string {
+	return liveWatchSubjectPrefix + accountID + "." + instanceID
+}
+
+// LiveSampleSubject carries one LiveSample per sampling period while watched.
+func LiveSampleSubject(accountID, instanceID string) string {
+	return liveSampleSubjectPrefix + accountID + "." + instanceID
+}
+
+// LiveWatch is the body of a watch. SinceMillis is the newest sample the
+// viewer already has; zero asks for the whole ring.
+type LiveWatch struct {
+	SinceMillis int64 `json:"since_ms,omitempty"`
+}
+
+// LiveRing answers a watch request with the samples newer than SinceMillis.
+type LiveRing struct {
+	Samples []LiveSample `json:"samples"`
+}
+
+// LiveSample is one live measurement. Every value is a rate over the interval
+// ending at TimeMillis, measured against the previous sample's clock.
+type LiveSample struct {
+	TimeMillis int64   `json:"t"`
+	CPUPercent float64 `json:"cpu"`
+	// NetworkIn and NetworkOut are bytes per second from the guest's side.
+	NetworkIn  float64 `json:"netIn"`
+	NetworkOut float64 `json:"netOut"`
+	// PacketsIn and PacketsOut are packets per second from the guest's side.
+	PacketsIn  float64 `json:"pktIn"`
+	PacketsOut float64 `json:"pktOut"`
+}

contracts/telemetry/v1/live_test.go

@@ -0,0 +1,42 @@
+package telemetryv1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The console parses these names, so a rename here breaks its live charts.
+func TestLiveSampleJSONContract(t *testing.T) {
+	data, err := json.Marshal(LiveSample{
+		TimeMillis: 1793450123250, CPUPercent: 12.5,
+		NetworkIn: 48213, NetworkOut: 1290, PacketsIn: 41, PacketsOut: 12,
+	})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"t":1793450123250,"cpu":12.5,"netIn":48213,"netOut":1290,"pktIn":41,"pktOut":12}`, string(data))
+}
+
+func TestValidLiveTarget(t *testing.T) {
+	assert.True(t, ValidLiveTarget("123456789012", "i-0123456789abcdef0"))
+	assert.True(t, ValidLiveTarget("123456789012", "i-0abc1234"))
+
+	for _, tc := range []struct{ account, instance string }{
+		{"", "i-0123456789abcdef0"},
+		{"12345678901", "i-0123456789abcdef0"},
+		{"123456789012", ""},
+		{"123456789012", ">"},
+		{"123456789012", "*"},
+		{"123456789012", "i-0123.abcdef"},
+		{"123456789012", "i-0123456789ABCDEF0"},
+		{"123456789012", "vol-0123456789abcdef0"},
+	} {
+		assert.False(t, ValidLiveTarget(tc.account, tc.instance), "%q %q", tc.account, tc.instance)
+	}
+}
+
+func TestLiveSubjectsAreOutsideMetrics(t *testing.T) {
+	assert.Equal(t, "telemetry.watch.ec2.123456789012.i-0abc1234", LiveWatchSubject("123456789012", "i-0abc1234"))
+	assert.Equal(t, "telemetry.live.ec2.123456789012.i-0abc1234", LiveSampleSubject("123456789012", "i-0abc1234"))
+}

contracts/viperblockd/legacy/v1/README.md

@@ -60,6 +60,10 @@ moved to another node; the local daemon subscribes to stop the guest that
 was using it. Node-addressed, like mount and unmount: the guest is local, and
 no other node's daemon has anything to stop.
 
+The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
+The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
+Repeats are harmless, since fencing a guest that is already fenced changes nothing.
+
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/fence.go

@@ -19,3 +19,11 @@ type VolumeFencedEvent struct {
 	Winner string `json:"winner"`
 	Reason string `json:"reason"`
 }
+
+// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
+// ends the sender's retries; Instance names the guest stopped, if one was, and
+// Error says why the daemon could not act on the event.
+type VolumeFencedAck struct {
+	Instance string `json:"instance,omitempty"`
+	Error    string `json:"error,omitempty"`
+}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -37,3 +37,18 @@ func TestVolumeFencedEvent_JSON(t *testing.T) {
 	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
 	assert.Equal(t, event, decoded)
 }
+
+// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
+// viperblockd's retries of a fence notice.
+func TestVolumeFencedAck_JSON(t *testing.T) {
+	golden := `{"instance":"i-1","error":"bad event"}`
+	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
+
+	marshaled, err := json.Marshal(ack)
+	require.NoError(t, err)
+	assert.JSONEq(t, golden, string(marshaled))
+
+	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
+	require.NoError(t, err)
+	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
+}

docs/COMMANDS.md

@@ -26,7 +26,7 @@ Every service is `PartOf=spinifex.target` and ordered by its own `After=`/`Wants
 | `spinifex-qmp-collector.service` | `spx service qmp-collector start` | Reads the `contracts/telemetry/v1` per-VM sidecar, polls telemetry QMP sockets and tap counters, publishes to NATS `metrics.ec2.*` |
 | `spinifex-shutdown.service` | `/bin/true`; the work is its `ExecStop` | Runs `spx admin node drain --local --unless-restarting`. `After=` every service, so it stops first and drains guests while the stack is still up |
 | `spinifex-nats-watchdog.service` / `.timer` | `nats-js-watchdog.sh` | JetStream liveness probe every 30s. `try-restart`s NATS when JetStream stops accepting writes while the process stays alive — what `Restart=on-failure` cannot catch |
-| `spinifex-firewall.service` | `spinifex-firewall-apply` | Oneshot nftables input policy, `WantedBy=sysinit.target` rather than the target. No `ExecStop`: remove it with `nft delete table inet spinifex_filter` |
+| `spinifex-firewall.service` | `spinifex-firewall-apply boot` | Oneshot nftables input policy, `WantedBy=sysinit.target` rather than the target. Loads the `inet spinifex_conntrack` tracking table on every node, and the policy only once the daemon has written a peer file. No `ExecStop`: remove the policy with `spinifex-firewall-apply disable` |
 | `regenerate-ssh-host-keys.service` | `ssh-keygen -A` | Oneshot for AMI-built nodes, conditional on the host key being absent |
 
 #### Operations
@@ -136,7 +136,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Flags | Description |
 |---------|-------|-------------|
 | `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. |
-| `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is allowed, and is preferred over leaving the instance unable to run, but it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
+| `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
 
 ### Upgrade Management
@@ -192,6 +192,21 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin eks restore-snapshot` | `--cluster` (required), `--snapshot` (optional, defaults to the latest snapshot in predastore), `--account` (optional, defaults to the bootstrap account) | Single-CP total-loss DR path (fail-safe): validates the snapshot exists in predastore BEFORE any mutation (a typo'd/missing key hard-fails, never resets into an empty datastore) → launches a fresh control-plane VM as a cluster-init seed (replaying the persisted create-time launch template) → sets a required-snapshot `RecoveryDirective` (`cluster-reset`) so the boot-time recovery agent aborts rather than resets-into-empty if it cannot fetch the snapshot → persists the replacement in cluster meta BEFORE re-pointing the NLB (so an NLB failure is convergeable by the reconciler, returned as a provisional status, not a hard error) → re-points the cluster NLB's apiserver and konnectivity target groups from the old CP's ENI to the new one → fences the old CP with retries, failing loudly if it cannot be confirmed terminated (split-brain guard). Any failure before the meta commit unwinds the fresh CP (terminate + clear directive) so a re-run does not stack a second resetting control plane. The returned status is provisional — success means the sequence completed, not that etcd is restored and serving; verify cluster health. HA clusters (a spread with a potentially surviving quorum) are rejected — recover those via quorum reformation instead. |
 
+### SQS System Instances
+
+These commands start the first SQS system instances on one node. They do not serve the SQS API, and a bound VM is not a ready service.
+
+Both send a signed `spinifex` admin action (`EnsureSQSWorker`, `EnsureSQSStore`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VMs, volumes, private VPC and ENIs belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. Each call waits up to 11 minutes.
+
+| Command | Flags | Description |
+|---------|-------|-------------|
+| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS worker VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The worker is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted 1 GiB system-account data volume on which the guest generates its own store key. The platform records only the public key. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. |
+| `spx admin sqs ensure-store` | `--node` (required, the worker's node), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's SQS store VM on the worker's node with an encrypted system-account data volume, or reports the VM already bound there: node, region, instance, instance type and address. Needs the worker bound with its public key recorded, else `IncorrectState`. The store accepts only that key. A different size, type or key than the first call is `IncorrectState`. |
+
+`ServiceUnavailable` never proves that no SQS resources exist, because an earlier attempt may already have created some. Its message says what it does establish. When no node serves the subject, this request was not delivered. When a volume create was not sent, that particular create did not happen. When the node cannot boot the guest yet or its launcher lacks a capability, this request stopped before allocating or recording anything. When the gateway times out waiting for the node's reply, nothing is known: the launch may still be running, so never assume it created nothing. In every case, call the command again with the same arguments. A retry first checks the durable record of the attempt and continues only when that is safe; it can proceed to the first launch after a create that was proven unsent, and otherwise reports the recorded outcome as bound or `IncorrectState`, or returns `ServiceUnavailable` again. `IncorrectState` from a volume, network interface or launch step means that step was recorded as sent but its outcome was never observed. It is never resent, the message names the volume when its ID is known, and no tool reconciles it yet: recovery is manual.
+
+`bound` says only that the VM was launched for the current record. The worker decides store readiness itself, by checking that the store is bound to its own public key and that a NATS connection with that key succeeds; nothing reports that readiness to the operator. The worker-to-store connection does not authenticate the store server. Real VPC routing, IMDS and gateway TLS from the guest, and worker-to-store traffic on a real interface have not been tested.
+
 ## Private Admin API
 
 Not an AWS API: a Spinifex-internal surface on the AWS gateway, JSON in and JSON out, one method per path segment at `POST /admin/<Method>`. It exists so a self-service signup form, a provisioning system or a load-test harness can create and remove tenant accounts without an operator shell on a node.

docs/DESIGN.md

@@ -181,12 +181,17 @@ type Daemon struct {
 
 Tracks available and allocated CPU/memory to prevent overcommit, and drives the dynamic `ec2.RunInstances.{type}` subscriptions:
 
+One instance vCPU is one CPU the kernel has online: a thread with SMT on, a core with SMT off or on ARM, as AWS counts a vCPU.
+The OS and Spinifex's own services keep 1 CPU on a host with up to 8, 2 up to 16, 4 up to 63 and 8 from 64, and every other CPU is schedulable one-to-one with no overcommit.
+[ADR-0015](https://github.com/mulgadc/mulga/blob/main/docs/adr/0015-host-cpu-capacity-and-reservation.md) fixes this rule.
+
+
 ```go
 type ResourceManager struct {
     mu            sync.RWMutex
-    hostVCPU      int                                // raw runtime.NumCPU
+    hostVCPU      int                                // online CPUs (ADR-0015:S1)
     hostMemGB     float64                            // raw /proc/meminfo MemTotal
-    reservedVCPU  int                                // held back for spinifex services
+    reservedVCPU  int                                // ADR-0015:S2 tier: 1, 2, 4 or 8
     reservedMem   float64                            // held back for spinifex services
     allocatedVCPU int
     allocatedMem  float64
@@ -259,6 +264,23 @@ These rules bound only the holder's own clock.
 The storage backend does not reject a stale writer, so a paused or partitioned host that resumes after its validity can still write; generation enforcement at the write boundary is required by Q-24 in the umbrella repository's `docs/specs/viperblock/rfc3-1_overview.md` and is not implemented.
 An export that survives a daemon restart writes without a renewed lease until recovery adopts it, and a survivor that recovery cannot adopt while a guest still uses it is left running without a lease or a fence.
 
+#### Unsealed writes and start elsewhere (interim)
+
+A volume is unsealed while its node holds writes the backend has not confirmed; the node records this in a dirty marker naming itself as owner (`providers/ebs/viperblock/volume_dirty.go`).
+Opening such a volume on another node loses those writes, so two guards stop it while the previous owner may still hold them.
+This is an interim guard until the instance lifecycle work defines ownership, and it is not a fence.
+
+When `ec2.start` is forwarded to the node that last ran a stopped instance and that node refuses for capacity, the forwarding node normally starts the instance locally.
+If the refusing node still holds unsealed writes for any of the instance's volumes, it sets the `X-Unsealed-Writes` NATS reply header and the forwarder relays `InsufficientInstanceCapacity` instead of starting elsewhere.
+The check asks the owner's viperblockd over the per-node `unsealed` verb with a 5 second bound, and a check that fails or times out counts as holding, so the refusal is kept.
+
+Independently, a node that finds another node's dirty marker on a volume refuses to open it when that node's daemon heartbeat (`heartbeat.<node>` in the cluster-state bucket) is under 60 seconds old.
+It also refuses when the heartbeat cannot be read or no heartbeat source is wired, and the refusal maps to the held-elsewhere error that recovery retries.
+A previous owner that has stopped heartbeating is opened over with a warning, as before.
+The heartbeat age is compared across node clocks, so the 60 second window is also the clock-skew allowance.
+
+EKS control-plane recovery treats both refusals, capacity and held-elsewhere, as retryable: it keeps retrying the stopped member and does not count them against its restart attempts.
+
 ### Predastore (S3)
 
 Object storage used for:
@@ -320,7 +342,7 @@ A release without a complete image removes any previously installed one, so a la
 
 At boot the guest reads three firmware config values: the data disk's serial, an IPv4 address on one of its interfaces, and one NATS user NKey public key.
 It mounts exactly one matching disk at `/data` after `e2fsck -p`, refuses a missing, duplicate, unformatted or unrepairable disk, and never formats.
-JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), and the server listens only on that address and accepts only that key.
+JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), `max_payload: 2MiB` admits a full 1 MiB SQS message with its headers (ADR-0011:S4), and the server listens only on that address and accepts only that key.
 On shutdown the store is stopped before `/data` is unmounted; a store that survives SIGKILL leaves every mount in place.
 
 A launch uses the versioned subject `system.LaunchSQSStore.v1`, which a daemon without this code does not serve, so it fails with no responder instead of booting another guest.
@@ -343,6 +365,91 @@ Evidence is local only: unit tests, real QEMU boots against disposable scratch d
 Viperblock, volume encryption, the EC2 volume record path, multiple nodes and host reachability of the guest address are not tested.
 Credential issuance and rotation, store placement and NATS readiness checks do not exist, and there is no failover: a lost node leaves its store unavailable.
 
+### SQS worker guest (operator-launched)
+
+The shared SQS worker guest holds the SQS store's client credential, so that no platform component does (ADR-0011:S2).
+Only an operator activates it, with the super-admin `EnsureSQSWorker` action described below; nothing starts it at daemon startup, and no worker service runs after boot.
+
+`make build-sqs-worker-microvm-image` builds it with the store image's script and network `/init` under `SQS_MICROVM_PROFILE=worker`; it carries the static `sqs-worker` binary and the ext4 tools, no `nats-server`, and an empty `/data` mountpoint.
+Releases build it beside the store image and ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker` with the store image's rules: a release without it removes an earlier one, and an incomplete one is refused.
+At boot `sqs-worker boot` reads three firmware config values under `opt/spinifex/sqs-worker/`: the worker disk's serial, its Region and the gateway URL; none is a first-store decision, because a static value survives a guest reset.
+A missing, empty or malformed value refuses before any disk is touched.
+
+It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and then confirms from `/proc/self/mountinfo` that this disk, as a whole ext4 filesystem, is what is mounted there; an empty mountpoint, a foreign mount or a failed check or mount refuses, and the guest never formats.
+Only then does it ask the gateway for the current decision with `ClaimFirstStore`, signed by `internal/sqsworkergw` with credentials from the instance metadata service only (`gwsign.NewInstanceRole`, no environment, profile or static-key fallback), over TLS 1.3 that trusts only the cluster CA from `opt/spinifex/ca-cert` and never follows a redirect.
+Under that decision `workerkey.Acquire` creates the key on `/data/sqs-worker-key`, owner-only and synced, records only its public key and then uses it, or loads the recorded key; a missing or different key once one is recorded refuses and is never replaced.
+The VM can start before its launch binds the instance, so its first calls are legitimately `AccessDenied`; that denial creates nothing and is asked again, as are a lost reply and a `ServiceUnavailable` or `Throttling` refusal, all within one bound (60 attempts 5 seconds apart), after which a worker that was never bound, or was replaced, refuses the boot.
+A signature or other refusal, or an answer that is neither "create" nor one valid public key, refuses the boot at once.
+
+`domains/sqs/firststore` is the SQS domain's first-store decision: one record per Region in the `spinifex-sqs-first-store` KV bucket moves from `claimed` to `key-recorded` to `launch-intent` under compare-and-swap, bound to the worker's instance ID, claim ID and generation.
+A claim authorizes key creation only while no public key is recorded; once one is, every claim returns only that key, and a store launch intent needs it recorded first.
+An older generation, a different identity at the same generation or an unreadable record authorizes nothing, and the record holds no private key material.
+Reads are stream message gets answered by the stream leader, never direct gets a lagging replica can answer, and every authorizing answer is also confirmed by a compare-and-swap at the revision it read, so a stale or unavailable read authorizes nothing.
+The authenticated caller that will carry this decision must derive the worker's instance ID and generation from authoritative instance state; a generation supplied in a request cannot itself authorize a takeover.
+
+`domains/sqs/workerident` is that authoritative instance state: one record per Region in the `spinifex-sqs-worker-identity` KV bucket holds a monotonic generation, claim ID, pinned node and encrypted data volume, and moves from `launching` to `bound` to `stopped`, or from `launching` to `failed`, under compare-and-swap.
+A launch intent is refused while the previous attempt is `launching` or `bound`, including one whose outcome is unknown, so only one caller ever holds an intent to start a VM on the pinned volume.
+A replacement takes the next generation only after `FailLaunch` or `RecordStopped` records that nothing holds the volume, and each request names the generation it supersedes, so a replayed request cannot take a generation it never observed; there is no automatic takeover.
+The launch path then raises the first-store fence to the successor's generation with `firststore.Fence` before `BeginLaunch`, which refuses a replacement until a leader read shows that fence; the fence only rises, and `firststore` refuses every call from a generation below it.
+Only the current claim ID is recorded, so claim IDs are minted fresh per attempt and a VM launch must be idempotent on generation and claim together.
+Binding writes the instance's index entry first and then binds the record to that instance.
+The index is only a lookup: `Authorize` accepts an instance only while the Region's record is `bound` to exactly that instance, generation and claim, confirmed by a compare-and-swap at the leader-read revision, so an old, orphaned, rewritten or unbound entry authorizes nothing.
+A same-instance reboot keeps its instance ID and identity; a replacement takes a new generation, and the first cut refuses another node or data volume and an unencrypted volume, although the encryption flag is the caller's claim and the launch wiring must verify it from the volume's metadata.
+Both buckets are opened through `OpenDurableKV`, which refuses an existing bucket whose effective stream configuration could expire, evict, lose, lag or refuse records: non-limits retention, an age, per-message or delete-marker TTL, memory storage, asynchronous flush, a size, count or message-size cap, a discard policy other than discard-new, unacknowledged writes, a sealed stream, a mirror or sources, or fewer replicas than the cluster needs.
+`workerkey.Acquire` drives that decision: it syncs a new key to disk, records its public key and only then returns it, so a crash before recording reuses the key already on disk.
+
+`domains/sqs/awsapi/worker` carries the decision to the worker as `ClaimFirstStore` and `RecordWorkerPublicKey`, served under the internal `spinifex-sqs` signing service, which no AWS SDK signs for; the public AmazonSQS action inventory is untouched.
+The gateway admits only an assumed-role session of `sqsWorkerRole` in the system account, then the role's policy, and only then resolves the session name, the instance ID STS set, through `workerident.Authorize`; the request carries no identity, and any parameter beyond the action's own is refused.
+The first-store identity is the current record's instance, claim and generation, and a worker bound to another Region is refused.
+A request that passed `Authorize` just before a confirmed stop and replacement still carries the old generation; the fence was raised before the successor began, so `firststore` refuses it whether or not the successor has claimed yet.
+The adapter never logs a caller-chosen parameter name, because one could carry a secret.
+`RecordWorkerPublicKey` accepts only a valid public user key, so a seed is refused without being recorded, logged or echoed.
+The gateway binds both buckets per request and never creates them, so it provisions nothing and answers `ServiceUnavailable` until a worker launch has created them; the worker launch below ensures the role.
+The private key stays on the worker disk; the guest logs and publishes only the public key to `/run/sqs-worker/public-key`.
+
+`domains/sqs/workerlaunch` realizes the worker on the node it runs on, its only node: a bound record is returned as is; otherwise it takes the intent, then on the first launch marks the intent's one volume create, prepares the volume and records its ID, reads the volume's own record and refuses it unless encrypted, ensures the role, finds or creates the network for the claim and reads the recorded store key, and marks the intent dispatched before launching.
+A dispatched launch is never sent again: a lost reply or error leaves the intent `launching` and dispatched, and every later call returns `ErrOutcomeUnknown` until the outcome is reconciled, since the launch path is not idempotent.
+A failure before dispatch leaves an undispatched intent that the next call resumes with the same claim; a confirmed stop or failure is replaced only after `firststore.Fence` excludes the old generation, reusing the pinned volume.
+The launch passes no store decision or key: the guest asks the gateway at boot.
+The intent exists before any volume, so concurrent first launches allocate nothing for the losers. A Prepare proven to have sent no create withdraws the mark and returns `ServiceUnavailable`, or `UnsupportedOperation` when the cluster storage key is not configured, and a retry may prepare the volume; a create that may have applied is never sent again, and every later call returns `IncorrectState` until it is reconciled. A volume the call knows of but cannot use is reported with its ID in a `VolumeError` and left in place, never deleted. An intent that has sent no create may be re-typed; one that has keeps its type until replaced. The worker role is created with EC2-only trust, and an existing role is checked for that trust before any policy is granted; the instance profile must carry exactly the worker role, and one holding any other role is refused unchanged.
+The daemon's `LaunchSQSWorkerInstance` boots the `sqs-worker` image through the same direct-boot path as the store, with the data volume attached at boot and kept on terminate, a required instance profile, and a required mgmt NIC; it re-reads the volume's own record and refuses one that is not available and encrypted, before anything is allocated.
+Its firmware config adds the gateway URL on this node's mgmt bridge and the cluster CA from the daemon's own trust store; the caller cannot supply either, and no value written is secret.
+A relaunch rebuilds that config from the VM record on the node that launched it and is refused anywhere else, and membership takeover already skips SQS-managed instances.
+`sqsWorkerRole` is ensured in the system account granting exactly `spinifex-sqs:ClaimFirstStore` and `spinifex-sqs:RecordWorkerPublicKey`, from the `domains/sqs/workerrole` contract the gateway gate also reads; an existing role whose trust admits more than the EC2 service is refused.
+
+Evidence is local only: unit tests with a fake firmware config, sysfs and mount table, tests running the boot through the real client against the real `spinifex-sqs` registration and records, and QEMU boots of the built image against disposable scratch disks and a host-side test gateway and metadata service, showing the first boot creates the key, reboots reuse it, a key lost after recording is never replaced, and the seed reaches no log or request.
+The worker's network is a private SQS system VPC per Region, built by `systemvpc.EnsurePrivate` from the `[sqs]` supernet (default `10.240.0.0/14`): one VPC and one /24 subnet, with no internet gateway, NAT gateway, EIP, route table or daemon-to-store node port. Each launch claim gets one ENI in that subnet, in the VPC default security group, found again by its claim on a retry; two ENIs for one claim are refused. The launch intent is marked before the one create is sent, so a create whose ENI is not yet visible is never sent again: the launch fails closed with `IncorrectState` until the ENI appears or the outcome is reconciled. Worker-to-store TCP and the guest's route to the metadata service are not yet proven by any traffic test.
+The super-admin `EnsureSQSWorker` action on the internal `spinifex` operator service takes a `NodeId`, which must be a node of this cluster and a single subject token, and an `InstanceType`, and sends a request to `sqs.worker.Ensure.v1.<node>` with no queue group, so only that node answers and the worker stays pinned.
+That node's daemon re-checks the node the request names and the instance type, and refuses as `ServiceUnavailable` before allocating a volume or network or recording an intent unless the worker image's kernel and initramfs are non-empty files, its mgmt bridge exists and the cluster CA is readable; a call after the image is installed then launches.
+It then runs its one composed launcher for its own Region, one launch at a time, in the daemon's own context with a 30-minute bound, so a caller's timeout does not cancel it.
+A reply lost to that timeout is reported as `ServiceUnavailable`; calling again reports the bound worker, or `IncorrectState` for a dispatched launch whose outcome is unknown, which is never resent; a Region pinned to another node is refused as `InvalidParameterValue`.
+The reply names the bound VM's node, Region, instance ID and generation; bound is not ready, and it says nothing about the guest's boot, its key or the store.
+Stop/detach evidence for `RecordStopped` is not wired, the guest's metadata route and gateway TLS on a real ENI and worker-to-store TCP are unproven, and Viperblock's at-rest key is held by the platform.
+
+The worker identity record holds the instance type of its generation; a request for another type is refused as `IncorrectState`, before dispatch and after binding, the reply names the recorded type, and only a replacement after a confirmed stop may change it.
+Operators call both actions with `spx admin sqs ensure-worker --node --instance-type` and `spx admin sqs ensure-store --node --instance-type --volume-size-gib`, signed for the `spinifex` service with the standard AWS credential chain like the other `spx admin` gateway commands.
+Each command refuses missing inputs before sending anything and reports only the bound VM; a bound VM is not a ready worker or store.
+The super-admin `EnsureSQSStore` action takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB` and sends a request to `sqs.store.Ensure.v1.<node>` with no queue group; only the node the Region's worker is bound to can launch the store.
+That node refuses an instance type whose memory does not exceed 512 MiB, the store's `GOMEMLIMIT` under ADR-0011:S11; this is a floor, not a validated size, and no type is defaulted.
+It refuses before recording or allocating anything unless the store image and the cluster storage key are present, the worker is bound on this node, and that worker's identity has recorded its public key in the first-store decision, which it then advances to a launch intent.
+The store's own launch record in `spinifex-sqs-store-launch` holds only that public key, the claim, the node and the operator's instance type and size, and marks each step before its uncertain create: the volume, the ENI in the worker's private subnet, and the launch.
+A later request for a different instance type or size is refused as `IncorrectState`, before dispatch and after binding, and the reply names the type the store was launched with.
+A volume preparation that failed before sending any create, such as a missing `mkfs.ext4` or a scratch or format failure, withdraws its mark and returns `ServiceUnavailable`, so a retry may try again once the host is fixed.
+Any other marked step whose result is not visible is reported as `IncorrectState` and never resent; a volume that is not encrypted by its own record is refused as `UnsupportedOperation` with its ID kept, and the daemon re-checks the volume's encryption and the guest's size at launch.
+The daemon never receives the worker's seed; the store's NATS server accepts only that public key, and the reply's bound VM and address are not readiness, which only the worker can check.
+
+After boot, the worker's long-running `sqs-worker ready` step mounts and verifies its disk again and calls the internal `spinifex-sqs` `DescribeStore` action, which admits only the current worker and returns the bound store's address and recorded public key, or pending.
+It loads its key only if it matches that public key, then connects to the store's NATS on port 4222 with it and asks JetStream for account info, each pass bounded to 30 seconds.
+The step runs until shutdown with no overall deadline, so a store the operator launches at any time is picked up: it asks again every 5 seconds while the store is pending, not answering or the reply was lost, and re-checks a ready store every 30 seconds.
+A pass writes only the address and public key to `/run/sqs-worker/store-ready`; any failed pass removes that file, and so does the step's own exit.
+A start that finds shutdown in progress or no worker key logs once and waits in place rather than exiting, so init does not respawn it in a loop; shutdown stops it like a running watcher.
+Init respawns the step, at most once every 5 seconds, and every start removes a marker left by a watcher that was killed or crashed before the new watcher runs.
+A ready store that fails is noticed up to one re-check interval plus one bounded call later, so the marker can be up to about a minute stale.
+Shutdown blocks further respawns, removes the marker, and stops the watcher by its recorded PID and start time, escalating to SIGKILL after a budget; it unmounts the worker disk only once the watcher is confirmed gone, and a survivor leaves every mount in place.
+A mismatched key, a store that refuses the key, a key missing from the disk or an unclassified gateway refusal is asked again only every 5 minutes, so it never spins and still recovers if the operator fixes it; a missing key is never replaced.
+The NKey authenticates the worker to the store, but nothing authenticates the store to the worker: there is no TLS, so a pass means only that something at that address accepted the key.
+This is store readiness for the worker alone; there is no SQS API, and no production sizing or support is claimed.
+
 ## Configuration
 
 Cluster configuration (`spinifex/bootstrap/config/config.go`):

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -353,12 +353,22 @@ while the branch is in flight; they do not describe the umbrella repository's
 | `spinifex#1273` | `spinifex/gateway/{ecs,elbv2}` | `spinifex/domains/{ecs,elbv2}/awsapi` (`awsapi`, imported as `ecsawsapi`, `elbv2awsapi`) | A mechanical relocation of the ECS and ELBv2 AWS action adapters to their ADR-0001 domain homes, as `spinifex#1265` did for IAM, STS, EKS and RDS; APIs, NATS subjects, KV buckets and behaviour are unchanged. The root `gateway` dispatch still imports each package, recorded as `gateway` (service registration) debt, and `domains/admission/quota` keeps its live ELBv2 counter import, now of `domains/elbv2/awsapi`. |
 | `spinifex#1336` | `spinifex/gateway/tagging.go` (`Tagging_Request`, `taggingActions`, `taggingActionFromTarget`) and the `GatewayConfig.TaggingResources` field | `spinifex/domains/tagging/awsapi` (`NewRegistration`, `Deps`, `Actions`, `OperationInventory`), composed by `spinifex/runtime/roles/awsgw` | Resource Groups Tagging registers through the ADR-0002 seam like ACM and Route 53. The check order is kept: action, scope, authorization, account, body, aggregator, so a denied caller still gets `AccessDenied` before any body error and an unwired aggregator answers `ServerInternal`. The `checkPolicyResources` stand-in of `*` for every action is unchanged. The legacy switch case, the `supportedServices` entry and the `jsonErrorService` case are removed, and the registration declares the JSON 1.1 envelope. No allowlist row existed for this path. The nil-NATS check moves to the gateway-wide gate, which answers `ServiceUnavailableException` before dispatch as it did for every service. `cmd/aws-model-coverage` output is byte-identical because there is no embedded Tagging model. The characterization tests that pin the pipeline behaviour were committed first and pass unchanged against the registration; only the `withTagging` test helper changed. |
 | `spinifex#1336` | `spinifex/gateway/elbv2.go` (`ELBv2_Request`, `elbv2Actions`, `elbv2Handler`, `accountLoadBalancerLimit`) | `spinifex/domains/elbv2/awsapi` (`NewRegistration`, `Deps`, `Quota`, `ActionNames`, `OperationInventory`), `spinifex/domains/admission/quota` (`Service.LoadBalancerLimit`), `spinifex/ingress/aws/query` (`ParseArgs`), composed by `spinifex/runtime/roles/awsgw` | ELBv2 registers through the ADR-0002 seam as the first AWS Query service. The check order is kept: query parse, action, account, input parse, scope, authorization, NATS, dispatch, so a denied caller still gets `AccessDenied` before the handler runs. The gateway members ELBv2 read are now supplied as `Deps`: `NATSConn` as `Deps.NATS`, `Quota` as the `Deps.Quota` interface declared in `awsapi` because `domains/admission/quota` imports `awsapi`, `Region` as `Invocation.Region`, and the policy gate as `Invocation.Authorize`. The `DescribeAccountLimits` cap resolution moves to `Service.LoadBalancerLimit` unchanged, and a nil `Deps.Quota` enforces nothing and reports the AWS default. AWSGW now wires the registry after it builds the quota service, because the registration captures it. The registration declares the XML envelope and the inventory name `elasticloadbalancingv2`, so `cmd/aws-model-coverage` keys it explicitly and its output is byte-identical. `gateway.ParseAWSQueryArgs` delegates to `query.ParseArgs`. The legacy switch case, the `supportedServices` entry, the inventory entry and the `gateway -> domains/elbv2/awsapi` allowlist row are removed. `cmd/aws-model-coverage` imports `domains/elbv2/awsapi` for the inventory, under a fourth named-exception allowlist row whose reason is now generic for all four. Finding, not fixed here: the XML for `DescribeAccountLimits` and `DescribeLoadBalancers` lists member fields in a different order from run to run because the marshal is map-ordered, so the characterization tests pin decoded content rather than bytes. The characterization tests that pin the pipeline behaviour were committed first and pass unchanged against the registration; only the `withELBv2` and `serveELBv2` test helpers and the setup of two older gateway tests changed. |
-| `spinifex#1337` | `spinifex/gateway/spinifex.go` (`Spinifex_Request`, `spinifexAdminActions`) | `spinifex/operator/admin` (`NewRegistration`, `Deps`, `ModelAccess`, `ActionNames`), composed by `spinifex/runtime/roles/awsgw` | The `spinifex` Query service registers through the ADR-0002 seam with the XML error envelope it had. The check order is kept: query parse, action, authorization with the wildcard resource, account, the super-admin account gate, then the action, so a denied caller learns nothing about which actions exist. The gateway members the actions read are now `Deps`: `Version` and `Commit` strings, `NATSConn` as `Deps.NATS`, `RootCAs`, `DiscoverActiveNodes` as `Deps.ActiveNodes`, `BedrockAccessAdmin` as the `ModelAccess` interface, and the credential and access resolvers as `Deps.Credentials` and `Deps.Access`. The query actions need only the caller account, so the actor is not widened. The cluster-unavailable answer stays in the shared request path ahead of authorization. `/admin/{method}` stays on its chi route in `gateway/admin*.go` until the actor carries `UserName`. |
+| `spinifex#1337` | `spinifex/gateway/spinifex.go` (`Spinifex_Request`, `spinifexAdminActions`) | `spinifex/operator/admin` (`NewRegistration`, `Deps`, `ModelAccess`, `ActionNames`), composed by `spinifex/runtime/roles/awsgw` | The `spinifex` Query service registers through the ADR-0002 seam with the XML error envelope it had. The check order is kept: query parse, action, authorization with the wildcard resource, account, the super-admin account gate, then the action, so a denied caller learns nothing about which actions exist. The gateway members the actions read are now `Deps`: `Version` and `Commit` strings, `NATSConn` as `Deps.NATS`, `RootCAs`, `DiscoverActiveNodes` as `Deps.ActiveNodes`, `BedrockAccessAdmin` as the `ModelAccess` interface, and the credential and access resolvers as `Deps.Credentials` and `Deps.Access`. The query actions need only the caller account, so the actor is not widened. The cluster-unavailable answer stays in the shared request path ahead of authorization. `/admin/{method}` moved onto the same registration in `mulga-3nbla.7`, recorded below. |
 | `spinifex#1344` | `spinifex/gateway/rds.go` (`RDS_Request`, `rdsCaller`) | `spinifex/domains/rds/awsapi` (`NewRegistration`, `Deps`, `Quota`, `CallerFromActor`, `OperationInventory`), composed by `spinifex/runtime/roles/awsgw` | RDS registers through the ADR-0002 seam with the XML envelope it had. The check order is kept: query parse, action, account, the principal-class gate for the internal agent actions, scope, authorization, NATS, quota and dispatch, so a denied caller still gets `AccessDenied` before the handler runs and no customer grant is evaluated against an agent action. The gateway members RDS read are now supplied as `Deps`: `NATSConn` as `Deps.NATS`, `ExpectedNodes` as `Deps.ExpectedNodes`, `Quota.EnforceRDSInstances` as the `Deps.Quota` interface, `Region` as `Invocation.Region` and the policy gate as `Invocation.Authorize`. A nil `Deps.Quota` skips the DB-instance cap. The caller comes from `dispatch.Actor`: the session name for an assumed-role session and the user name otherwise, and the role name from `Actor.RoleName()`, which parses the underlying role ARN and never the session name. The account comes from `Invocation.AccountID`, so a caller with no composable ARN still reaches the policy check and is refused there as before. The legacy switch case, the `supportedServices` entry, the inventory entry and the `gateway -> domains/rds/awsapi` allowlist row are removed, and `cmd/aws-model-coverage` reads the inventory under a fifth named-exception allowlist row. The characterization tests that pin the pipeline behaviour were committed first with spinifex#1336 and pass unchanged against the registration; only the `withRDS` and `serveRDS` test helpers and the setup of the direct-call RDS tests changed. |
 | `#1277` | `spinifex/accountteardown` | `spinifex/operator/accountlifecycle` (package `accountlifecycle`, imported as `accountteardown`) | A mechanical whole-package relocation under the 2026-10-27 ADR-0001 amendment: the account deletion coordinator keeps its APIs, NATS subjects, KV access and behaviour; importers keep the `accountteardown` name as an alias, so only their import paths change. Replacing its direct domain and adapter cleanup calls with owner capabilities stays deferred debt. |
 | `#1285` | `spinifex/gateway/requestaudit.go` and `slogRequestLogger` from `spinifex/gateway/gateway.go` | `spinifex/ingress/aws/audit` (package `audit`) | The first generic-ingress extraction from the root gateway: the request-audit middleware and access-log middleware move unchanged in behaviour. The gateway's authentication flow and dispatch still populate the record, now through the exported `audit.From` and `Record` setters at the same call sites; the package imports only `foundation/telemetry` and `ingress/http`. Its unit tests moved with it; the tests that run it behind SigV4 stay in `gateway`. |
 | `spinifex#1334` | `spinifex/gateway/spx` (`spx.go`, `storage.go`, `promote_image.go` and their tests) | `spinifex/operator/admin` (package `admin`) | The version, node, VM and storage-status gather helpers and the image-promote NATS request move unchanged with no shim; the `spinifex` service dispatch that calls them stays in `gateway/spinifex.go` until the `spinifex` registration slice. `operator/admin` already imports `domains/ec2/image` and `contracts/operator/v1`, so the move adds no edge and the layering ratchet needed no allowlist row. The `gateway/spx` frozen-legacy entry is removed. |
 | `spinifex#1334` | `spinifex/gateway` (`CreateAccountRequest`/`Response`, `DeleteAccountRequest`, `DescribeAccountDeletionRequest`, `DeletionState*`, `AccountSummary`, `ListAccountsResponse`, `AdminMethodNames`) | `contracts/operator/v1` (`admin.go`) | The `/admin` bodies the CLI shares with the gateway move unchanged in field names, JSON tags and omitempty, pinned by golden JSON written against the gateway types first and moved with them. `AdminMethodNames` is now owned by the contract and the gateway builds its `adminMethods` accept set from it, with a test that the router accepts exactly that list. `AccountQuotaRequest`/`AccountQuotaResponse` (embed `quota.Overrides` from `domains/admission/quota`) and `DeleteAccountResponse` (embeds `accountlifecycle.Result`) stay in `gateway`: `contracts` may import only `foundation` and `contracts`, so they are settled when the `/admin` handlers move to `operator/admin`. `accountlifecycle.Stage` serialises as an integer, not its name; the golden test pins that. |
+| `spinifex#1345` | `spinifex/gateway/admin*.go` (the `/admin/{method}` handlers, gates and body cap) and the `AccountQuotaRequest`/`AccountQuotaResponse`/`DeleteAccountResponse` wire types | `spinifex/operator/admin` (`rest.go`, `account_*.go`), served by the same `spinifex` registration | `/admin/{method}` stays a chi route in `gateway`, ahead of the service selector, but `Admin_Request` now only mints the request ID and header and hands the request to the registered entry through `dispatchRegistered`, the path every Query request takes. `dispatch.Invocation` gains `Service`, the credential-scope service, so the scope gate is the first identity gate in the dispatcher and a key signed for another service is denied exactly as before. The order is unchanged: verb and method, then the five fail-closed gates (scope, super-admin account, IAM user, identity, `spinifex:<Method>` policy), then the cluster check, then the 8 KiB body cap. The dispatcher returns errors and the route renders them as the JSON envelope with `requestId`, then logs once from the real result, so a denial or a 5xx is never logged as a completed request; the registration's `ResolveAction` labels the method, which removes the `auth.go` special case. The caller is read from `dispatch.Actor` (`UserName` for the gate and `updatedBy`). `BucketStore` and `SignupMaxAccounts` leave `GatewayConfig` for `operator/admin.Deps`, composed in `awsgw` beside `IAM`, `Quota`, `ExpectedNodes` and `NodeIDs`. The three wire types that could not enter `contracts` move to `operator/admin` with their handlers, `operator/cli` imports them from there, and the last `operator/cli` → `gateway` allowlist entry is deleted. |
+| `spinifex#1346` | `spinifex/gateway/iam.go` (`IAM_Request`, the IAM action table, `getUserForCaller`, the PassRole check) | `spinifex/domains/iam/awsapi` (`NewRegistration`, `Deps`, `ActionNames`), composed by `spinifex/runtime/roles/awsgw` | IAM registers through the ADR-0002 seam with the XML error envelope it had. The check order is kept: query parse, action, service and account checks, then one parse of the input, the `GetUser` default for a missing `UserName`, scope resolution, authorization, the `iam:PassRole` check for `AddRoleToInstanceProfile`, then the handler. The caller is read from `dispatch.Actor` (`PrincipalType`, `UserName`), with no ARN parsing. The dependencies are `Deps.IAM` and `Deps.CountInstanceProfileAssociations`, composed in `awsgw`, and the gateway keeps no IAM fallback. The `cmd/aws-model-coverage` named exception gains an `iam` row. |
+| `spinifex#1346` | `spinifex/gateway/sts.go` (`STS_Request`, the STS action table, the anonymous STS dispatch) | `spinifex/domains/sts/awsapi` (`NewRegistration`, `Deps`, `ActionNames`, `NewAnonymousAssumeRoleWithWebIdentity`), composed by `spinifex/runtime/roles/awsgw` | STS registers through the ADR-0002 seam with the XML error envelope it had. The check order is kept: query parse, action, service check, caller, the `AssumeRole` identity gate scoped to the stored role ARN, then the handler. The caller is read from `dispatch.Actor` (`CallerARN`, `UserName`, `SessionName`, `UserID`, `AccessKeyID`), with no ARN parsing. The gateway's authorization failure wraps a generic `dispatch.PolicyDenialError`, and STS matches it to answer with the caller, action and resource, so the gateway holds no STS-specific denial code. An unsigned `AssumeRoleWithWebIdentity` stays ahead of SigV4 in the gateway and reaches the handler through `GatewayConfig.AnonymousSTS`, a func composed in `awsgw`, never the STS service. The `cmd/aws-model-coverage` named exception gains an `sts` row. |
+| `spinifex#1348` | `spinifex/gateway/ecs.go` (`ECS_Request`) and `spinifex/gateway/eks.go` (`EKS_Request`, the EKS route table) | `spinifex/domains/ecs/awsapi` and `spinifex/domains/eks/awsapi` (`NewRegistration`, `Deps`, `OperationInventory`), composed by `spinifex/runtime/roles/awsgw` | ECS (AWS JSON 1.1) and EKS (REST JSON) register through the ADR-0002 seam with the JSON error envelopes they had, both declared as AWS JSON 1.1 (see the envelope divergence below). The check order and every response are kept, pinned by the pipeline characterisation tests committed first. EKS and ECS resolve roles through `foundation/aws/passrole` `Check` with `domains/iam` `RoleResolver`: a malformed or foreign role is refused first, a missing role is authorized as written before it is reported, and a resolver outage is returned unchanged. Role handling now follows the AWS capture of 2026-10-30. ECS RegisterTaskDefinition refuses a foreign role with `ClientException` "Role is not valid" before PassRole and accepts a nonexistent, invented-path or non-ARN role from a caller who can pass roles. ECS RunTask refuses a foreign `taskRoleArn` or `executionRoleArn` override the same way and answers a nonexistent or invented-path one with "ECS was unable to assume the role". EKS CreateCluster and CreateNodegroup answer a non-ARN or foreign role with `AccessDeniedException` "Cross-account pass role is not allowed.". EKS public routes now report every parameter fault as `InvalidParameterException`, the code the EKS model declares, while the internal control-plane routes keep `InvalidParameterValue` because the guest agents match on it. Two deliberate EKS divergences remain. First, CreateCluster still refuses a nonexistent or invented-path role with `InvalidParameterException` "role ... does not exist", where AWS accepts it and the cluster later ends FAILED. Our EKS only stores, validates and echoes the cluster role and never assumes it, so accepting would leave an ACTIVE cluster bound to a role that does not exist. Second, CreateNodegroup keeps its existing refusal of a nonexistent role because AWS was not captured for it. A third divergence is the error envelope. EKS is restJson1 but registers `ErrorEnvelopeJSON`, so errors carry `application/x-amz-json-1.1` and a `__type` body where AWS sends `application/json` and a `message` body. The `X-Amzn-Errortype` header is correct and SDKs read the code from it, so impact is low. Switching to `ErrorEnvelopeRESTJSON` also changes the throttle code and the default internal-error code (`ServiceException` where the EKS model declares `ServerException`), so it is left for its own change. Verified live on env14: the ECS and EKS role cases above, and a valid CreateCluster reaching ACTIVE with the control-plane agent publishing through the gateway. The EKS route table and its tests move with the registration, and the gateway keeps no ECS or EKS fallback. The `cmd/aws-model-coverage` named exception gains an `ecs` and an `eks` row, and the two `gateway` allowlist rows are removed. |
+| `spinifex#1350` | `spinifex/gateway` (`ecr.go`, `ecrauth_middleware.go`, the `/v2/token` exchange, `ecr_principal.go` with `ResolveECRPrincipal`, the per-operation `/v2` authorization, the registry host matcher, `ctxECRPrincipal`) | `spinifex/domains/ecr/registry` (`Bridge`, `NewBridge`, `BridgeConfig`, and its own `Principal`, `Rehydrator`, `Authorizer`, `Admission` types with `ErrInvalidCredential` and `ErrUnavailable`), `spinifex/ingress/http` (`HostMatcher`, `TargetAccount`, `RequestClientIP`; names no service), `spinifex/ingress/aws/credential` (`Rehydrator`, `Principal.CallerARN`), and under `spinifex/runtime/roles/awsgw`: `credverify` (`Rehydrate`, sharing the SigV4 key and session lookups), `policyauth`, `accountadmission`, `registryadapt` | ADR-0001 amendment 2026-10-29 and ADR-0005:S2: the OCI `/v2` bridge and its token auth move out of the gateway, and ECR resolves its principal through the credential capability instead of reading `IAMService` and `STSService` directly. ADR-0002:S5 keeps `/v2` outside the seam, so the gateway keeps only a `RegistryHandler` mount that answers 501 when unset. `domains/ecr/registry` imports no ingress package (`TestADR0001_Layering` unchanged): the principal, rehydrator and authorizer are registry-local types and `registryadapt` maps `credential.Failure` codes onto `ErrInvalidCredential` (401) and `ErrUnavailable` (503). No widening of `dispatch.Actor` or `dispatch.Invocation`. The SigV4 path and the bridge share one policy evaluator (`policyauth`) and one account-admission check. Deliberate change: the open-registry mode for a nil token verifier is deleted and `NewBridge` refuses a nil Verifier, Issuer, Rehydrator, Authorizer or Admission, so tests that want an unauthenticated registry mount `registry.Handler` directly. The pre-move pipeline characterisation (status codes, error bodies, the `WWW-Authenticate` challenge, referrers, token exchange, revoked and deleted principals) passes unchanged apart from setup, and the retired `ResolveECRPrincipal` parity rows live on as `credverify` `Rehydrate` rows and `Bridge` claim-check rows. Live on env14 (DHCP external IP): crane, skopeo and docker push and pull (`tests/e2e/ecr`), a cosign sign and verify in the legacy tag mode and in OCI 1.1 referrers mode with the signature returned by `GET /v2/{repo}/referrers/{digest}`, `/v2/token` refresh from fresh logins, and a deactivated access key answering 401 on `/v2/token`, manifest GET, crane pull and crane push, then 200 again once reactivated. The ECR signing-key deadlock bead (`mulga-6wqce`) is untouched. |
+| `spinifex#1351` | `spinifex/gateway/{bedrock,bedrockruntime,bedrockagent,bedrockagentruntime}.go` (`Bedrock_Request`, `BedrockRuntime_Request`, `BedrockAgent_Request`, `BedrockAgentRuntime_Request`, the four route tables), `spinifex/domains/ochre/authz.go`, and the 13 `Bedrock*` fields of `GatewayConfig` | `spinifex/domains/ochre/awsapi` (`NewRegistration`, `Deps`, `Quota`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` | The four Bedrock services register through the ADR-0002 seam as one registration, per ADR-0002:S1 (one per signing service) and the ADR-0001 amendment of 2026-10-06: the service is `bedrock` and `bedrock-runtime`, `bedrock-agent` and `bedrock-agent-runtime` are aliases. A request signed with a sub-service scope is dispatched to that sub-service whatever its path, so a path owned by another sub-service answers `InvalidActionException` as before; only the shared `bedrock` scope is split by path. The pipeline characterisation tests committed first (status, error type and envelope per service, metering order, nil-capability answers) pass unchanged apart from setup. The 13 gateway fields become `awsapi.Deps`: `Credentials`, `Endpoints`, `Access`, `Recorder`, `Logging`, `Provisioned`, `Guardrails`, `Embedder`, `KB`, `DataSources`, `Vector` and `Quota`, with the documented nil fallbacks (no grants denies every model, a nil store errors rather than panics); `BedrockAccessAdmin` had no reader outside one test and is gone, and `Deps.NATS` is dropped because the nil-NATS answer is given by the gateway-wide gate before dispatch. `Logging`, `Provisioned`, `Guardrails`, `KB` and `DataSources` are the concrete exported store types, a temporary edge until Ochre exposes capability interfaces for them. The wire-typed action functions stay in the `domains/ochre` root and the SDK-typed adapters move; `authz.go` moves with them, which removes the `domains/ochre` to `ingress/aws/bodyscope` allowlist row, the only allowlist change. The legacy switch cases, the `supportedServices` and `jsonErrorService` entries and the path-based sub-service rewrite in `GetService` are removed, so `ec2` is the only service left on the legacy table; the rewrite's rows are pinned in `awsapi` as `resolveSubservice` and dispatch tests. The registration declares the AWS JSON 1.1 envelope it had and `cmd/aws-model-coverage` is not wired, so no coverage allowlist row is added. Recorded divergences, not changed by this move: Bedrock is restJson1 in AWS but answers with the JSON 1.1 envelope (`application/x-amz-json-1.1`, `__type` body), and CreateGuardrail answers 200 where AWS answers 202; both are tracked by `mulga-xug8b`. No change to `dispatch.Actor` or `dispatch.Invocation`. |
+| `spinifex#1354` | `spinifex/gateway/ec2.go` (the EC2 action table, `ec2Handler`, `ec2Actions`, the dispatch pipeline), `ec2_idempotency.go`, `ec2_validation.go`, and the EC2 token-bucket fields of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`Handler`, `NewHandler`, `Deps`, `Quota`, `Dispatch`, `ActionNames`, `InputHasField`), composed by `spinifex/gateway/ec2.go` | Slice 9a of the EC2 move: the capabilities first, with no behaviour change. The EC2 action table, idempotency and validation move to `domains/ec2/awsapi` behind `Deps{NATS, IAM, Quota, ActiveNodes, ExpectedNodes, NodeIDs, AZ, Endpoint, InstanceStatus}`; the gateway keeps the composition and `EC2_Request`, still on the legacy switch until slice 9b registers EC2 through the ADR-0002 seam. `Quota` is declared in `awsapi` because `domains/admission/quota` imports the EC2 action packages and cannot be imported back, and the gateway wraps `quota.Service` to supply the NATS instance-type resolver its retype gate takes. The instance and spot-instance adapters take the two-method `InstanceProfiles` interface (`ResolveInstanceProfile`, `CanonicalResourceARN`) instead of `iamdomain.IAMService`, which pays off the `spotinstance` to `domains/admission/quota` and `domains/iam` allowlist rows. Allowlist changes are 20 deletions and no additions: the 18 `gateway` to `domains/ec2/awsapi/*` rows and those 2. The pipeline characterisation tests committed first (error envelopes, check order, gateway-local actions, idempotent replay, PassRole, ImportKeyPair padding, launch-template expansion before authorization, RunInstances instance-profile refusal order) pass unchanged. Package-local tests drive every routed action through `Dispatch` against a stub daemon. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and MaxResults errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. |
+| `spinifex#1355` | `spinifex/gateway/ec2.go` (`EC2_Request` in production, the legacy service switch, its `supportedServices` and JSON tables) and the `Quota` field of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`NewRegistration`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` (`ec2.go`: `newEC2Registration`, `advertisedEndpoint`; `ec2quota`) | Slice 9b of the EC2 move: EC2 registers through the ADR-0002 seam as service `ec2` and the legacy gateway switch is empty. The EC2 query-protocol error envelope is kept through `ErrorEnvelopeXML`, and the policy denial match is now `*dispatch.PolicyDenialError`. The characterisation tests committed first also pin the Authorize service name and action, the trace span and audit action, the throttle key (account plus action) and the cluster-unavailable gate, which runs before policy and action lookup for gateway-local and NATS-backed actions alike. They and the 9a tests pass unchanged apart from setup: STS and EC2 test setups now register the services they sign as, and the two tests that claimed a legacy service name are removed because no legacy name remains. `EC2_Request` survives only as a test shim. Allowlist changes are two deletions (the `gateway` to `domains/ec2/awsapi` and `domains/ec2/awsapi/instance` rows) and one addition, the per-service model-coverage row `cmd/aws-model-coverage` to `domains/ec2/awsapi`. `startInstanceStatus` stays in `awsgw.go` because ADR-0007:S6 records the raw instance-state uses by file. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. The gateway request logs carry `service=ec2` and the per-action label. Not exercised live: a throttled request (the default limits are not reachable from one client; the throttle key is pinned by the characterisation tests) and an agent-signed EC2 call. |
+| `spinifex#1363` | `spinifex/gateway` context keys (`ctxService`, `ctxRegion`, `ctxAction`, the principal and access key id keys, and the unread `ctxQueryArgs`), the duplicate `callerPrincipal` and `principalContext` types, `gateway/caller_arn.go` (`buildCallerARN`), and the direct `policyauth`, `accountadmission` and NATS-connection reads in the auth pipeline | `spinifex/ingress/aws/reqctx` (the request record), `credverify.CallerARN`, and the gateway-consumed capabilities `PolicyEvaluator`, `AccountAdmission` and `Readiness`, composed by `spinifex/runtime/roles/awsgw` (`clusterready`) | Slice 11a of the gateway removal: the request-context record and the ADR-0002:S8 capabilities, with no behaviour change. The record is a copy-on-write value under one context key; the principal is sealed once and a second `WithPrincipal` keeps the first caller. The layering rule `S7_RequestRecordPrivateToIngress` lets only `gateway` and `ingress/aws/*` import `reqctx`, with a fault-injection test, and slice 11c moves the package to `ingress/aws/internal/reqctx` and drops the rule. The caller ARN is built once in `credverify` (root cases kept) and filled on `Principal.CallerARN`. The policy evaluator, account admission (`credential.ErrAccountNotActive`, with `accountadmission.ErrNotActive` an alias) and cluster readiness (`Available() bool`) are interfaces the gateway consumes and fails closed on when nil. The step 1 characterisation tests (throttle against both readiness gates and the policy gate for ECS and IAM, `/admin` skipping `GetService` and the second gate, the nil-connection difference between the two gates, five error writers each minting a request id that differs from the audit record, and the credential-scope region reaching only the span) pass unchanged apart from setup. No allowlist rows change, and there is no change to `dispatch.Actor` or `dispatch.Invocation`. Debt for slice 11c: the test-only `defaultCapabilities` seam (a package variable set by a test `init` so about 60 gateway test files can build a config from `IAMService` alone), the nil-connection case of the early readiness gate, the remaining `NATSConn` readers (`DiscoverActiveNodes` and the field that `awsgw` and the integration harness set), and the `IAMService`, `STSService`, `Config`, `RootCAs` and `InternalSuffix` fields of `GatewayConfig`, which no production code reads. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a bad access key refused with InvalidClientTokenId on a Query service and InvalidClientTokenIdException on a JSON service, a no-policy user refused with UnauthorizedOperation, and an assumed-role session allowed and refused per its policy. Not exercised live: a suspended-account refusal, a throttled request and the NATS-down 503, which the step 1 tests pin. |
+| `spinifex#1378` | `spinifex/gateway` `oidc_discovery.go` and `registry.go` (the `/oidc/eks` routes and the `/v2` mount), the `DiscoverActiveNodes` method and its cache, and the `GatewayConfig` fields `Config`, `RootCAs`, `InternalSuffix`, `Version`, `Commit`, `IAMService`, `STSService`, `NATSConn` and `ExpectedNodes` | `spinifex/runtime/roles/awsgw` (`PublicRoutes`, `NodeDiscovery.ActiveNodes`, `composeGateway`); the gateway keeps one `PublicRoutes` mount hook and the `Admin` registration name supplied by the role |
 | `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
 The EC2 contract row is intentionally different from the directory moves: it creates
 a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
@@ -394,9 +404,9 @@ the canonical IAM/STS principal before dispatch, because ECR must not depend on
 gateway identity context or ARN construction.
 
 All implemented ECR control-plane actions now dispatch through explicit ECR
-capabilities. The remaining ECR-named gateway code is generic `/v2/*` route
-assembly, SigV4/registry authentication and policy enforcement; it remains in
-gateway rather than inverting those dependencies into the ECR domain.
+capabilities. The OCI `/v2` bridge, its token exchange and its principal
+resolution now live in `domains/ecr/registry`; the gateway keeps only the
+`RegistryHandler` mount, and `awsgw` composes the bridge.
 
 ### ACM relocation residuals
 
@@ -445,7 +455,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `runtime/roles/vpcd` (placement): the role still mixes process/role supervision with network-controller responsibilities. `lease_reaper.go` (orphaned external-pool DHCP lease sweep), `ovn_controller_watchdog.go` (OVN controller liveness and restart), `host_eip_loop.go` (host-EIP reconcile loop) and `lease_datapath_reconcile.go` (lease datapath reconcile) are periodic network-control loops, not role wiring; `vpcd.go` itself also carries bridge-mode detection and verification, chassis-list parsing, external-CIDR resolution, DHCP manager startup and lease/IP-change handling, and gateway/host-EIP allocator and neighbour flush/prime construction alongside `Service.Start`. The future placement question is whether these belong in `domains/network`. Removal condition: an accepted decision on network-controller placement, and extraction of those loops and helpers behind the role.
 - ECR control-plane registration: `awsapi.Deps.NATS` is a raw `*nats.Conn`, not a narrow capability. Every NATS-backed `Handler` passes it to `domains/ecr.NewNATSMetaStore`, which calls `natsmsg.NATSRequest`, so narrowing it means changing the exported `Handler` type, every handler and the meta store; it must not be broadened further meanwhile.
 - ACM control-plane registration: `domains/acm/awsapi.Deps.NATS` (`registration.go`) is likewise a raw `*nats.Conn` carried by every `Handler`, for the same reason; it must not be broadened further meanwhile.
-- `cmd/aws-model-coverage` imports `ingress/aws/dispatch`, `gateway` and the `awsapi` package of each registered service (`domains/ecr/awsapi`, `domains/acm/awsapi`, `domains/route53/awsapi`, `domains/elbv2/awsapi`, `domains/rds/awsapi`) to read their declared inventories. It is a non-serving tool and the only named exception to the ADR-0002 composition invariant; it must not construct a registration.
+- `cmd/aws-model-coverage` imports `ingress/aws/dispatch`, `gateway` and the `awsapi` package of each registered service (`domains/ecr/awsapi`, `domains/acm/awsapi`, `domains/route53/awsapi`, `domains/elbv2/awsapi`, `domains/rds/awsapi`, `domains/ecs/awsapi`, `domains/eks/awsapi`) to read their declared inventories. It is a non-serving tool and the only named exception to the ADR-0002 composition invariant; it must not construct a registration.
 - Ingress pipeline (ADR-0002 phase 4): the request-audit record, the trace action enricher, the throttle key functions and the gateway request-context keys stay in `gateway` until the auth/IAM and request-context boundary is designed. SigV4 authentication writes the audit record and context keys from four places, so moving them now would export exactly the setters ADR-0002 says must not be exported solely to make a directory move possible.
 - NAT projection delivery is inconsistent and is preserved as found: EIP `AssociateAddress` sends `vpc.add-nat` unacknowledged (`AnnounceNATAdd`), skipping the 45s OVN barrier every other add path waits on, and `DeleteNetworkInterface` sends `vpc.delete-nat` unacknowledged (`AnnounceNATDelete`), skipping the ack and the ErrNoResponders retry. The reconciler's orphan sweep is the only backstop for both. Clears when those call sites use `AddNAT`/`AddNATBestEffort` and `RemoveNAT`, a reliability change of its own.
 - `vpc.add-system-egress` has a `vpcd` subscriber and a contract entry but no producer (its former `domains/ec2/eip` publisher is gone); `domains/eks/cluster` still publishes the delete. Either a producer is missing or the add route is dead; it needs an owner decision.
@@ -453,7 +463,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `foundation/messaging/nats/nats_trace.go` still names its OpenTelemetry tracer `github.com/mulgadc/spinifex/spinifex/utils`. It is an emitted instrumentation-scope name that dashboards may filter on, so renaming it is a telemetry change, not a path fix.
 - `runtime/compute/cache.New`: two construction-time panics carry a lint exemption pending a constructor that returns an error.
 - `domains/ec2/systeminstance`: managed-service parent metadata is only the `ManagedBy` tag value.
-- `domains/ochre` (action adapters): since `spinifex#1266` the Bedrock action layer (handlers, action authorization, event-stream framing and the CRUD, catalogue and Converse/Invoke entry points) shares package `ochre` with the domain core, stores and inference engine, and imports `ingress/aws/bodyscope`. Moving it to `domains/ochre/awsapi` is not mechanical: the adapters reach package-private store internals (the guardrail and provisioned-throughput store handles and get/put/update methods), and adapter tests override the package-private model catalogue. It needs core capabilities first, then the move; the top-level `gateway/bedrock*.go` HTTP handlers additionally wait for the ingress registration seam.
+- `domains/ochre` (action adapters): cleared by the Bedrock registration in `spinifex#1351`. The SDK-typed adapters and `authz.go` moved to `domains/ochre/awsapi`, the `ingress/aws/bodyscope` allowlist row is deleted, and the adapters take the concrete exported Ochre stores through `awsapi.Deps` as a temporary edge until Ochre exposes capability interfaces for them.
 - `domains/network`: `reconcile/intent.go` builds desired network state by reading EC2's KV buckets and decoding EC2's private VPC, subnet, ENI, route-table, gateway, EIP, NAT and security-group records, and `reconcile/drift.go` watches those buckets. ADR-0001 names this pair explicitly: EC2 and network communicate through a capability/projection or versioned contract, never by one being the other's KV reader. Step 7 fix: an EC2-owned `domains/ec2/projection` exposing network intent with list and watch (it may stay KV-backed, so no data moves); a narrow intent interface declared by `reconcile`; wiring at the composition root; and the duplicated `PurposeEIP`/`PurposeIGWLRP` constants and mirrored `SGRule`/`SGEvent` folded into the projection or `contracts/ec2/v1`. The drift-watch coverage test and reconcile scenario tests are the evidence the projection must keep. `domains/network` also imports `runtime/compute/vm` and `runtime/host/command` (domain-to-runtime).
 - `domains/ec2/instance`: imports `runtime/compute/vm` and `runtime/compute/gpu` (domain-to-runtime) and `domains/dns` (cross-domain implementation import).
 - `domains/ec2/instance/state` (dependency): the vpcd IMDS local reader imports `runtime/compute/vm` to return `vm.VM`. Removal condition: that type moves into a domain-owned package that `runtime/compute/vm` consumes (runtime to domain is allowed).
@@ -463,14 +473,12 @@ These residuals were recorded by individual slices and must be resolved once the
 - `operator/installer/firstboot` (security): `Config.InstallCallback` is rendered into `spinifex-firstboot.sh`, which is world-readable (0755). The field is currently unused (`fireInstallCallback` reads `SPINIFEX_INSTALL_CALLBACK` from the environment instead), but any future wiring must not place secrets or tokenised callback URLs in the generated script; sensitive callback credentials should reach first boot through a protected channel such as a 0600 file or a credential store.
 - `operator/cli` (known security risk): `spx admin join` contacts the formation server with `InsecureSkipVerify`, sends the join token as a bearer credential and receives the cluster CA certificate and key, master key, Viperblock key and shared credentials, so an on-path attacker during formation can capture the token and the cluster root secrets or substitute a CA. This predates the move; lint surfaced it when the tree left `cmd/`. It is accepted for now and carries a line-level gosec annotation stating it is not a false positive. Removal condition: authenticate formation TLS with a pinned key or certificate delivered with the join material, or an out-of-band CA, with tests proving a mismatched server certificate fails before credentials are sent.
 - `domains/eks/cluster` (potential security risk): `ClusterMeta.ControlPlaneTemplate` (`domains/eks/cluster/service_impl.go`) clears the template's credentials but keeps `OIDCPrivateKeyPEM` and the k3s `JoinToken`, so both are persisted in the `eks-account-{acct}` cluster record, which the gateway also reads. Whether that record is encrypted at rest is unconfirmed; if it is not, a reader of the bucket can forge service-account tokens for the cluster's OIDC issuer and join nodes to it. This predates the refactor and is not fixed in it. Removal condition: the key and token are held by a dedicated secret store (a future KMS or Secrets Manager capability is the likely owner) and the cluster record keeps only a reference, with a test proving the persisted record contains neither secret.
-- `operator/cli` (dependency): the command tree still imports domain and service implementations directly (`handlers/{iam,eks}`, `domains/admission/quota`, `gateway`, `domains/ochre`, `vpcd`, the Viperblock adapter and runtime roles). The ADR's `cmd/spinifex` current-to-target map row requires command packages to reach domains through public capabilities; splitting supported use cases into `operator/*` packages is later work.
+- `operator/cli` (dependency): the command tree still imports domain and service implementations directly (`handlers/{iam,eks}`, `domains/admission/quota`, `domains/ochre`, `vpcd`, the Viperblock adapter and runtime roles). The ADR's `cmd/spinifex` current-to-target map row requires command packages to reach domains through public capabilities; splitting supported use cases into `operator/*` packages is later work.
 - `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
-- `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
 - `gateway` (service registration): the root `gateway` dispatch for IAM, STS, EKS, RDS and ECS imports `domains/{iam,sts,eks,rds,ecs}/awsapi`, which only AWSGW may wire. It clears when each service registers through the ADR-0002 ingress registration seam; as with EC2, moving the dispatch must not be done as a package move.
 - `operator/accountlifecycle` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The amended ADR-0001 `accountteardown` row keeps replacing these with domain-owned cleanup capabilities as deferred debt; it was not a relocation prerequisite. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet. It also holds the StageAccount settings-seal reaper, a consumer of `contracts/ec2/v1`'s `ec2.v1.SealAccountSettings` and per-node `ec2.v1.AccountSettingsSealCapability.<node>` subjects, served by `domains/ec2/account` through the daemon; the reaper reads no KV and imports no domain package, reaching both subjects only through NATS and the configured node set `NewClusterEngine` already receives.
 - `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `domains/elbv2/awsapi` and `domains/rds/instance` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
-- `domains/ec2/awsapi/instance` (dependency): imports `domains/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
-- `domains/ec2/awsapi/spotinstance` (dependency): imports `domains/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
+- `domains/ec2/awsapi/instance` (dependency): imports `domains/iam` for the `InstanceProfile` type that `InstanceProfiles.ResolveInstanceProfile` returns (`RunInstances.go`, `IamInstanceProfileAssociation.go`), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge is now the two-method `InstanceProfiles` interface plus that type; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
 - Instance adapter consumers outside EC2: `domains/rds/awsapi/enginecatalog.go`, `runtime/daemon/eks_cp_control.go`, `runtime/daemon/rds_deps.go`, `domains/admission/quota`'s `vcpu.go` and `gateway/iam.go` (`CountInstanceProfileAssociations`) call `domains/ec2/awsapi/instance` directly; each needs an EC2 capability rather than the AWS action adapter.
 - `runtime/daemon` (Q-28 ownership and deployment): the daemon moved whole, keeping its current ownership, deployment and `daemon → gateway → handlers` dependencies. Q-28's write-side consolidation and any daemon decomposition or split remain deferred by ADR-0001 and are not paid by the move.
 
@@ -680,10 +688,10 @@ series above; "needs decision" is open.
 | `handlers/elbv2` | — | Done. The implementation moved to `domains/elbv2` (package `elbv2`) with no API, NATS subject, storage or behaviour change. Q-30's data-plane boundary remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. Its cross-domain, IAM and `agents/elbv2` imports remain dependency-rule debt, with their allowlist entries now sourced from `domains/elbv2`. |
 | `handlers/iam/mock` | — | Done. Moved with the implementation to `domains/iam/mock`. |
 | `gateway` (root) | Generic HTTP route assembly, SigV4/registry authentication, the policy gate, request-audit middleware, request-context keys, and the legacy per-service dispatch switch for every service not yet registered. | Deferred: the ADR-0002 phases above (auth/IAM and request-context boundary design) gate moving the audit, context-key and legacy-dispatch mechanics. |
-| `gateway/bedrock` | — | Done. The whole package moved into `domains/ochre` in `spinifex#1266` with its package-private core and tests intact and no behaviour change. Separating the action adapters into `domains/ochre/awsapi` remains deferred, recorded as `domains/ochre` (action adapters) debt above. |
+| `gateway/bedrock` | — | Done. The whole package moved into `domains/ochre` in `spinifex#1266` with its package-private core and tests intact and no behaviour change. Separating the action adapters into `domains/ochre/awsapi` remains deferred, recorded as `domains/ochre` (action adapters) debt above. The action adapters then moved to `domains/ochre/awsapi` behind one ADR-0002 registration in `spinifex#1351`. |
 | `gateway/{iam,sts,eks,rds}` | — | Done. The action adapters moved to `domains/{iam,sts,eks,rds}/awsapi` in `spinifex#1265` with no API, NATS subject, storage or behaviour change. ADR-0001 does not make ADR-0002 registration a prerequisite for relocating a domain's adapters, as the EC2 adapter moves already showed; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
 | `gateway/{ecs,elbv2}` | — | Done. The action adapters moved to `domains/{ecs,elbv2}/awsapi` in `spinifex#1273` with no API, NATS subject, storage or behaviour change; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
-| `gateway/spx` | — | Done. The gather helpers, storage-status config and image-promote request moved to `operator/admin` with no behaviour change and the package is removed. The `spinifex` Query dispatch that calls them now lives in `operator/admin` as a seam registration; `/admin/{method}` stays in the root `gateway` until the actor carries `UserName`. |
+| `gateway/spx` | — | Done. The gather helpers, storage-status config and image-promote request moved to `operator/admin` with no behaviour change and the package is removed. The `spinifex` Query dispatch and the `/admin/{method}` handlers that call them now live in `operator/admin` as one seam registration. |
 | `gateway/tagging` | — | Done. `domains/tagging` owns the read-only cross-resource `GetResources` aggregation and its API semantics (`0d830e901`, `aedd0b2c4`); AWSGW composes the owner listers, the EC2 tag index and ELBv2 (`f3545e3ea`); the dispatch moved to `domains/tagging/awsapi` and the root `gateway` no longer serves Tagging. The package is removed, and its two allowlist rows and frozen-legacy entry are gone. Owner tag mutation capabilities remain future direction, so the RDS and ECS tag-writing edges above stay open. |
 | `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the read-only `dirty` reader), the shared vocabulary is `contracts/viperblockd/legacy/v1`, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
 | `utils` | — | Done and deleted. `ec2_subject.go` went to `contracts/ec2/v1`, the vpcd protocol to `contracts/network/v1` with the typed client in `domains/network/projection`, and the master-key loader to `domains/ec2/ebs/encryption`. |
@@ -703,7 +711,7 @@ ADR-0001 completion is structural, so none of these is cleared by remediating se
 | Class | Paths | Status |
 |---|---|---|
 | No missing boundary (candidate moves) | `handlers/ecs/bus` → `contracts/ecs/v1` (`a130f6b8e`); `admin/images_promote.go` → `domains/ec2/image` (`263c58167`); `admin` bootstrap account identifiers → `foundation/aws/identifiers` (`3fdaa54d9`); `admin` credential-format generators → `foundation/aws/credentials` `0368e861a`, retiring `admin` | Done. |
-| One narrow missing boundary | Done: `handlers/sts` EKS JWKS lookup → `handlers/eks` `ClusterJWKSResolver` (`254d898a3`); `gateway/tagging` → `domains/tagging` with AWSGW-composed listers (`0d830e901`, `f3545e3ea`, `aedd0b2c4`); `gateway/spx` → `operator/admin`. Open: the owner of the `spinifex` service and `/admin` dispatch registration | Each needs one decision or one capability, then a mechanical move. |
+| One narrow missing boundary | Done: `handlers/sts` EKS JWKS lookup → `handlers/eks` `ClusterJWKSResolver` (`254d898a3`); `gateway/tagging` → `domains/tagging` with AWSGW-composed listers (`0d830e901`, `f3545e3ea`, `aedd0b2c4`); `gateway/spx` → `operator/admin`. Done: the `spinifex` service and `/admin` dispatch registration, owned by `operator/admin` | Each needs one decision or one capability, then a mechanical move. |
 | Broader boundary or follows another package | `gateway` root (ADR-0002 auth and request-context phase) | Recorded debt with removal conditions above. |
 
 Per-consumer IAM capabilities are the widest unblocker: they gate STS's IAM reads, Ochre's IAM reads, the ELBv2 role path and the ECS, EKS and RDS ensurers.

docs/admin/host-firewall/README.md

@@ -155,6 +155,8 @@ Restarting the daemon is what re-arms it: the node rebuilds its peer list from t
 
 `disable` removes Spinifex's own table and the peer file and leaves every other nftables table alone. It is idempotent, and it is the recovery path — it runs even on a node with no policy installed, so it cannot leave you with a node you can neither arm nor disarm.
 
+`disable` leaves `table inet spinifex_conntrack` loaded, and the boot-time unit loads it on every node whether or not the policy is armed. It accepts everything and exists only to keep connection tracking running. Without it, a connection opened while the policy is off, such as an OVN RAFT link formed during that window, is first seen mid-stream when the policy comes back, and its replies are dropped as new connections. Do not delete it.
+
 > [!WARNING]
 > Do not flush the whole nftables ruleset to clear the policy. The VPC daemon writes SNAT and per-elastic-IP forward rules into the `ip nat` and `ip filter` tables and reinstalls them only when it starts, so `nft flush ruleset` breaks elastic IPs silently until the next `systemctl restart spinifex-vpcd`. Use `disable`, which only ever touches `inet spinifex_filter`.
 > On Oracle Linux with IPsec on, a flush also removes `inet spinifex_ipsec_guard`, so Geneve to a peer without an SA goes out in clear until the daemon puts the table back, within 60 seconds.
@@ -171,6 +173,7 @@ Everything lives under `/etc/spinifex/firewall/`. Which files are safe to edit f
 | `open-ports.nft` | The apply helper — the formation window only | No. Reset closed on install and upgrade |
 | `custom.nft` | **You** — `trusted_ssh_peers` and anything else you add | **Yes.** Created once, never rewritten |
 | `mode` | `setup.sh` — whether the install path armed the policy | No |
+| `conntrack.nft` | `setup.sh` — the tracking table, loaded armed or not | No. Edits are overwritten |
 
 The daemon cannot write `/etc` itself, and a blanket `nft` grant would be root-equivalent, so changes go through `/usr/local/lib/spinifex/spinifex-firewall-apply` — a root helper that accepts a fixed set of verbs and re-validates every address handed to it.
 
@@ -190,10 +193,10 @@ Dropped packets are logged, rate-limited to five a minute, so this tells you whe
 sudo journalctl -k | grep 'spinifex-fw drop'
 ```
 
-An empty log with a connection still failing means the policy is not the cause. A node with no table loaded at all is not armed:
+An empty log with a connection still failing means the policy is not the cause. A node with no peer file is not armed, and its boot-time unit loads only the tracking table:
 
 ```bash
-systemctl is-enabled spinifex-firewall.service
+sudo test -e /etc/spinifex/firewall/peers.nft && echo armed || echo not armed
 ```
 
 ## Troubleshooting

docs/admin/setting-up-your-cluster/README.md

@@ -106,7 +106,7 @@ done
 | `spinifex-rds-postgres` | RDS, `--engine postgres`         | Alpine 3.24.1 + PostgreSQL 18 + `rds-init`      |
 | `spinifex-rds-mariadb`  | RDS, `--engine mariadb`          | Alpine 3.24.1 + MariaDB 11.8 + `rds-init`       |
 | `spinifex-ecs-node`     | ECS container instances          | Alpine 3.21.7 + containerd + ecs-agent          |
-| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
+| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook + eks-etcd-member |
 
 All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
 

docs/compute/launching-instances/README.md

@@ -150,13 +150,17 @@ aws ec2 describe-instance-types \
   --query "InstanceTypes[?InstanceType=='t3.small']"
 ```
 
-Show capacity (how many of each type can still be launched):
+List only the types that can still be launched:
 
 ```bash
 aws ec2 describe-instance-types \
   --filters Name=capacity,Values=true
 ```
 
+Each type appears once, whatever its free capacity.
+This filter is a Spinifex extension and AWS does not declare it, so it is ignored by a region that does not implement it.
+AWS reports exhaustion at launch instead, with `InsufficientInstanceCapacity`, and so does Spinifex.
+
 ## SSH
 
 To SSH into instances via their public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).

docs/containers/ecr/README.md

@@ -46,7 +46,7 @@ Each repository holds image manifests and their layers. Images are stored in obj
 - Repository lifecycle — `CreateRepository`, `DeleteRepository`, `DescribeRepositories`.
 - OCI push and pull — blobs, manifests, and tags, including cross-repository blob mounts.
 - OCI 1.1 referrers — the referrers API, so cosign v3 signatures and other artifacts attached to an image can be listed (see [OCI referrers and image signing](#oci-referrers-and-image-signing)).
-- Token auth — `GetAuthorizationToken` plus bearer/basic auth on `/v2/`.
+- Token auth — `GetAuthorizationToken` plus bearer/basic auth on `/v2/`. A token is valid for 12 hours. One minted by an assumed-role session (an EKS node, for example) stays valid after that session expires and is checked against the live role on every request, so it stops working when the role is deleted or loses its ECR permissions.
 - Repository policies — set, get, and delete a repository policy document.
 
 **Current limitations**

docs/containers/eks/README.md

@@ -35,6 +35,8 @@ resources:
 
 An EKS cluster on Spinifex has two parts: a **control plane** (the managed Kubernetes API server, provisioned on a Spinifex-managed VM and fronted by a network load balancer) and one or more **managed node groups** of worker VMs that run your pods. You provide the VPC, subnets, and IAM roles; Spinifex creates and reconciles the cluster, its load balancer, and its security groups.
 
+Control-plane hosts are not Kubernetes Nodes, as on AWS: `kubectl get nodes` lists only your node group workers, so a cluster with no node group lists none, and no pod runs on a control-plane host. Add-ons that used to tolerate the control-plane `CriticalAddonsOnly` taint, such as CoreDNS, metrics-server and konnectivity-agent, run on workers, so a Ready worker is required before they become available.
+
 Authentication uses **EKS access entries** (the API authentication mode) — IAM principals are mapped to Kubernetes RBAC groups, and `kubectl` authenticates with `aws eks get-token` exactly as it does on AWS.
 
 <p align="center">
@@ -58,6 +60,10 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
+- **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
+  This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
+  A node group launch template that sets `metadata_options` overrides each field it sets, and any field it leaves unset keeps the default.
+  Existing workers keep their hop limit until they are replaced or changed with `ModifyInstanceMetadataOptions`.
 - **Default Kubernetes version is `1.32`.**
 
 ### Control plane readiness and kubectl authentication
@@ -168,6 +174,8 @@ aws eks update-kubeconfig --name demo
 kubectl get nodes
 ```
 
+The list shows the node group workers only. The control-plane VMs never appear as Nodes.
+
 ### 4. Expose a workload
 
 Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
@@ -280,6 +288,12 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 :::
 
+### Upgrade notes
+
+**Control-plane hosts no longer register as Nodes.** `kubectl get nodes` shows workers only, so node counts drop by the number of control planes, for example 6 to 3 on a three-control-plane cluster with three workers. Add-ons that tolerated `CriticalAddonsOnly` now run on workers.
+
+**Existing clusters keep the old behaviour.** A cluster created before this change keeps registering its control planes as tainted Nodes until it is recreated. Recreate the cluster to adopt the new behaviour: automatic control-plane replacement does not currently work when the replacement reuses the failed member's IP, so it is not a supported way to upgrade.
+
 ## Troubleshooting
 
 **Cluster stuck in `CREATING`.** The control plane is still booting. Check `aws eks describe-cluster --name <name> --query 'cluster.health'` for reported issues. The control plane needs egress to bootstrap, so confirm its subnet has an Internet Gateway route.

docs/security/network-connections/README.md

@@ -237,6 +237,7 @@ Notes that make the difference between this working and locking you out:
 - **Use a dedicated table and never flush the others.** `vpcd` writes MASQUERADE and per-EIP FORWARD rules into the `ip filter` and `ip nat` tables, and only reinstalls them when the service starts. `iptables -F`, `nft flush ruleset`, `ufw enable` and `firewalld` all destroy them silently, and instance networking stays broken until the next restart.
 - **Do not add a `forward` hook.** nftables evaluates every table registered on a hook and any `drop` is final, so a default-deny forward chain here would override `vpcd`'s accepts in the other table and break every routed-NAT instance and every EIP. Filtering forwarded guest traffic is the security group's job, not the host firewall's.
 - **`vpcd` owns a second table named `spinifex_mss`, separate from `inet spinifex_filter`.** It is a `bridge` table on the veth uplink and an `inet` table on the routed uplink, it has a `forward` hook with policy `accept`, and it only rewrites the MSS option of TCP SYN packets on the host hop to the gateway. It never drops or accepts traffic and does not replace the forward-hook advice above. `vpcd` re-installs it every 30 seconds, so a flush is repaired within that interval, and there is none on a physical uplink.
+- **Keep connection tracking running while the policy is unloaded.** The kernel tracks connections only while some rule uses conntrack, so a node with no ruleset tracks nothing. A connection that opens then, such as an OVN RAFT link, is first seen mid-stream when the policy loads, and if the server's end speaks first its packet looks like a new connection to an ephemeral port and is dropped. Spinifex keeps `table inet spinifex_conntrack`, which accepts everything, loaded on every node for this reason.
 - **`output` is deliberately untouched.** The metadata service's reply path egresses through per-instance policy routing; filtering `output` breaks it in ways that are hard to attribute.
 - **On a single-NIC node the cluster and encap sets are the node's public addresses**, because the planes collapse. The rules are still correct, but they are no longer a boundary — an upstream ACL is the only real control there.
 

docs/terraform-workbooks/bastion-private-subnet/README.md

@@ -69,8 +69,8 @@ Clone the Terraform examples from the Spinifex repository:
 ```bash
 git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
 cd spinifex-tf
-git sparse-checkout set docs/terraform
-cd docs/terraform/bastion-private-subnet
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/bastion-private-subnet
 ```
 
 Or create a `main.tf` file and paste the full configuration below.

docs/terraform-workbooks/eks-quickstart/README.md

@@ -153,6 +153,8 @@ kubectl get nodes
 kubectl get pods -o wide
 ```
 
+`kubectl get nodes` lists the worker nodes only, because control-plane hosts are not Kubernetes Nodes.
+
 `kubectl get pods -o wide` shows the demo pods and which node each landed on.
 
 ### Cleanup

docs/terraform-workbooks/nginx-alb/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Nginx Web Server (Load Balanced)"
 seoTitle: "Load-Balanced Nginx Behind an AWS ALB — Spinifex Docs"
-description: "Deploy a VPC with two private EC2 instances running Nginx behind an internet-facing Application Load Balancer on Spinifex, using Terraform or OpenTofu."
+description: "Deploy Nginx EC2 instances behind an internet-facing Application Load Balancer on Spinifex with Terraform, then spread them across nodes and test failover."
 category: "Terraform Workbooks"
 tags:
   - terraform
@@ -10,6 +10,8 @@ tags:
   - elbv2
   - alb
   - vpc
+  - placement-group
+  - resiliency
   - workbook
 resources:
   - title: "Terraform AWS Provider"
@@ -22,7 +24,7 @@ resources:
 
 # Terraform: Nginx Web Servers with ALB
 
-> Deploys two Nginx web server instances behind an Application Load Balancer on Spinifex.
+> Deploys Nginx web server instances behind an Application Load Balancer on Spinifex, optionally spread one per node so the site survives losing a node.
 
 ## Table of Contents
 
@@ -34,18 +36,25 @@ resources:
 
 ## Overview
 
-Deploy two Nginx web servers behind an internet-facing Application Load Balancer on Spinifex using Terraform/OpenTofu. This workbook provisions a VPC with public and private subnets, an internet gateway and NAT Gateway, route tables, security group, SSH key pair, an application load balancer (ALB) and two EC2 instances with cloud-init user-data that installs and starts Nginx. Only the ALB is reachable from outside the VPC — the Nginx instances live in the **private subnets** and reach the internet only for cloud-init bootstrapping via the NAT Gateway.
+Deploy Nginx web servers behind an internet-facing Application Load Balancer on Spinifex using Terraform or OpenTofu.
+This workbook provisions a VPC with two public subnets, an internet gateway, a route table, a security group, an SSH key pair, an Application Load Balancer (ALB) and two or more EC2 instances with cloud-init user-data that installs and starts Nginx.
+The web servers get public IPs so cloud-init can install Nginx, and the ALB reaches them over the VPC by their private addresses.
+
+On a cluster of three or more nodes, a spread placement group puts each web server on a different node.
+Losing a node then takes out at most one server, the ALB stops sending it traffic, and Spinifex restarts the lost instances on the surviving nodes.
+[Step 6](#step-6-test-resiliency) shows how to watch that happen.
 
 <p align="center">
-  <img src="../../../.github/assets/diagrams/tf-nginx-alb.svg" alt="Nginx + ALB VPC — IGW, two public subnets with ALB ENIs and NAT GW, two private subnets hosting nginx" width="900">
+  <img src="../../../.github/assets/diagrams/tf-nginx-alb.svg" alt="Nginx + ALB VPC — IGW, two public subnets with the ALB and the nginx web servers" width="900">
 </p>
 
 **What you'll learn:**
 
 - Configuring the AWS Terraform provider to target Spinifex
-- Creating a VPC with public + private subnets, an IGW and a NAT Gateway
-- Provisioning a multi-AZ internet-facing ALB fronting private workers
-- Provisioning an EC2 instance with cloud-init user-data
+- Creating a VPC with public subnets and an internet gateway
+- Provisioning an internet-facing ALB in front of a group of web servers
+- Provisioning EC2 instances with cloud-init user-data
+- Spreading instances across nodes with a placement group
 - Generating SSH key pairs with the TLS provider
 
 **What gets created**
@@ -53,22 +62,33 @@ Deploy two Nginx web servers behind an internet-facing Application Load Balancer
 | Resource | Name | Purpose |
 |---|---|---|
 | VPC | `nginx-alb-vpc` | Isolated network (10.20.0.0/16) |
-| Public Subnets | `nginx-alb-public-a`, `nginx-alb-public-b` | Two AZs hosting the ALB and NAT Gateway |
-| Private Subnets | `nginx-alb-private-a`, `nginx-alb-private-b` | Two AZs hosting the Nginx workers |
+| Public Subnets | `nginx-alb-public-a`, `nginx-alb-public-b` | Host the ALB and the web servers |
 | Internet Gateway | `nginx-alb-igw` | Routes internet traffic for the public subnets |
-| Elastic IP | `nginx-alb-nat-eip` | Public address for the NAT Gateway |
-| NAT Gateway | `nginx-alb-nat` | Outbound internet for the private subnets (cloud-init apt bootstrap) |
 | Security Group | `nginx-alb-sg` | Allows SSH (22) and HTTP (80) inbound |
-| EC2 Instances | `nginx-alb-1`, `nginx-alb-2` | Ubuntu 26.04 with Nginx via cloud-init (private subnets) |
+| Placement Group | `nginx-alb-spread` | Only with `placement_strategy`; spread puts one web server per node |
+| EC2 Instances | `nginx-alb-1`, `nginx-alb-2`, … | Ubuntu with Nginx via cloud-init, one per `server_count` |
 | ALB | `nginx-alb` | Internet-facing Application Load Balancer on port 80 |
-| Target Group | `nginx-alb-tg` | HTTP health-checked group for both instances |
+| Target Group | `nginx-alb-tg` | HTTP health-checked group of every web server |
 | Listener | HTTP :80 | Forwards traffic to the target group |
 
+**Variables**
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `instance_type` | `t3.small` | Instance type of the web servers |
+| `server_count` | `2` | Number of web servers behind the ALB |
+| `placement_strategy` | none | `spread` puts each web server on a different node; `cluster` puts them all on one |
+| `region` | `ap-southeast-2` | Spinifex region |
+| `spinifex_endpoint` | `https://127.0.0.1:9999` | Spinifex AWS endpoint |
+
+A spread placement group on Spinifex spreads across nodes, so `server_count` can be at most the number of nodes in the cluster.
+
 **Prerequisites:**
 
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
-- An Ubuntu 26.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- An Ubuntu 26.04 or 24.04 AMI imported (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
 - OpenTofu or Terraform installed
+- For [Step 6](#step-6-test-resiliency), a cluster of three nodes
 
 ## Instructions
 
@@ -79,20 +99,20 @@ Clone the Terraform examples from the Spinifex repository:
 ```bash
 git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
 cd spinifex-tf
-git sparse-checkout set docs/terraform
-cd docs/terraform/nginx-alb
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/nginx-alb
 ```
 
 Or create a `main.tf` file and paste the full configuration below.
 
 <!-- INCLUDE: main.tf lang:hcl -->
 
-### Step 2. Install Debian AMI
+### Step 2. Import the Ubuntu AMI
 
-Install the Debian 13 AMI which is used in the example to host the `nginx` webservers as an EC2 instance.
+Import the Ubuntu 26.04 AMI the web servers run on, from any node of the cluster:
 
 ```bash
-spx admin images import --name debian-13-x86_64
+spx admin images import --name ubuntu-26.04-x86_64
 ```
 
 > **Note:** The load balancer itself runs as a direct-boot QEMU microvm using the kernel + initramfs bundled with the Spinifex distribution (`/usr/share/spinifex/microvm/`), so no separate LB AMI import is required.
@@ -133,9 +153,16 @@ Next, apply and launch the template:
 tofu apply
 ```
 
+On a three-node cluster, launch three web servers instead, one on each node:
+
+```bash
+tofu apply -var server_count=3 -var placement_strategy=spread
+```
+
 ### Step 5. Verify
 
-> **Note:** EC2 instances can take 30+ seconds to boot after apply, and the NAT Gateway must be `available` before cloud-init on the workers can reach the apt repository. If the ALB returns 5xx or HTTP is unreachable, wait and retry — the target group health checks need a moment to mark both instances healthy once Nginx has installed.
+> **Note:** EC2 instances can take 30+ seconds to boot after apply, and Nginx installs after that.
+> If the ALB returns 5xx or HTTP is unreachable, wait and retry, since the target group health checks need a moment to mark the instances healthy once Nginx has installed.
 
 The ALB is internet-facing, but the DNS name Spinifex returns (`*.elb.spinifex.local`) will not resolve from your host. Fetch the ALB's public IP with the AWS CLI:
 
@@ -145,7 +172,7 @@ ALB_IP=$(aws elbv2 describe-load-balancers --names nginx-alb \
   --output text)
 ```
 
-Then hit the ALB — successive requests should alternate between Server 1 and Server 2:
+Then hit the ALB, and successive requests rotate between the servers:
 
 ```bash
 curl http://$ALB_IP
@@ -154,7 +181,8 @@ curl http://$ALB_IP
 
 Open `http://$ALB_IP` in your browser and refresh to see the page alternate content served from each instance.
 
-The Nginx instances themselves only have private IPs (see the `instance_1_private_ip` / `instance_2_private_ip` outputs) and are only reachable from inside the VPC — go through the ALB.
+Each web server also has a public IP of its own, listed by `tofu output instance_public_ips`.
+The ALB reaches them by their private IPs, listed by `tofu output instance_private_ips`.
 
 Check target health via AWS CLI:
 
@@ -166,12 +194,40 @@ TG_ARN=$(aws elbv2 describe-target-groups \
 aws elbv2 describe-target-health --target-group-arn $TG_ARN
 ```
 
+### Step 6. Test resiliency
+
+On a cluster of three nodes, apply with `-var server_count=3 -var placement_strategy=spread`, then check that each web server is on its own node:
+
+```bash
+spx get vms
+```
+
+The `NODE` column names each node once for the three `t3.small` web servers, and the load balancer is the `sys.micro` instance.
+
+Request the page once a second in a second terminal:
+
+```bash
+while :; do
+  printf '%s %s\n' "$(date +%T)" "$(curl -s -m2 "http://$ALB_IP/" | grep -o 'Server [0-9]' || echo FAILED)"
+  sleep 1
+done
+```
+
+Then power off one node, the way a hardware failure would, without shutting it down first.
+On Vultr, `deploy-vultr.sh stop node3` does this, as described in [Running Spinifex on Vultr](/docs/vultr-integration#8-test-resiliency).
+
+Within about a minute the load balancer stops sending requests to the lost web server, and the other two answer every request.
+About three minutes after the power-off, Spinifex has restarted the lost web server on a surviving node with the same instance ID, private IP and public IP, and the load balancer sends it requests again once its health checks pass.
+Power the node back on and wait until `spx get nodes` shows it `Ready` before you test another node.
+
 ### Cleanup
 
 ```bash
 tofu destroy
 ```
 
+If you applied with `-var` options, pass the same ones to `tofu destroy`.
+
 ## Troubleshooting
 
 ### AMI Not Found
@@ -213,18 +269,19 @@ If targets stay unhealthy, verify the instances are running:
 aws ec2 describe-instances --profile spinifex
 ```
 
-If cloud-init on the workers never finished, confirm the NAT Gateway is `available` (the private subnets rely on it for outbound apt access):
-
-```bash
-aws ec2 describe-nat-gateways --query 'NatGateways[].[NatGatewayId,State]'
-```
+If cloud-init on the workers never finished, SSH to one by its public IP and read its log, as described in [Nginx Not Responding](#nginx-not-responding).
 
 ### Nginx Not Responding
 
-The Nginx instances have no public IP, so you can't SSH in directly from your host. If you need to inspect cloud-init logs, launch a small jump host in the same VPC or run commands via the Spinifex console, then:
+SSH to the web server by its public IP, from `tofu output instance_public_ips`, and check cloud-init and Nginx:
 
 ```bash
-ssh -i nginx-alb-demo.pem ubuntu@<instance_private_ip>
+ssh -i nginx-alb-demo.pem ubuntu@<instance_public_ip>
 sudo journalctl -u cloud-init --no-pager
 sudo systemctl status nginx
 ```
+
+### Spread Launch Fails With InsufficientInstanceCapacity
+
+A spread placement group puts at most one of its instances on each node.
+Set `server_count` no higher than the number of nodes, or leave `placement_strategy` empty.

docs/terraform-workbooks/nginx-alb/main.tf

@@ -12,10 +12,16 @@
 #
 # Demonstrates: VPC, public subnets, internet gateway, route tables,
 # security group, key pair, cloud-init user-data, EC2 instances with
-# auto-assigned public IPs, ALB, target group, and listener.
+# auto-assigned public IPs, an optional placement group, ALB, target group,
+# and listener.
+#
+# server_count sets how many web servers launch (default 2). Setting
+# placement_strategy = "spread" puts each one on a different Spinifex node,
+# so a three-node cluster with server_count = 3 can lose any node and keep
+# serving through the ALB.
 #
 # Usage:
-#   cd spinifex/docs/terraform/nginx-alb
+#   cd spinifex/docs/terraform-workbooks/nginx-alb
 #   export AWS_PROFILE=spinifex
 #   tofu init && tofu apply
 #
@@ -28,6 +34,9 @@
 #
 # Then:
 #   curl http://<alb_public_ip>    # Load-balanced Nginx (alternates between instances)
+#
+# Three servers, one per node:
+#   tofu apply -var server_count=3 -var placement_strategy=spread
 
 terraform {
   required_version = ">= 1.6.0"
@@ -62,6 +71,28 @@ variable "instance_type" {
   default = "t3.small"
 }
 
+variable "server_count" {
+  type        = number
+  default     = 2
+  description = "Number of Nginx web servers behind the ALB"
+
+  validation {
+    condition     = var.server_count >= 1
+    error_message = "server_count must be at least 1."
+  }
+}
+
+variable "placement_strategy" {
+  type        = string
+  default     = ""
+  description = "Placement group strategy for the web servers: spread (one per node), cluster (all on one node), or empty for none"
+
+  validation {
+    condition     = contains(["", "spread", "cluster"], var.placement_strategy)
+    error_message = "placement_strategy must be spread, cluster, or empty."
+  }
+}
+
 variable "spinifex_endpoint" {
   type        = string
   default     = "https://127.0.0.1:9999"
@@ -241,14 +272,28 @@ resource "aws_security_group" "web" {
 }
 
 # ---------------------------------------------------------------------------
-# EC2 Instances — two Nginx servers with distinct landing pages
+# Placement Group — optional; spread puts each web server on its own node
 # ---------------------------------------------------------------------------
 
-resource "aws_instance" "nginx_1" {
-  ami           = data.aws_ami.ubuntu.id
-  instance_type = var.instance_type
+resource "aws_placement_group" "web" {
+  count    = var.placement_strategy == "" ? 0 : 1
+  name     = "nginx-alb-${var.placement_strategy}"
+  strategy = var.placement_strategy
+}
 
-  subnet_id              = aws_subnet.public_a.id
+# ---------------------------------------------------------------------------
+# EC2 Instances — Nginx servers with distinct landing pages, alternating
+# between the two public subnets
+# ---------------------------------------------------------------------------
+
+resource "aws_instance" "nginx" {
+  count = var.server_count
+
+  ami             = data.aws_ami.ubuntu.id
+  instance_type   = var.instance_type
+  placement_group = one(aws_placement_group.web[*].name)
+
+  subnet_id              = count.index % 2 == 0 ? aws_subnet.public_a.id : aws_subnet.public_b.id
   vpc_security_group_ids = [aws_security_group.web.id]
   key_name               = aws_key_pair.nginx.key_name
 
@@ -266,7 +311,7 @@ resource "aws_instance" "nginx_1" {
     <head><title>Spinifex ALB Demo</title></head>
     <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
       <h1>Hello from Spinifex!</h1>
-      <p><strong>Instance:</strong> $INSTANCE_ID (Server 1)</p>
+      <p><strong>Instance:</strong> $INSTANCE_ID (Server ${count.index + 1})</p>
       <p>This Nginx server is behind an Application Load Balancer.</p>
       <hr>
       <p><small>Provisioned via cloud-init user-data.</small></p>
@@ -280,48 +325,19 @@ resource "aws_instance" "nginx_1" {
   )
 
   tags = {
-    Name = "nginx-alb-1"
+    Name = "nginx-alb-${count.index + 1}"
   }
 }
 
-resource "aws_instance" "nginx_2" {
-  ami           = data.aws_ami.ubuntu.id
-  instance_type = var.instance_type
+# Earlier versions declared the two servers separately.
+moved {
+  from = aws_instance.nginx_1
+  to   = aws_instance.nginx[0]
+}
 
-  subnet_id              = aws_subnet.public_b.id
-  vpc_security_group_ids = [aws_security_group.web.id]
-  key_name               = aws_key_pair.nginx.key_name
-
-  user_data_base64 = base64encode(<<-USERDATA
-    #!/bin/bash
-    set -euo pipefail
-
-    apt-get update -y
-    apt-get install -y nginx
-
-    INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id 2>/dev/null || hostname)
-    cat > /var/www/html/index.html <<HTML
-    <!DOCTYPE html>
-    <html>
-    <head><title>Spinifex ALB Demo</title></head>
-    <body style="font-family: sans-serif; max-width: 600px; margin: 80px auto;">
-      <h1>Hello from Spinifex!</h1>
-      <p><strong>Instance:</strong> $INSTANCE_ID (Server 2)</p>
-      <p>This Nginx server is behind an Application Load Balancer.</p>
-      <hr>
-      <p><small>Provisioned via cloud-init user-data.</small></p>
-    </body>
-    </html>
-    HTML
-
-    systemctl enable nginx
-    systemctl restart nginx
-  USERDATA
-  )
-
-  tags = {
-    Name = "nginx-alb-2"
-  }
+moved {
+  from = aws_instance.nginx_2
+  to   = aws_instance.nginx[1]
 }
 
 # ---------------------------------------------------------------------------
@@ -365,19 +381,25 @@ resource "aws_lb_target_group" "nginx" {
 }
 
 # ---------------------------------------------------------------------------
-# Register both instances as targets
+# Register every web server as a target
 # ---------------------------------------------------------------------------
 
-resource "aws_lb_target_group_attachment" "nginx_1" {
+resource "aws_lb_target_group_attachment" "nginx" {
+  count = var.server_count
+
   target_group_arn = aws_lb_target_group.nginx.arn
-  target_id        = aws_instance.nginx_1.id
+  target_id        = aws_instance.nginx[count.index].id
   port             = 80
 }
 
-resource "aws_lb_target_group_attachment" "nginx_2" {
-  target_group_arn = aws_lb_target_group.nginx.arn
-  target_id        = aws_instance.nginx_2.id
-  port             = 80
+moved {
+  from = aws_lb_target_group_attachment.nginx_1
+  to   = aws_lb_target_group_attachment.nginx[0]
+}
+
+moved {
+  from = aws_lb_target_group_attachment.nginx_2
+  to   = aws_lb_target_group_attachment.nginx[1]
 }
 
 # ---------------------------------------------------------------------------
@@ -401,12 +423,11 @@ resource "aws_lb_listener" "http" {
 
 output "note" {
   value = <<-EOT
-    EC2 instances can take 30+ seconds to boot after apply — if HTTP is
-    unreachable, wait and retry.
+    EC2 instances can take 30+ seconds to boot after apply, and Nginx installs
+    after that — if HTTP is unreachable, wait and retry.
 
-    The Nginx instances have private IPs only. The ALB DNS name ends in
-    .elb.spinifex.local and will not resolve from your host, so fetch the
-    ALB's public IP with:
+    The ALB DNS name ends in .elb.spinifex.local and will not resolve from
+    your host, so fetch the ALB's public IP with:
 
       aws elbv2 describe-load-balancers --names nginx-alb \
         --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
@@ -428,18 +449,18 @@ output "alb_dns_name" {
   value = aws_lb.web.dns_name
 }
 
-output "instance_1_id" {
-  value = aws_instance.nginx_1.id
+output "instance_ids" {
+  value = aws_instance.nginx[*].id
 }
 
-output "instance_1_private_ip" {
-  value = aws_instance.nginx_1.private_ip
+output "instance_private_ips" {
+  value = aws_instance.nginx[*].private_ip
 }
 
-output "instance_2_id" {
-  value = aws_instance.nginx_2.id
+output "instance_public_ips" {
+  value = aws_instance.nginx[*].public_ip
 }
 
-output "instance_2_private_ip" {
-  value = aws_instance.nginx_2.private_ip
+output "placement_group" {
+  value = one(aws_placement_group.web[*].name)
 }

docs/terraform-workbooks/nginx-webserver/README.md

@@ -56,8 +56,8 @@ Clone the Terraform examples from the Spinifex repository:
 ```bash
 git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
 cd spinifex-tf
-git sparse-checkout set docs/terraform
-cd docs/terraform/nginx-webserver
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/nginx-webserver
 ```
 
 Or create a `main.tf` file and paste the full configuration below.

docs/terraform-workbooks/s3-webapp/README.md

@@ -74,8 +74,8 @@ Clone the Terraform examples from the Spinifex repository:
 ```bash
 git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
 cd spinifex-tf
-git sparse-checkout set docs/terraform
-cd docs/terraform/s3-webapp
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/s3-webapp
 ```
 
 Or create the files manually and paste the full configuration below.

docs/testing/e2e-coverage.md

@@ -29,6 +29,7 @@ The largest suite, 27 top-level tests. Notable entry points:
 - `TestInstanceMetadata` — IMDS from inside a guest.
 - **`TestIMDSDatapathSurvivesVswitchdRestart`** restarts `ovs-vswitchd` under a guest with an Elastic IP, once through systemd and once with SIGKILL. After each, the guest must answer over its EIP and fetch its instance ID through IMDSv2 within 60 seconds, and `br-imds` must hold all 7 of the guest's own flows. It is sequential because it takes the whole node's datapath down.
 - `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
+- **`TestRebootLoop`** is opt-in and skipped in a whole-suite run. Name it in `-test.run`, or set `SPINIFEX_REBOOT_LOOP_ROUNDS` (default 10), and it reboots one fresh guest through the API that many times and then once from inside, requiring a new boot ID every time.
 - `TestNegativeErrorPaths` — the `8a`–`8t` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, unknown or malformed instance IDs on the lifecycle verbs, and so on).
 - **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across every host and network permutation.
 
@@ -36,6 +37,8 @@ The largest suite, 27 top-level tests. Notable entry points:
 
 Runs **on** a node, shelling out to `ip`/`iptables`/`ovn-nbctl` locally. Skips unless `spinifex.toml` says `external_mode = "nat"`. Nine sequential phases: host wiring (transit veth, `ip_forward`, NAT egress rules), config, the EIP surface (enabled or disabled depending on whether a public pool exists), default-subnet public IP mapping, OVN gateway and SNAT on the transit net, instance egress **proven via the serial console**, host-to-guest Tier 1 ingress, a second VPC getting a unique transit gateway IP, and the EIP ingress lifecycle.
 
+**The EIP ingress phase also proves hairpin traffic.** On a single node a second guest must complete a TCP handshake to the first guest's EIP on port 22, as an AWS instance reaches a public address in its own VPC. Without the `SPX-EIP-HAIRPIN` SNAT the packet returns to OVN from the transit range, which is exempt from EIP DNAT, and is never answered; found when an EKS worker on Vultr could not reach its control plane's endpoint.
+
 **Phase 9 has two shapes.** On one node it is the EIP lifecycle: allocate, associate, host delivery plumbing, a TCP handshake to the address, a vpcd restart that must replay the plumbing, and disassociate tearing it down. On a cluster it is the distributed lane instead — a guest on every node, each public IP plumbed by the node running its guest and by **no other**, every transit veth carrying the one cluster-wide MAC, each `dnat_and_snat` row carrying `external_mac` and `logical_port`, inbound proven by a handshake and outbound by each guest's own console. **That lane is the regression guard for the two defects that made routed NAT work on one node only**, and neither is visible to a single-node run.
 
 Phases 7 and 8 are gated on a cluster by what the datapath can actually do: the transit segment is a localnet, so host-to-guest Tier 1 ingress only exists where the VPC's gateway router port and the guest are both on this host. The VPC ingress route itself is installed on every node, so phase 8 checks it unconditionally.
@@ -79,6 +82,8 @@ Three entry points, in the order they run:
 
 **Asserted from both sides, and the pair is the point.** `InstanceAutoRecovery` takes a node down and requires the guest to come back on exactly one survivor, identified by the qemu process rather than by what the API says, then brings the node back and requires it to drop the local copy instead of relaunching it. `InstanceRecoveryRefusesStorageFault` freezes predastore cluster-wide first, so the guest is paused by `werror=stop` when its node goes away, and requires that **nothing moves** — a store that refuses every node cannot be fixed by moving a guest, and the relaunch would cost it the request QEMU is holding. A reconciler that cannot tell a host failure from a storage failure passes the first and fails the second. Neither turns recovery on, because there is no setting for it: both wait for every node to report the reconciler active on a cluster nothing configured, so a build that reintroduced a switch fails here.
 
+**`InstanceRecoveryKeepsAnOlderInstanceAndItsDisk` asserts the guest survives with its data, which the first test never did.** Its guest writes a token to its root volume and prints it from the volume at every boot, and it is left running past the pending watchdog's timeout before its node is taken away, because a real instance is always older than a minute. After the survivor relaunches it, the instance must stay running for three watchdog ticks with its root volume present, and the new console must print the token, so the guest booted from the disk it wrote. The node then returns, and the instance is stopped and started once its last launch is older than the timeout again, with the same checks and a later `LaunchTime`. Before ADR-0003:S7, both relaunches carried the original launch time, and the watchdog terminated the instance seconds after QEMU started and deleted its root volume.
+
 **`InstancePartitionRecovery` is the fault this feature exists for, and the only one of the three that produces it rather than approximating it.** It drops NATS on 4222 and 4248 between one node and its peers and nothing else, so that node keeps its services, keeps its QEMU and keeps answering its own local clients while it cannot reach consensus — half healthy, which is what a real network fault looks like and what a service stop cannot imitate. It then requires that the node gives up the volume on a clock it can evaluate without reaching anything, that a survivor takes the guest over, and that the rejoining node forgets its copy rather than relaunching it. **The assertion it exists for is the one no single sample can reach:** a watch samples every node every three seconds for the whole run and fails on any sample naming two hosts, because two guests writing one volume through one object store is data loss already under way rather than a race that resolves. The evidence for *why* the guest stopped is the surrender and the fence in the node's own journal — a guest killed for any other reason satisfies the process count and means the opposite.
 
 **The partition gets its own nft table and a systemd deadman timer, for reasons worth keeping.** The node's real firewall is `inet spinifex_filter`, so a test that flushed or edited it would leave the node unfirewalled with no sign of which rules it lost; a separate table at priority -300 says exactly "these packets, dropped" and heals in one atomic delete. `systemd-run --on-active` removes it independently, so a cancelled workflow or a panicking test does not leave a node cut off from its cluster until somebody notices. It also proves SSH still works and proves a peer port is actually unreachable before asserting anything — a rule set that installed cleanly and dropped nothing would otherwise read as the system tolerating a partition.
@@ -120,7 +125,7 @@ An ALB whose targets all fail their health check keeps serving from them instead
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, and an IRSA pod wired with a projected SA token. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 
@@ -131,7 +136,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 - **`storagefault`** — what a guest does when its storage backend stops answering, and what stop/start does when the seal failed. **The claim is about corruption, not availability.** SIGSTOPs a cluster-wide service, so it always runs last and in its own cell. Three-node, because the RS(2,1) shard floor and cross-node takeover after a failed seal do not exist on one node.
 - **`partialblock`** — concurrent sub-block writes to the same block surviving a cold reread through NBD from a real guest.
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
-- **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest reset once its budget expires. Proves a real reboot by requiring a *new* boot ID.
+- **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.
 - **`diskperf`** — guest disk performance gate, pinned fio profile on a dedicated blank volume. Also requires `external_mode=pool` for guest SSH.
 
 ### Other

docs/ui/design-system.md

@@ -0,0 +1,341 @@
+# Spinifex UI design system
+
+The console's tokens, primitives and the rules that hold pages to them.
+The live catalogue is the `/design` route: every token and primitive below rendered in both themes.
+It is served by the Vite dev server only and is absent from release builds; a release build answers it with "Page not found".
+The rules are accepted as ADR-0016, `docs/adr/0016-console-design-system.md` in the mulga repository, and every console change follows them; cite a clause as `ADR-0016:S10`.
+
+Source: `spinifex/runtime/roles/spinifexui/frontend/`.
+Tokens are in `src/styles.css`, primitives in `src/components/ui/`, and the catalogue in `src/routes/_auth/-components/design/`.
+
+## Principles
+
+- **Pages compose primitives; they do not restyle them.** If a page needs something the system lacks, add it to the system first.
+- **Colour is for status.** Surfaces are neutral, and brand orange marks the one primary action on a view.
+- **Borders, not shadows.** Overlays (dialogs, sheets, popovers, menus) are the only elevated surfaces.
+- **AWS semantics, our presentation.** States, field names and actions follow AWS.
+
+## Tokens
+
+All colours are OKLCH and come from mulgadc.com, so product and site match.
+
+### Surfaces
+
+| Token | Use |
+|---|---|
+| `background` | The app page. Warm off-white in light, ink `#14181D` in dark. |
+| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. |
+| `muted`, `secondary`, `accent` | Hover rows, secondary buttons, quiet fills. |
+| `border` | Every divider and card edge. `#DCD8CE` in light, `oklch(1 0 0 / 8%)` in dark. |
+| `brand` | Brand orange `#EE4B12`, for the logo and decoration only. It fails text contrast, so it never carries text. |
+| `primary` | Darker orange in light, brighter orange in dark: primary button, link, focus ring, active tab. |
+| `sidebar` | The navigation column. |
+| `console`, `console-foreground` | A serial console log. Light grey on the light theme and near-black on the dark one, so the log follows the theme. |
+| `console-stamp`, `console-muted`, `console-ok`, `console-fail` | The log's tones: the kernel timestamp (brand yellow in dark, a darker amber in light), the subsystem or device that logged the line, and systemd's `[ OK ]` and `[FAILED]` markers. |
+
+### Status
+
+Each status has a solid token plus `-foreground` (text on a tinted background), `-subtle` (tinted background) and `-border`.
+
+| Status | Hue | AWS states |
+|---|---|---|
+| `success` | green | running, available, completed, healthy |
+| `warning` | brand yellow `#F5B500` | pending, stopping, creating, modifying, deleting, draining |
+| `danger` | cool red, hue 17 | stopped, error, failed, unhealthy, unavailable |
+| `info` | blue | informational notes |
+| `neutral` | grey `#9AA1A8` | terminated (muted text), and any unmapped state |
+
+`stopped` is danger because EC2 reads it that way, although RDS treats it as a resting state; splitting the map by service is its own change.
+
+`StateBadge` (`src/components/state-badge.tsx`) is the only mapping from an AWS state to a tone, and it exposes the result as `data-tone` for tests.
+Every status badge measures above 4.8:1 against its tinted background in both themes.
+
+**Danger is separated from the primary accent by hue alone in dark mode**: the two have the same lightness.
+So a destructive action is never a solid danger fill, and a danger state always carries a text label or an icon.
+
+### Chart palette
+
+`--chart-1` to `--chart-8` are categorical, with lightness spread so adjacent series separate under protanopia, deuteranopia and tritanopia.
+Measured with a simulated CVD ΔE, the worst pair is 7.4 in light and 7.2 in dark.
+Colour is never the only key: label every series.
+
+### Typography
+
+| Role | Font |
+|---|---|
+| Page titles and section headings (`h1`–`h3`) | Archivo (`font-heading`) |
+| UI text, tables and forms | Geist (`font-sans`) |
+| IDs, ARNs, IPs, CIDRs, code and eyebrow labels | JetBrains Mono (`font-mono`), with `tabular-nums` for figures |
+
+| Step | Size | Use |
+|---|---|---|
+| `2xl` | 26px | page title (`PageHeading`) |
+| `xl` | 22px | section heading |
+| `lg` | 17px | |
+| `base` | 15px | |
+| `sm` | 14px | body and table default |
+| `xs` | 13px | badges, secondary text |
+| `2xs` | 12px | mono eyebrow labels |
+
+The scale sits a step above Tailwind's defaults, as consoles such as Vultr's do, because 13px body text read as cramped on a wide screen.
+Code blocks are set at 12px (`--text-code-block`) so a long command still fits.
+
+### Primary and contrast
+
+The brand orange cannot be the primary: white on `#EE4B12` is 3.72:1, and as link text on a light card it is also 3.72:1, against WCAG AA's 4.5:1.
+So `primary` is its own token, and `brand` keeps the exact site colour for decoration.
+
+| Theme | `primary` | Fill text | Fill contrast | As a link |
+|---|---|---|---|---|
+| light | `oklch(55% 0.19 36.6)` | white | 5.30:1 | 5.30:1 on a card, 4.69:1 on the page |
+| dark | `oklch(68% 0.18 38)` | ink | 5.70:1 | 5.27:1 on a card |
+
+Dark mode puts ink on the fill rather than white, because no orange is both bright enough to read as a link on ink and dark enough to carry white text.
+
+### Radius
+
+`--radius` is 0.5rem; `rounded-md` is that, `rounded-sm` and `rounded-lg` step 2px either side, and status pills are `rounded-full`.
+
+### Themes
+
+The console defaults to dark, and the header switch toggles light and dark (`localStorage["spinifex-ui-theme"]`).
+`public/preboot.js` runs before the bundle. It applies the stored theme and `color-scheme` before first paint, so a dark-mode user never sees a white page while the bundle loads, and it turns off zod's eval probe (below).
+It is an external file because the console's CSP forbids inline scripts.
+
+## Constraints from the console's CSP
+
+The Go console serves `script-src 'self'; style-src 'self'`.
+
+- **No inline `<script>` or `<style>` elements.** A library that injects either breaks in production while working under `vite dev`, which applies no CSP. This is why `sonner` is not used.
+- **No runtime CSS variables set through `style`.** Primitives use fixed class maps instead, as `ToggleGroup` does for `spacing`.
+- **Base UI's `ScrollArea` injects a `<style>` element** to hide native scrollbars. `main.tsx` wraps the app in `CSPProvider disableStyleElements`, and `styles.css` carries the rule instead (`.base-ui-disable-scrollbar`).
+- **zod compiles validators with `new Function`** unless it is told not to, and it probes for that at module load, before any app code runs. `public/preboot.js` sets `globalThis.__zod_globalConfig = { jitless: true }` first.
+- **Chrome reports a violation whenever `DOMParser` meets a body that is not XML**, because the `<parsererror>` element it builds carries an inline style. The AWS SDK parses every query-protocol error body, so a proxy that answers with plain text or HTML produces a console violation as well as the error. Harmless, but it is noise in exactly the place someone debugging a failed call will look.
+- Base UI overlays animate through `data-starting-style` and `data-ending-style` transitions, not keyframe animation libraries, and every transition has `motion-reduce:transition-none`.
+
+## Primitives
+
+`src/components/ui/` holds the shadcn primitives on Base UI.
+Added in Phase 1: breadcrumb, checkbox, command (`cmdk`), dialog, hover-card, popover, resizable (`react-resizable-panels`), scroll-area, slider, switch, table, toggle and toggle-group.
+`button-group` joins buttons edge to edge, as the instance power buttons are.
+`radio-group` is one of a few exclusive choices inside a form, such as a load balancer's type; a raw `<input type="radio">` is not used.
+
+A dialog only confirms (ADR-0016:S3): a destructive action, a typed phrase, or a secret shown once.
+A form never opens in one.
+Every dialog closes on Escape.
+⌘↵ or Ctrl+Enter submits it: an `AlertDialog` clicks its action unless that is disabled, and a `Dialog` submits its form, including from a textarea where Enter alone adds a line.
+Enter in a typed confirmation confirms once the phrase matches.
+The confirm button shows the shortcut, ⌘↵ on a Mac and Ctrl ↵ elsewhere (`src/lib/shortcuts.ts`), and names it in `aria-keyshortcuts`.
+
+A disabled control stays visible and shows `cursor: not-allowed`, so it says why it does nothing rather than vanishing.
+An on/off setting is a `Switch`, orange when on; a `Checkbox` only selects table rows, and a multi-select picker such as subnets, security groups or load balancer targets is a `SwitchList`.
+
+`InfoTip` (`src/components/info-tip.tsx`) is UniFi's (i): a small button beside a label that opens a short explanation on hover or click, with `InfoList` for the facts inside it.
+`PanelField` takes it through `info`, so the login user names the image's distribution and the image names its AMI details.
+
+Base UI marks orientation with `data-orientation`, so `styles.css` defines the `data-horizontal` and `data-vertical` variants the shadcn sources expect.
+
+Base UI marks the selected tab with `data-active`, not `data-selected`, so `ui/tabs` styles that attribute.
+
+shadcn's `chart` is not used: it pulls `recharts`, whose `reselect@5.1.1` dependency fails the workspace's `trustPolicy: no-downgrade`.
+Charts are uPlot instead (see Charts).
+
+## Enforcement
+
+Three lint rules and three source tests keep pages on the system.
+
+| Rule | Where | What it catches |
+|---|---|---|
+| `shadcn/no-unknown-classes` | all source, not tests | a class Tailwind does not generate, such as a typo or a removed token |
+| `shadcn/no-restyle` | all source | a page passing layout or colour classes to a primitive |
+| `eslint/no-restricted-imports` | `src/routes/**` | a route importing a chart library directly |
+| `src/routes/route-files.test.ts` | test suite | a route that builds a table by hand, with `<table>` or the `ui/table` primitives |
+| `design-coverage.test.ts` | test suite | a shared component in `src/components` that `/design` neither shows nor lists with a reason |
+
+The table test allows two files, each with its reason: the catalogue's specimen of the `ui/table` primitives, and the RDS parameter editor, an editable grid.
+Every other list is a `DataTable`, and the test also fails when an allowed file stops building a table so the list cannot go stale.
+
+`src/routes/design-rules.test.ts` reads the source of every route and shared component and holds one test per rule, named by its ADR-0016 clause, so a failure names the rule it breaks:
+
+| Clause | Fails when |
+|---|---|
+| S3 | a file that imports a dialog renders a form field or `<form>`, or a `create-*` route does not redirect to its list with the panel open |
+| S4 | a file renders `FormActions` without `FormLock` |
+| S5 | anything but `DataTable` uses a checkbox |
+| S6 | a file renders `FormActions` without `CliCommandPanel` |
+| S9 | a file renders `ResourceView` without `PageHeading` |
+| S10 | a route declares its own `selected`, `tab` or `create` search field instead of spreading `panelSearchFields` |
+| S12 | a panel's tabs do not start with Overview, or a retired `DetailCard`, `DetailRow` or `ListCard` comes back |
+| S13 | a component outside `ui/` renders a raw `<input>`, `<select>` or `<textarea>` |
+| S15 | a route imports the `ui/alert` primitive instead of using `ErrorBanner` |
+
+An exception is named in that rule's allowlist with its reason, and the test fails when an allowed file stops needing it.
+S11, S14 and S16 are held by review, since none has a shape in the source a test can read without false alarms.
+
+## Shell
+
+`src/lib/nav.ts` is the one description of the console's navigation.
+The sidebar, the command palette and the breadcrumbs all read it, and `nav.test.ts` checks every destination against the generated route tree, so a renamed route fails a test rather than leaving a dead link.
+
+- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. On a phone the sidebar is a sheet, and any navigation closes it.
+- **Command palette.** ⌘K or Ctrl K, or the Search button in the header. It lists every page the user can reach, every create flow (`create` on a nav item), and a theme toggle. It is navigation only, with no resource search yet.
+- **Breadcrumbs.** Service, then list page, then the resource or the create flow. A resource ID is set in mono. On a phone only the last crumb shows.
+- **Sign in.** The sign-in card sits over the shell on a light, barely blurred overlay rather than on a page of its own, so the console stays in view. The shell behind it is `inert`, which keeps pointer, keyboard and screen reader out of it until the user signs in.
+Signing in does not reload the page: the card shrinks and fades with its veil over 300ms, then the console routes to the dashboard in place, and the page content fades in. Signing out is a 300ms view-transition cross-fade into the sign-in card, which says "Signed out." in a warning-coloured notice. Under reduced motion both switch at once.
+
+## Layout
+
+The app shell's `<main>` is `min-w-0`, so a wide table scrolls inside its card instead of widening the page.
+The page gutter is 16px on a phone, 24px from `md` and 32px from `xl`.
+`PageHeading` wraps its actions below the title when they do not fit beside it.
+Every page must fit at 390px with `document.documentElement.scrollWidth` equal to the window width.
+
+## Data tables
+
+`DataTable` (`src/components/data-table.tsx`) renders `@tanstack/react-table` over the `ui/table` primitives, so sorting, row selection and shift-click range selection are the library's and not ours to maintain.
+Pages describe columns with the small `DataColumn` shape (`id`, `header`, `cell`, `sortValue`, `hideBelow`, `align`), which the component maps to TanStack column definitions; a column with a `sortValue` sorts, with missing values last and numbers compared as numbers.
+Sort and selection stay controlled by the caller, so a page can keep them in the URL.
+
+A column hides itself through `hideBelow`, which is a **container** query on the table's card (`@max-lg`, `@max-xl`, `@max-4xl`, `@max-5xl`), not a viewport one.
+So columns drop out when a side panel narrows the list as well as on a small screen.
+
+List pages keep their filter, state and sort in the URL, so a filtered view can be bookmarked, shared and survives a reload.
+EC2 Instances is the reference page (`ec2/(instances)/-components/instances-page.tsx`):
+
+- The first column names the resource: the `Name` tag in the primary colour, with the ID in mono beneath it, opening its side panel.
+- Bulk actions appear above the table once a row is selected. Each action counts only the selected instances it applies to, so a mixed selection never sends a call that must fail, and Terminate asks first and lists every ID.
+- An empty account gets a call to action, and an empty filter result says the filters are why.
+
+VPCs and Internet gateways follow the same pattern.
+
+### Selecting rows to delete
+
+Every list with a delete action has row checkboxes. Selecting a row shows `SelectionBar` (`src/components/selection-bar.tsx`) above the table, with the count, the actions and a way to clear, and `useRowSelection` (`src/hooks/use-row-selection.ts`) keeps the selection to rows still in the list.
+`BulkDelete` (`src/components/bulk-delete.tsx`) is the bar's destructive action, and its dialog names every selected resource before anything is sent.
+
+- **One call per resource, one at a time.** Most AWS delete calls take a single ID, so each refusal lands against its own row with AWS's message. A partial failure keeps the dialog open, and its confirm becomes "Retry N" for only the ones that failed.
+- **A batch call where AWS has one.** `deleteMany` instead of `deleteOne` hands the whole selection to one function that returns the refusals by ID, as S3's `DeleteObjects` does for up to 1000 keys.
+- **No pre-filtering.** The bar sends what was selected and lets AWS refuse, rather than guessing which deletes would fail. Instances are the exception, since one call takes many IDs and a mixed selection would fail as a whole.
+- **The bulk delete and the panel's delete are the same function.** Where a single delete is several calls, such as removing dependants first, the sequence lives in one place and both use it.
+- The verb follows the AWS console: Release for an Elastic IP, Deregister for an image.
+
+### Typed confirmation
+
+A delete that cannot be walked back asks the user to type a word first, as the AWS console does before deleting a volume.
+`ConfirmPhrase` (`src/components/confirm-phrase.tsx`) is the field, and the confirm stays disabled until it matches, ignoring case and surrounding space.
+`DeleteConfirmationDialog`, `ResourceDeleteDialog` and `BulkDelete` take it through `confirmPhrase`, so the panel's delete and the bulk delete ask the same way.
+
+- **"delete"** for volumes, snapshots, RDS instances and snapshots, EKS and ECS clusters and Route 53 hosted zones. An RDS instance deleted without a final snapshot asks for its identifier instead, which is stronger.
+- **"terminate"** for instances.
+- Not for anything cheap to recreate, such as a key pair, a security group rule or a tag.
+
+Key pairs is the smallest example (`ec2/(key)/-components/key-pairs-page.tsx`).
+
+### Object browser
+
+A bucket opens as a page rather than a panel, since its contents need the width (`s3/-components/object-browser.tsx`).
+Folders and objects share one table, folders first as S3 lists them; a folder row enters the folder, and an object row opens its panel with `HeadObject`'s properties, its user metadata and a download.
+
+- **The find box is S3's prefix.** It sends `Prefix` = folder + typed text to `ListObjectsV2` after a 250ms pause, so it finds keys the current page has not loaded, and it matches from the start of the name only.
+- **Pages are S3's.** 100 entries a page with `MaxKeys`, and Next follows `NextContinuationToken`. Tokens only run forward, so the browser keeps the stack of tokens that opened each page, and Previous pops it; sorting a column sorts the page on screen.
+- **Selecting a folder selects everything under it.** Delete lists every key under the folder, then sends `DeleteObjects` in batches of 1000, and a folder that was only partly deleted says how much was left.
+- The bucket list deletes only empty buckets; S3 refuses the rest with `BucketNotEmpty`, and the dialog says so before anything is sent.
+
+#### Uploads
+
+Uploads run in a store outside React (`src/lib/uploads.ts`), so they carry on, and keep their progress, while the user browses elsewhere in the console.
+
+- **A small, quick upload stays on the button.** While a batch runs, Upload becomes "Uploading N%" with a ring that fills, and a click on it opens the progress dialog.
+- **A large or slow batch opens the dialog.** It opens at once for 16 MiB or more, and otherwise after 2 seconds if the batch is still going, so a quick upload never flashes a dialog. Hide keeps the batch on the button, and the dialog does not reopen on its own after that.
+- **The dialog shows what a user would ask about.** Bytes sent of the total, the rate in bits a second with SI prefixes (kbit/s, Mbit/s, Gbit/s) over the last 3 seconds, an approximate time left, and a row per file with its own progress and cancel. A rate that has fallen to nothing after bytes moved reads "Stalled". A finished batch gives its elapsed time and average rate, and failures can be retried.
+- **Files go three at a time, each in 8 MiB parts sent four at a time**, with lib-storage's `Upload` over `XhrHttpHandler`, since `fetch` reports no upload progress. Each file has its own client, as the handler's progress event does not say which request it came from.
+- Leaving the page while uploads run asks first, and signing out cancels them.
+- The console's S3 proxy gives each request an hour, in place of the server's 15-second read and 120-second write timeouts, which would cut off a part or a download on a slow link.
+
+### Instance launch
+
+The image picker shows each distribution's own mark from `src/assets/os/` (Ubuntu, Debian, Rocky, Alpine, CentOS), imported with `?no-inline` so it is served as a file and satisfies `img-src 'self'`. Any other distribution gets a generic server icon, and the "Other images" tile a disc.
+The instance type select states the chosen type's vCPUs and memory beneath it, from `DescribeInstanceTypes`.
+
+## Side panels
+
+Modelled on UniFi's console: a click on a row opens the resource in a panel beside the list, and a create flow opens there too.
+
+- **URL.** The panel is search state on the list route: `selected=<id>`, `tab=overview|monitoring|settings`, and `create=true`. Defaults are stripped, so a closed panel leaves the plain list address. Opening or closing a panel pushes a history entry, so Back closes it; a tab or filter change replaces the entry.
+- **Layout.** From `lg` (1024px) the panel is fixed to the right edge, full height below the header, flush with no outer margin, and it slides in over the list. The list keeps its width and columns underneath, so nothing reflows when a panel opens, and a soft gradient on the panel's left edge reads as the panel lying on top. Below `lg` there is no room to share, so the panel replaces the list as a full page with a back arrow. `useWideLayout` decides, through `matchMedia`.
+- **`DetailPanel`.** A header with the resource's own page (left) and close (right), the title and its ID, a summary of state and quick actions, then icon tabs with tooltips. Escape closes it unless a dialog or menu is open over it. Closing slides the panel out over 150ms, or at once under reduced motion. Each tab, and a create form, sits in its own error boundary, so a failed query shows in the panel and the list keeps working.
+- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `PanelTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
+- **One component is both the panel and the page.** Each resource has a single details component (`InstanceDetails`, `VpcDetails`, `InternetGatewayDetails`) that takes `layout="auto" | "page"`. The list renders it as the panel; `/ec2/describe-instances/$id` and its kind render the same component as a page, as one centred column under a standard left-aligned heading, with state and actions on the right and text tabs. A feature added to the panel is on the page by construction, and the page ignores Escape and has a back arrow in place of close. Expanding the panel to its page keeps the tab, and query results such as fetched console output come along from the cache.
+- **A click on the list's background closes the panel**, as UniFi's does. A row selects another resource instead, and links, buttons and inputs keep their own behaviour. Selecting rows for a bulk action closes it too, so the bulk bar and the panel never compete.
+- **Actions.** State actions sit in the summary as a button group, and the rest go in an Actions menu, Unikraft style. Stop, Reboot and every destructive action confirm in a dialog.
+- **Loading.** A panel or page still loading shows `PanelSpinner`, a centred spinner on the background, not a grey skeleton block.
+- **`DetailCard`, `DetailRow` and `ListCard` are retired.** Every resource page uses the panel sections instead.
+
+### Forms from a resource's panel
+
+A form that acts on an open resource, such as attaching a volume, scaling a node group or adding a listener, opens in `FormPanel` (`src/components/form-panel.tsx`), never in a dialog.
+From `lg` it lies over the resource's panel at the same right edge and the create panel's width; below `lg` it covers the page, with a back arrow.
+It takes Escape from the panel beneath it, takes focus on open and hands it back to the button that opened it, and keeps a failure inside itself.
+Each such form is its own `*-form.tsx` file with `FormLock`, `CliCommandPanel` and `FormActions`, like a create form.
+It is held in the details component's state rather than the URL, because a form opened from one row of a tab has no list route to carry it, so Back closes the resource's panel and the form with it.
+
+## Create flows
+
+Every create flow opens in the side panel over its list, through `CreatePanel`, and never in a dialog or on a page of its own.
+
+- **URL.** `create=true` opens it, plus a list-local field where the flow needs context (`volume` for a snapshot, `instance` for an RDS snapshot, `launch=service|task` on ECS clusters). The old `create-*` addresses redirect to the list with the panel open, so links and bookmarks keep working.
+- **Form.** The fields come in the order AWS asks for them. Settings that are on or off are `Switch`es, and a multi-select picker (subnets, security groups) is a `SwitchList`, a boxed list with one switch per row. Checkboxes only select table rows.
+- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends.
+- **Submit.** `FormActions` turns the submit button into a spinner with its pending label in place, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
+- **Failure.** A refused request shows beside the button through `FormActions`'s `error`, and the form unlocks. The handler wraps only the mutation call in `try`, with an empty `catch` that returns, because react-hook-form rethrows whatever the handler throws and the error is already on screen from the mutation's state.
+- **Success.** `onClose(id)` closes the panel and selects the new resource, where the list has a panel to select it in.
+- **Appliance images.** A create that needs an ECS, EKS or RDS appliance image shows the missing-image callout inside its panel, never on the list, and only the step that needs the image is blocked.
+
+## Charts
+
+`MetricChart` (`src/components/charts/metric-chart.tsx`) draws a time series with uPlot, chosen for size and because it draws on a canvas, which the CSP allows where inline style attributes are not; `csp.e2e.ts` renders one under the production header.
+The look follows UniFi's statistics: smoothed lines over a fading fill, a light grid, a legend chip per series that reads the value under the cursor (or the latest), and `-24 hrs … Now` beneath.
+Series colours are the chart palette, and the chart redraws when the theme changes.
+
+Instance monitoring streams live statistics from the host: CPU utilisation, network bytes in and out, and packets in and out, one sample every 500 ms over the last five minutes, with no history to range over.
+The stream is `GetInstanceMetrics` on the spinifex service, a signed POST answered as `text/event-stream` (`src/lib/live-metrics.ts`), which ends every 60 s and reconnects with `Since` so nothing is lost or repeated.
+It runs only while the tab is visible and the instance is running; a segment that fails or delivers nothing backs off.
+A live chart draws stepped paths, and its x axis slides with the host's clock one second behind the newest sample (`LiveClock`), so the plot keeps moving between samples and a slow link does not jitter it.
+A gap of more than three sample periods breaks the line rather than drawing across missing time.
+
+## Dates
+
+`formatDateTime` writes the AWS console's form in the viewer's zone, `2026/06/04 02:00 GMT-5`, with a 24-hour clock and UTC as plain `GMT`.
+
+## Testing
+
+`make test-ui-e2e` rebuilds `dist` and runs `pnpm e2e`, which drives Playwright against two servers (`playwright.config.ts`):
+
+- **dev** (Vite on 3101) for behaviour: the `/design` catalogue in both themes and four widths, theme bootstrap, primitives' keyboard behaviour, the shell, and the Instances, VPC and Internet gateway list and panel flows.
+- **preview** (the built `dist` on 3102) for `csp.e2e.ts`, which serves the console's own CSP header and fails on any violation. Vite dev applies no CSP, so this is the only place an inline style or `eval` is caught before production.
+
+`e2e/console.ts` is the fixture. It seeds a fake session and answers `/proxy/**` by action, with empty defaults for what the dashboard and shell ask for; a test adds handlers for its own data, and any call nobody answered fails the test through `api.unhandled`.
+Unhandled calls get an error in the protocol's own shape, since a plain-text body would itself trip the CSP test.
+
+### Screenshots
+
+Screenshot baselines live in `e2e/__screenshots__/` and are Chromium renders taken in `en-AU` and UTC, so dates do not move with the host.
+They are drawn inside Playwright's own container image, because font rasterisation differs between hosts and a baseline drawn on a workstation fails on the runner.
+`playwright.image` in the frontend directory pins that image by version and digest, and is the one place it is named.
+Its version must equal the lockfile's `@playwright/test`, since the image carries the browsers for exactly one release, so bump both together.
+
+- `make ui-baselines` redraws every baseline in the pinned image, with `pnpm e2e --update-snapshots=all`, since the default mode keeps any baseline within the diff tolerance and so would keep one drawn on another host. Commit the changed files under `e2e/__screenshots__/` with the change that moved them.
+- `make ui-e2e-container` runs the suite in the same image and compares the screenshots, exactly as CI does.
+
+Both run `scripts/ui-playwright.sh`, which uses podman, or docker where podman is absent, and `CONTAINER_RUNTIME` overrides the choice.
+It mounts the frontend directory, installs with `pnpm install --frozen-lockfile` into a separate `node_modules` under `~/.cache/spinifex-ui-playwright`, so the host's `node_modules` is never touched, then builds `dist` and runs the suite.
+It refuses to run when the image's Node differs from `.nvmrc` or its Playwright differs from the lockfile's.
+`make test-ui-e2e` and `pnpm e2e` still run on the host, where screenshots compare reliably only against baselines drawn on that host, so do not commit baselines from `pnpm e2e:update`.
+
+The `UI End-to-End` job in `.github/workflows/ui.yml` runs the suite inside the pinned image on every pull request touching the frontend, and screenshots gate the merge with every behavioural and CSP test.
+A preceding `UI End-to-End Image` job reads `playwright.image`, so the workflow and the `make` targets cannot disagree about the image.
+On a failure the job uploads `test-results/` as the `ui-e2e-test-results` artifact, holding each failed screenshot's expected, actual and diff images and the trace.
+
+## Open
+
+- **Monitoring is live only.** Nothing is stored, so there is no `GetMetricData` and no range beyond the five-minute window; the history plan is `docs/development/feature/ec2-basic-metrics.md` in the mulga repository.

docs/vultr-integration/README.md

@@ -0,0 +1,403 @@
+---
+title: "Spinifex on Vultr"
+seoTitle: "Run Spinifex on Vultr with reserved IPs — Spinifex Docs"
+description: "Deploy Spinifex on Vultr VMs or bare metal with one command, and run EC2, EBS, S3, VPC, load balancers and EKS on public Vultr reserved IPs it manages."
+category: "Cloud Install"
+tags:
+  - install
+  - vultr
+  - cloud
+  - terraform
+  - cluster
+resources:
+  - title: "Setting Up Your Cluster"
+    url: "/docs/setting-up-your-cluster"
+  - title: "Spinifex on Oracle Cloud"
+    url: "/docs/oci-integration"
+  - title: "Single-Node Install"
+    url: "/docs/install"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+---
+
+# Running Spinifex on Vultr
+
+> Run the AWS surface (EC2, EBS, S3, VPC, load balancers and EKS) on Vultr, with guests that get real public addresses through Vultr's own API.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Instructions](#instructions)
+- [1. Download Spinifex](#1-download-spinifex)
+- [2. Choose a deployment](#2-choose-a-deployment)
+- [3. Deploy](#3-deploy)
+- [4. Restrict the API key](#4-restrict-the-api-key)
+- [5. Check it worked](#5-check-it-worked)
+- [6. Obtain AWS credentials](#6-obtain-aws-credentials)
+- [7. Set up your cluster](#7-set-up-your-cluster)
+- [8. Test resiliency](#8-test-resiliency)
+- [Removing the deployment](#removing-the-deployment)
+- [Limits](#limits)
+- [Troubleshooting](#troubleshooting)
+
+## Overview
+
+Spinifex is an open-source infrastructure platform that brings core AWS services to bare-metal, edge, and on-prem environments.
+It serves the EC2, EBS, S3, VPC, IAM and STS APIs, plus load balancers and EKS, over an endpoint of your own, so the AWS CLI, the AWS SDKs and the AWS Terraform provider all work against it unmodified.
+
+This guide runs Spinifex on Vultr, on one node or three, on VMs or on bare metal.
+It is the same product you would install on your own servers with [Single-Node Install](/docs/install), so everything you build on it behaves the same wherever Spinifex runs.
+
+**Why run it on Vultr:**
+
+- **A global network.** Vultr has 33 locations across six continents, so you can put an AWS-compatible region close to your users or your data.
+- **Compute when you need it.** VMs and bare metal are available by the hour in most Vultr locations, with no long-term commitment.
+- **Flexibility.** Start on one VM for development, then move to three nodes or bare metal for production with the same command.
+- **Portable workloads.** Your applications and Terraform code against the AWS APIs, not against Vultr, so they move between Vultr, AWS, your own rack or an edge site as a deployment decision.
+- **Keep your AWS shape.** VPCs, subnets, security groups, instances, volumes and load balancers stay exactly as you define them today.
+  No rewrite of your application or your infrastructure code is required.
+
+### How Spinifex uses your Vultr API key
+
+Spinifex includes Vultr support, which needs a Vultr API key to provide public networking.
+Whenever something you create on Spinifex needs a public address, such as an EC2 instance in a public subnet, an Elastic IP, an internet-facing load balancer or a NAT gateway, Spinifex creates a Vultr reserved IP through the API, attaches it to the node the resource runs on, and routes it to that resource.
+It deletes the reserved IP again when the resource goes away.
+
+The key is copied to the nodes over SSH and stored at `/etc/spinifex/vultr/api-key`, readable only by the Spinifex daemon.
+It is never written to the Terraform state or to the instance metadata.
+
+## Prerequisites
+
+You need:
+
+- **A Vultr account** with room for the nodes you choose in [step 2](#2-choose-a-deployment).
+- **A Vultr API key.** In the Vultr portal, open your user menu, then **API Access**, and under your organization choose **Create API Key**.
+  Under **Access Control List**, choose **Add IP to Allowlist** and add `0.0.0.0/0` for now.
+  The nodes call the API to manage their addresses, and their addresses do not exist until the deploy creates them.
+  You replace `0.0.0.0/0` with the real addresses in [step 4](#4-restrict-the-api-key).
+- **OpenTofu or Terraform**, version 1.5 or newer. Install [OpenTofu](https://opentofu.org/docs/intro/install/) or [Terraform](https://developer.hashicorp.com/terraform/install); the deploy script uses whichever it finds, preferring OpenTofu.
+- **The [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)** and `python3`.
+- **An SSH key pair** at `~/.ssh/id_ed25519`.
+  Create one with `ssh-keygen -t ed25519` if you do not have one.
+  The deploy installs the public key on every node.
+
+The API key page, with the key and the temporary `0.0.0.0/0` entry in its access control list:
+
+<p align="center">
+  <img src="../../.github/assets/images/vultr/vultr-api-keys.png" alt="Vultr API Access page, showing an API key and an access control list that allows 0.0.0.0/0" width="900">
+</p>
+<p align="center"><em>Create a Vultr API key in the web console, with 0.0.0.0/0 allowed until the deploy finishes.</em></p>
+
+## Instructions
+
+## 1. Download Spinifex
+
+The deploy script, `deploy-vultr.sh`, is in the Spinifex repository:
+
+```bash
+git clone https://github.com/mulgadc/spinifex.git
+cd spinifex
+git switch dev
+cd scripts/terraform/vultr-spx
+```
+
+> [!NOTE]
+> Vultr support is on the development branch and not yet in a published release.
+> `git switch dev` gives you the deploy script for it, and `--channel dev` in [step 3](#3-deploy) installs the matching Spinifex build, the pre-release published from every change to `dev`.
+> Once Vultr support is in a release, skip `git switch dev` and pass `--channel latest`.
+
+## 2. Choose a deployment
+
+| Use | Nodes | Plan | Cost while running |
+| --- | --- | --- | --- |
+| Development and testing | 1 | `vx1-g-16c-64g` VM (16 vCPU, 64 GB) | $0.48 an hour |
+| Production | 3 | `vx1-g-16c-64g` VM | $1.44 an hour |
+| Production on bare metal | 1 or 3 | `vbm-8c-64gb-amd` bare metal (8 cores, 16 threads, 64 GB, 2 × 1.9 TB NVMe) | $0.55 an hour per server |
+
+**For development and testing, use one `vx1-g-16c-64g` VM.** It is the smallest size we recommend, because the EC2 instances, load balancers, EKS and ECS clusters you create all run inside it and need headroom.
+Spinifex keeps 2 of its 16 vCPUs for the host and its own services and offers the other 14 to what you create, one vCPU each, with no overcommit.
+
+**For production, use three VMs, or bare metal.** On three nodes, an instance whose node fails is restarted on another node, and its public address follows it.
+`vbm-8c-64gb-amd` is the smallest bare-metal server we recommend for production workloads, as one server or as three.
+
+Bare metal takes longer to deliver than a VM, and Vultr bills a bare-metal server from the moment it is created, whether it is running or stopped.
+
+## 3. Deploy
+
+`deploy-vultr.sh` is a helper script around Terraform or OpenTofu and the official [Vultr provider](https://registry.terraform.io/providers/vultr/vultr/latest), so the Vultr side of the deployment is infrastructure as code you can read, keep and change in `scripts/terraform/vultr-spx/`.
+OpenTofu or Terraform creates a private Vultr VPC and the nodes.
+The script then installs Spinifex on every node over SSH and forms the cluster.
+
+<p align="center">
+  <img src="../../.github/assets/diagrams/vultr-deployment.svg" alt="Spinifex on Vultr: deploy-vultr.sh builds a Vultr VPC with one or three nodes, which cluster over the private network, call the Vultr API for reserved IPs, and run EC2, load balancers, EKS, ECS, RDS, S3, EBS and IAM across the fleet" width="900">
+</p>
+<p align="center"><em>One node or three in a private Vultr VPC, running AWS services across the fleet.</em></p>
+
+Each node runs **Ubuntu 24.04 LTS** as its host operating system, and Spinifex runs on top of it.
+Vultr installs the operating system when it creates the node, so there is no host to prepare first.
+
+Set your API key, then run the command for the deployment you chose:
+
+```bash
+export VULTR_API_KEY=<your key>
+
+# Development and testing: one VM
+./deploy-vultr.sh up --channel dev --plan vx1-g-16c-64g
+
+# Production: three VMs
+./deploy-vultr.sh up --channel dev --plan vx1-g-16c-64g --nodes 3
+
+# Production on bare metal: one server, or three
+./deploy-vultr.sh up --channel dev --plan vbm-8c-64gb-amd
+./deploy-vultr.sh up --channel dev --plan vbm-8c-64gb-amd --nodes 3
+```
+
+One VM takes about 4 minutes and three VMs about 16 minutes.
+Bare metal is slower, because Vultr takes 5 to 6 minutes to deliver a server: one server took 10 minutes in our test, and three took 18½.
+The default Vultr region is `ord` (Chicago); pass `--region` for another, and run `./deploy-vultr.sh --help` for every option.
+
+If a step fails, fix the cause and run the same command again: it resumes from the step that failed.
+
+When it finishes, the deploy prints:
+
+```ini
+endpoint=https://203.0.113.10:9999
+profile=spinifex
+ca=/home/you/.spinifex/vultr-default-ca.pem
+console=https://203.0.113.10:3000/
+node1=203.0.113.10
+api_key_allowlist=203.0.113.10/32
+```
+
+A three-node deployment lists `node2` and `node3` as well, and all three addresses in `api_key_allowlist`.
+
+## 4. Restrict the API key
+
+Now that the node addresses are known, close the API key to everything else.
+In the Vultr portal, open your user menu, then **API Access**, and under **Access Control List**:
+
+1. Choose **Add IP to Allowlist** for each address from the `api_key_allowlist` line.
+2. Add your own machine's public address, so that `./deploy-vultr.sh` can still remove the deployment later.
+3. Remove the `0.0.0.0/0` entry, and the IPv6 `::/0` entry if your list has one.
+
+`./deploy-vultr.sh status` prints the node addresses again whenever you need them.
+
+## 5. Check it worked
+
+```bash
+./deploy-vultr.sh status
+```
+
+This lists each node and its address, then every VM or bare-metal server, VPC, volume and reserved IP the deployment holds in your Vultr account, then the completed deploy steps.
+
+In the Vultr portal, the nodes are under **Compute → Instances**, bare-metal servers included.
+Each is named `spinifex-<deployment>-nodeN` and tagged `spinifex` and `deployment:<deployment>`, which is `deployment:default` unless you passed `--name`.
+These are the servers that run Spinifex, and the AWS workloads and services you create on it run inside them.
+
+<p align="center">
+  <img src="../../.github/assets/images/vultr/vultr-instances.png" alt="Vultr Instances page, showing Spinifex nodes named spinifex-deployment-nodeN with the spinifex and deployment tags" width="900">
+</p>
+<p align="center"><em>Spinifex nodes in the Vultr console, tagged with their deployment.</em></p>
+
+To log in to a node:
+
+```bash
+./deploy-vultr.sh ssh            # node1
+./deploy-vultr.sh ssh node2      # another node of a three-node deployment
+```
+
+For a three-node deployment, check that every node has joined:
+
+```bash
+./deploy-vultr.sh ssh 'spx get nodes'
+```
+
+Expect three nodes with `Ready` status:
+
+```text
+NAME  │ STATUS │ ROLES         │ IP        │ REGION    │ AZ         │ UPTIME │ VMs │ SERVICES
+──────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+node1 │ Ready  │ nats:leader   │ 10.10.8.5 │ us-east-1 │ us-east-1a │ 12m    │ 0   │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+node2 │ Ready  │ nats:follower │ 10.10.8.4 │ us-east-1 │ us-east-1a │ 11m    │ 0   │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+node3 │ Ready  │ nats:follower │ 10.10.8.3 │ us-east-1 │ us-east-1a │ 11m    │ 0   │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
+```
+
+## 6. Obtain AWS credentials
+
+The deploy generated an AWS access key and secret on node 1.
+Display them with:
+
+```bash
+./deploy-vultr.sh credentials
+```
+
+> [!WARNING]
+> This reveals an AWS secret.
+> Do not redirect its output, or paste the credentials into chat, tickets, source control or logs.
+
+The output has this shape:
+
+```ini
+[spinifex]
+aws_access_key_id = <access-key>
+aws_secret_access_key = <secret-key>
+```
+
+Use these to sign in to the web console at the `console` address the deploy printed, `https://<node1 address>:3000/`.
+
+The deploy has also written them to an AWS profile named `spinifex` on your machine, with the endpoint and the cluster's certificate already set, so the AWS CLI works straight away:
+
+```bash
+export AWS_PROFILE=spinifex
+aws ec2 describe-instance-types
+```
+
+The same profile is set up on node 1, if you prefer to work from there:
+
+```bash
+./deploy-vultr.sh ssh 'export AWS_PROFILE=spinifex; aws ec2 describe-instance-types'
+```
+
+If this returns a list of available instance types, your installation is working.
+
+<p align="center">
+  <img src="../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
+</p>
+<p align="center"><em>The Spinifex web console, on port 3000 of node 1.</em></p>
+
+**Congratulations, Spinifex is installed.**
+
+## 7. Set up your cluster
+
+Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
+
+Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
+
+## 8. Test resiliency
+
+A three-node deployment is for high availability: it protects your workloads against any one machine going down.
+The three nodes form one cluster, a lightweight AWS region on Vultr, and the cluster and everything on it keep running when a node fails.
+Spinifex restarts the failed node's instances on the surviving nodes, and moves each instance's Vultr reserved IP with it, so its public address keeps working.
+To watch it happen, run a load-balanced website across the three nodes, then power one of them off.
+
+### Launch a website on every node
+
+Import the Ubuntu image the web servers run on:
+
+```bash
+./deploy-vultr.sh ssh 'spx admin images import --name ubuntu-26.04-x86_64'
+```
+
+The [Nginx Web Server (Load Balanced)](/docs/nginx-alb) workbook launches web servers behind an Application Load Balancer.
+With these two variables it launches three, and a spread placement group puts each one on a different node:
+
+```bash
+cd docs/terraform-workbooks/nginx-alb
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply -var region=us-east-1 \
+  -var spinifex_endpoint="$(aws configure get endpoint_url)" \
+  -var server_count=3 -var placement_strategy=spread
+```
+
+Use `terraform` in place of `tofu` if that is what you installed.
+Check that each web server is on its own node:
+
+```bash
+../../../scripts/terraform/vultr-spx/deploy-vultr.sh ssh 'spx get vms'
+```
+
+The `NODE` column shows `node1`, `node2` and `node3` once each, and the load balancer is the `sys.micro` instance.
+
+### Keep requests running
+
+Get the load balancer's address, and request the page once a second in a second terminal:
+
+```bash
+ALB_IP=$(aws elbv2 describe-load-balancers --names nginx-alb \
+  --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' --output text)
+while :; do
+  printf '%s %s\n' "$(date +%T)" "$(curl -s -m2 "http://$ALB_IP/" | grep -o 'Server [0-9]' || echo FAILED)"
+  sleep 1
+done
+```
+
+Each line names the web server that answered.
+
+### Power off a node
+
+From the `vultr-spx` directory, power off a node that runs a web server but not the load balancer:
+
+```bash
+./deploy-vultr.sh stop node3
+```
+
+This is a hard power-off through the Vultr API, as a hardware failure would be, so the node hands nothing over first.
+Watch the requests and `spx get vms`:
+
+1. For up to about a minute, a few requests fail while the load balancer's health checks find that one web server is gone.
+2. After that, the other two web servers answer every request.
+3. About three minutes after the power-off, the lost web server is `running` on one of the surviving nodes.
+4. About four minutes after the power-off, its own public address answers again, and the load balancer sends it requests once its health checks pass.
+
+Power the node back on:
+
+```bash
+./deploy-vultr.sh start node3
+```
+
+It rejoined the cluster about a minute after a VM started, and about five minutes after a bare-metal server started.
+Wait until `spx get nodes` shows it `Ready` before you power off another node.
+The instances that moved stay where they are.
+
+### What we measured
+
+On three `vx1-g-16c-64g` VMs and on three `vbm-8c-64gb-amd` bare-metal servers in Chicago, with the workbook above:
+
+| Node powered off | Deployment | Instance `running` elsewhere | Its public address answers | Load balancer requests failed |
+| --- | --- | --- | --- | --- |
+| A node with one web server | VMs | 173 s | 244 s | 5 of 91, all within 45 s |
+| A node with one web server | Bare metal | 172 s | 223 s | 6 of 83, all within 60 s |
+| node1, with a web server, the load balancer and the cluster's NATS leader | VMs | 167 s | 253 s | 22 of 66, all within 115 s |
+| A node with two web servers and the load balancer | Bare metal | 204 s | 271 s | 21 of 79, all within 114 s |
+
+Each case was measured once.
+The requests column counts one-second probes, so a run with more timed-out requests sends fewer of them.
+The load balancer recovers first, because it moves to a surviving node like any other instance.
+While node1 is off, the AWS endpoint in your `spinifex` profile does not answer, because it is node1's address.
+Call another node meanwhile by adding `--endpoint-url https://<node2-address>:9999` to an `aws` command.
+
+When you are done, remove the workbook with `tofu destroy` and the same `-var` options before [removing the deployment](#removing-the-deployment).
+
+## Removing the deployment
+
+Remove anything you built on the cluster first, then:
+
+```bash
+./deploy-vultr.sh destroy
+```
+
+It shows what it will delete and asks you to type the deployment name, `default`, to confirm.
+It deletes the nodes and the VPC, plus anything else the deployment left in your Vultr account that keeps billing: VM boot volumes, and any reserved IP Spinifex created.
+When it finishes, nothing of the deployment is left, and if anything is, it lists it and fails.
+
+After removing the deployment, remove the node addresses from the API key's **Access Control** list, since Vultr may give them to someone else.
+
+## Limits
+
+- **One or three nodes.** Other cluster sizes are not supported.
+- **16 vCPU per node is the recommended minimum.** On a 16-vCPU node, Spinifex keeps 2 vCPU for the OS and its own services, and the other 14 are available for your EC2 instances and AWS services.
+- **A deleted reserved IP keeps its quota slot for 24 hours**, by Vultr's documentation, so creating and deleting many addresses in a day can reach the account's limit.
+- **A three-node deployment survives one failed node at a time.** The cluster needs a majority of its nodes, two of three, online to keep running.
+
+## Troubleshooting
+
+| Symptom | Cause and fix |
+| --- | --- |
+| `the API key was refused; check its IP allowlist` | Your machine's address is not in the key's **Access Control** list. Add it in the portal. |
+| `the node's API calls are refused` or `never logged its Vultr allocator` | A node's address is not in the key's **Access Control** list. Add `0.0.0.0/0` until the deploy finishes, then follow [step 4](#4-restrict-the-api-key), and run `up` again. |
+| `plan ... is not available in this region` | Pick another region with `--region`, or check the plan's availability in the Vultr portal. |
+| `formation failed` | The full log is in `~/.spinifex/vultr/default/form.log`. Run `up` again once the cause is fixed. |
+| Any other failure | The logs are under `~/.spinifex/vultr/default/`, and the message names the log on the node. `up` resumes from the failed step. |

go.mod

@@ -43,6 +43,9 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
+	go.etcd.io/etcd/api/v3 v3.5.35
+	go.etcd.io/etcd/client/pkg/v3 v3.5.35
+	go.etcd.io/etcd/client/v3 v3.5.35
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
 	go.opentelemetry.io/otel v1.47.0
 	go.opentelemetry.io/otel/metric v1.47.0
@@ -57,6 +60,7 @@ require (
 	golang.org/x/term v0.47.0
 	golang.org/x/time v0.16.0
 	golang.org/x/tools v0.52.0
+	google.golang.org/grpc v1.83.2
 	gopkg.in/ini.v1 v1.67.3
 )
 
@@ -107,6 +111,8 @@ require (
 	github.com/containerd/plugin v1.1.0 // indirect
 	github.com/containerd/ttrpc v1.2.9 // indirect
 	github.com/containerd/typeurl/v2 v2.3.0 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/dgraph-io/badger/v4 v4.9.6 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -122,6 +128,8 @@ require (
 	github.com/go-playground/validator/v10 v10.30.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/gofrs/flock v0.10.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -195,7 +203,10 @@ require (
 	go.opentelemetry.io/otel/log v1.47.0 // indirect
 	go.opentelemetry.io/otel/sdk/log v1.47.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
+	go.uber.org/zap v1.17.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
 	golang.org/x/mod v0.42.0 // indirect
 	golang.org/x/net v0.61.0 // indirect
@@ -204,7 +215,6 @@ require (
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
-	google.golang.org/grpc v1.83.2 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
 	tags.cncf.io/container-device-interface v1.1.1 // indirect
 	tags.cncf.io/container-device-interface/specs-go v1.1.1 // indirect

go.sum

@@ -128,6 +128,10 @@ github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uih
 github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs=
 github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
 github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
+github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -177,6 +181,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k=
 github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -243,6 +249,8 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc=
 github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
 github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
@@ -332,6 +340,7 @@ github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -400,9 +409,17 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
 go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
+go.etcd.io/etcd/api/v3 v3.5.35 h1:lB//Av9mFrkljv94XDNAY+qiyQA9W6S/TTA2Uwx2jsI=
+go.etcd.io/etcd/api/v3 v3.5.35/go.mod h1:z1FGQolu/tpmQ2MPFwQNt7G3cXBiSkTevZBhfOtEArQ=
+go.etcd.io/etcd/client/pkg/v3 v3.5.35 h1:nUZutZyJVvsKddK6zddXMBxIhmVi0SzPMBRS0FnNM3o=
+go.etcd.io/etcd/client/pkg/v3 v3.5.35/go.mod h1:0MNjMMqajrOI0/l8OLFkJBhE5Vm6BAi7cerKlrO38hw=
+go.etcd.io/etcd/client/v3 v3.5.35 h1:dk0aSLjY5D1eOR0SNnPXWOWCVA9yIzJoB8a912qxmoI=
+go.etcd.io/etcd/client/v3 v3.5.35/go.mod h1:LjtCl6G5li7PgsGjoQIpYXz/OvSbI1pCxpSp1kqo9P4=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
@@ -439,18 +456,26 @@ go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrF
 go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
 go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
 go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b h1:FHYZT12VzXnb6P399kd1i9Ai7aLF+CFSUsOFylDt7Lk=
 go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
 go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a h1:CsRd824LAzMmlpu9nYv2PHOqPQYv4Zjd4kGM+g0ahS0=
 go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a/go.mod h1:i9Hi8GeZFLBtUCXP73ZNSHKZ4K2JtxT0pLmo3sbJzlo=
+go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
 golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
@@ -458,24 +483,33 @@ golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+h
 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 h1:GTD/WuaexTazIG/SxLOz4rEKZPDVilmVVC2nz4xhwfE=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
 golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
 golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
 golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -508,6 +542,8 @@ golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
 golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
@@ -519,6 +555,9 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGN
 golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY=
 golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
@@ -536,6 +575,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
 pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=

internal/gwsign/signer.go

@@ -15,6 +15,8 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 )
 
 // Signer holds a credential provider and signs requests in place.
@@ -42,6 +44,16 @@ func NewIMDS(ctx context.Context, region string) (*Signer, error) {
 	return &Signer{provider: cfg.Credentials}, nil
 }
 
+// NewInstanceRole returns a Signer whose credentials come only from the
+// instance metadata service: no environment, profile or static-key fallback.
+func NewInstanceRole() *Signer {
+	return newInstanceRole(imds.New(imds.Options{}))
+}
+
+func newInstanceRole(client *imds.Client) *Signer {
+	return &Signer{provider: aws.NewCredentialsCache(ec2rolecreds.New(func(o *ec2rolecreds.Options) { o.Client = client }))}
+}
+
 // EnsureCredentials retrieves once from the provider, surfacing the error so a
 // caller can warm the IMDS instance-role datapath before the first Sign. The
 // SDK's IMDS client fast-fails each probe (250ms dial), so a cold per-tap

internal/gwsign/signer_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
 )
 
@@ -123,3 +124,41 @@ func TestNewIMDS_BuildsProvider(t *testing.T) {
 		t.Fatal("expected non-nil credentials provider")
 	}
 }
+
+// The instance-role signer ignores environment keys and signs only with the
+// credentials the metadata service issues.
+func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", testAccessKey)
+	t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretKey)
+	const imdsKey = "ASIAIMDSONLYEXAMPLE1"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + imdsKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2099-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
+
+	s := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
+	if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
+		t.Fatal(err)
+	}
+	if auth := req.Header.Get("Authorization"); !strings.Contains(auth, "Credential="+imdsKey+"/") {
+		t.Fatalf("Authorization = %q, want the IMDS credential", auth)
+	}
+
+	srv.Close()
+	cold := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
+	if err := cold.EnsureCredentials(context.Background()); err == nil {
+		t.Fatal("environment keys were used when IMDS was unreachable")
+	}
+}
+
+const emptySHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

internal/sqsworkergw/client.go

@@ -0,0 +1,242 @@
+// Package sqsworkergw is the shared SQS worker guest's SigV4 client for the
+// gateway's internal spinifex-sqs service. It sends only public values: the
+// worker's identity is the signing session, and the store seed never leaves
+// the guest.
+package sqsworkergw
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
+	"github.com/nats-io/nkeys"
+)
+
+const (
+	// The SigV4 credential scope the gateway routes the internal surface on.
+	signingService = "spinifex-sqs"
+	apiVersion     = "2026-10-30"
+	callTimeout    = 30 * time.Second
+	maxResponse    = 64 << 10
+
+	actionClaim    = "ClaimFirstStore"
+	actionDescribe = "DescribeStore"
+	actionRecord   = "RecordWorkerPublicKey"
+)
+
+// Signer signs a request in place for service and region.
+type Signer interface {
+	Sign(r *http.Request, payloadHash, service, region string) error
+}
+
+// Client calls the internal first-store actions for this worker.
+type Client struct {
+	baseURL    string
+	region     string
+	signer     Signer
+	httpClient *http.Client
+}
+
+// New builds a TLS 1.3 client that trusts only caPEM, the cluster CA. An
+// https URL, a Region and a CA are all required: there is no fallback.
+func New(baseURL, region string, caPEM []byte, signer Signer) (*Client, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil || u.Scheme != "https" || u.Host == "" {
+		return nil, errors.New("sqsworkergw: gateway URL must be an https URL")
+	}
+	if region == "" {
+		return nil, errors.New("sqsworkergw: Region is required")
+	}
+	if signer == nil {
+		return nil, errors.New("sqsworkergw: signer is required")
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(caPEM) {
+		return nil, errors.New("sqsworkergw: gateway CA has no usable certificates")
+	}
+	tlsCfg := &tls.Config{MinVersion: tls.VersionTLS13, CurvePreferences: tlsconfig.Curves, RootCAs: pool}
+	return &Client{
+		baseURL: strings.TrimRight(baseURL, "/"),
+		region:  region,
+		signer:  signer,
+		httpClient: &http.Client{
+			Timeout:   callTimeout,
+			Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 1, IdleConnTimeout: 30 * time.Second},
+			// A redirect would resend a signed request somewhere else.
+			CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+		},
+	}, nil
+}
+
+// APIError is a gateway refusal; Code is the AWS error code.
+type APIError struct {
+	Action     string
+	StatusCode int
+	Code       string
+}
+
+func (e *APIError) Error() string {
+	return fmt.Sprintf("spinifex-sqs %s: %s (HTTP %d)", e.Action, e.Code, e.StatusCode)
+}
+
+// Retryable reports a refusal that a later identical call may pass:
+// contention or a store that is not yet available. Every other refusal,
+// including AccessDenied, is final for this boot.
+func (e *APIError) Retryable() bool {
+	return e.Code == "ServiceUnavailable" || e.Code == "Throttling" || e.StatusCode == http.StatusServiceUnavailable
+}
+
+// Pending reports an AccessDenied, which a worker that has started but is
+// not yet bound to its identity legitimately receives. It authorizes
+// nothing; the caller may ask again within its own bound.
+func (e *APIError) Pending() bool { return e.Code == "AccessDenied" }
+
+// TransientError is a call that may not have reached the gateway, or whose
+// reply was lost: signing, sending or reading failed. Both actions are safe
+// to repeat, so it is retryable.
+type TransientError struct{ Err error }
+
+func (e *TransientError) Error() string   { return e.Err.Error() }
+func (e *TransientError) Unwrap() error   { return e.Err }
+func (e *TransientError) Retryable() bool { return true }
+
+type claimResult struct {
+	CreateKey bool   `xml:"CreateKey"`
+	PublicKey string `xml:"PublicKey"`
+}
+
+// Claim asks for this worker's current first-store decision. A decision
+// that is neither "create" nor one valid public key is refused here.
+func (c *Client) Claim(ctx context.Context) (bool, string, error) {
+	var res claimResult
+	if err := c.call(ctx, actionClaim, nil, &res); err != nil {
+		return false, "", err
+	}
+	switch {
+	case res.CreateKey && res.PublicKey == "":
+		return true, "", nil
+	case !res.CreateKey && nkeys.IsValidPublicUserKey(res.PublicKey):
+		return false, res.PublicKey, nil
+	}
+	return false, "", errors.New("spinifex-sqs ClaimFirstStore: response is not a valid decision")
+}
+
+// Store is the Region's store launch as the gateway reports it. Bound means
+// a VM was launched for it, not that it is ready.
+type Store struct {
+	Bound     bool
+	Address   string
+	PublicKey string
+}
+
+type describeResult struct {
+	State     string `xml:"State"`
+	Address   string `xml:"Address"`
+	PublicKey string `xml:"PublicKey"`
+}
+
+// DescribeStore reports this worker's store. A bound reply must carry an
+// IPv4 address and a valid public key; a pending one must carry neither.
+func (c *Client) DescribeStore(ctx context.Context) (Store, error) {
+	var res describeResult
+	if err := c.call(ctx, actionDescribe, nil, &res); err != nil {
+		return Store{}, err
+	}
+	switch {
+	case res.State == "pending" && res.Address == "" && res.PublicKey == "":
+		return Store{}, nil
+	case res.State == "bound" && isIPv4(res.Address) && nkeys.IsValidPublicUserKey(res.PublicKey):
+		return Store{Bound: true, Address: res.Address, PublicKey: res.PublicKey}, nil
+	}
+	return Store{}, errors.New("spinifex-sqs DescribeStore: response is not a valid store state")
+}
+
+func isIPv4(s string) bool {
+	ip := net.ParseIP(s)
+	return ip != nil && ip.To4() != nil && !strings.Contains(s, ":")
+}
+
+// RecordPublicKey records the worker's public key. Anything other than a
+// user public key is refused before it is sent.
+func (c *Client) RecordPublicKey(ctx context.Context, publicKey string) error {
+	if !nkeys.IsValidPublicUserKey(publicKey) {
+		return errors.New("spinifex-sqs RecordWorkerPublicKey: refusing to send a value that is not a user public key")
+	}
+	return c.call(ctx, actionRecord, url.Values{"PublicKey": {publicKey}}, nil)
+}
+
+func (c *Client) call(ctx context.Context, action string, params url.Values, out any) error {
+	form := url.Values{"Action": {action}, "Version": {apiVersion}}
+	maps.Copy(form, params)
+	body := []byte(form.Encode())
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
+	if err != nil {
+		return fmt.Errorf("spinifex-sqs %s: build request: %w", action, err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
+	sum := sha256.Sum256(body)
+	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), signingService, c.region); err != nil {
+		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: sign: %w", action, err)}
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: send: %w", action, err)}
+	}
+	defer resp.Body.Close()
+	respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse))
+	if err != nil {
+		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: read response: %w", action, err)}
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return parseError(action, resp.StatusCode, respBody)
+	}
+	if out == nil {
+		return nil
+	}
+	return decodeResult(action, respBody, out)
+}
+
+func decodeResult(action string, body []byte, out any) error {
+	want := action + "Result"
+	dec := xml.NewDecoder(bytes.NewReader(body))
+	for {
+		tok, err := dec.Token()
+		if err != nil {
+			return fmt.Errorf("spinifex-sqs %s: response carries no <%s>", action, want)
+		}
+		if start, ok := tok.(xml.StartElement); ok && start.Name.Local == want {
+			if err := dec.DecodeElement(out, &start); err != nil {
+				return fmt.Errorf("spinifex-sqs %s: decode response: %w", action, err)
+			}
+			return nil
+		}
+	}
+}
+
+// The error body is not echoed: only the parsed code is kept.
+func parseError(action string, status int, body []byte) error {
+	var env struct {
+		Error struct {
+			Code string `xml:"Code"`
+		} `xml:"Error"`
+	}
+	code := fmt.Sprintf("HTTP%d", status)
+	if xml.Unmarshal(body, &env) == nil && env.Error.Code != "" {
+		code = env.Error.Code
+	}
+	return &APIError{Action: action, StatusCode: status, Code: code}
+}

internal/sqsworkergw/client_test.go

@@ -0,0 +1,235 @@
+package sqsworkergw
+
+import (
+	"context"
+	"encoding/pem"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/nats-io/nkeys"
+)
+
+type nopSigner struct{ err error }
+
+func (s nopSigner) Sign(r *http.Request, _, service, region string) error {
+	if s.err != nil {
+		return s.err
+	}
+	r.Header.Set("Authorization", "test "+service+"/"+region)
+	return nil
+}
+
+func userKey(t *testing.T) (pub, seed string) {
+	t.Helper()
+	kp, err := nkeys.CreateUser()
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, _ := kp.PublicKey()
+	s, _ := kp.Seed()
+	return p, string(s)
+}
+
+// server answers with body and status, counting and keeping requests.
+func server(t *testing.T, status int, body string) (*Client, *atomic.Int32, *[]string) {
+	t.Helper()
+	var n atomic.Int32
+	var seen []string
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n.Add(1)
+		b, _ := io.ReadAll(r.Body)
+		seen = append(seen, r.Header.Get("Authorization")+"|"+string(b))
+		w.WriteHeader(status)
+		_, _ = io.WriteString(w, body)
+	}))
+	t.Cleanup(srv.Close)
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
+	c, err := New(srv.URL, "ap-southeast-2", ca, nopSigner{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c, &n, &seen
+}
+
+func TestNew_RefusesWithoutHTTPSRegionCAOrSigner(t *testing.T) {
+	ca := []byte("not a certificate")
+	cases := map[string]func() error{
+		"http":      func() error { _, err := New("http://10.0.0.1:9999", "ap-southeast-2", ca, nopSigner{}); return err },
+		"no host":   func() error { _, err := New("https://", "ap-southeast-2", ca, nopSigner{}); return err },
+		"no region": func() error { _, err := New("https://10.0.0.1:9999", "", ca, nopSigner{}); return err },
+		"no signer": func() error { _, err := New("https://10.0.0.1:9999", "ap-southeast-2", ca, nil); return err },
+		"no CA":     func() error { _, err := New("https://10.0.0.1:9999", "ap-southeast-2", ca, nopSigner{}); return err },
+	}
+	for name, fn := range cases {
+		if fn() == nil {
+			t.Errorf("%s: client built", name)
+		}
+	}
+}
+
+func TestClaim_SignsForTheInternalServiceAndRegion(t *testing.T) {
+	c, _, seen := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>true</CreateKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
+	create, pub, err := c.Claim(context.Background())
+	if err != nil || !create || pub != "" {
+		t.Fatalf("claim = %v %q %v", create, pub, err)
+	}
+	if got := (*seen)[0]; !strings.HasPrefix(got, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(got, "Action=ClaimFirstStore") {
+		t.Fatalf("request = %q", got)
+	}
+}
+
+// Only "create" or exactly one valid public key is a decision.
+func TestClaim_RefusesAnythingButOneDecision(t *testing.T) {
+	pub, seed := userKey(t)
+	cases := map[string]string{
+		"create with a key": `<CreateKey>true</CreateKey><PublicKey>` + pub + `</PublicKey>`,
+		"neither":           `<CreateKey>false</CreateKey>`,
+		"seed as key":       `<CreateKey>false</CreateKey><PublicKey>` + seed + `</PublicKey>`,
+		"no result":         ``,
+	}
+	for name, inner := range cases {
+		t.Run(name, func(t *testing.T) {
+			body := `<ClaimFirstStoreResponse><ClaimFirstStoreResult>` + inner + `</ClaimFirstStoreResult></ClaimFirstStoreResponse>`
+			if inner == "" {
+				body = `<Other/>`
+			}
+			c, _, _ := server(t, 200, body)
+			_, _, err := c.Claim(context.Background())
+			if err == nil {
+				t.Fatal("accepted a non-decision")
+			}
+			if strings.Contains(err.Error(), seed) {
+				t.Fatal("the error echoes a seed")
+			}
+		})
+	}
+	t.Run("existing", func(t *testing.T) {
+		c, _, _ := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>false</CreateKey><PublicKey>`+pub+`</PublicKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
+		create, got, err := c.Claim(context.Background())
+		if err != nil || create || got != pub {
+			t.Fatalf("claim = %v %q %v", create, got, err)
+		}
+	})
+}
+
+func TestRecordPublicKey_SendsOnlyAPublicKey(t *testing.T) {
+	pub, seed := userKey(t)
+	c, n, seen := server(t, 200, `<RecordWorkerPublicKeyResponse/>`)
+	if err := c.RecordPublicKey(context.Background(), seed); err == nil || strings.Contains(err.Error(), seed) {
+		t.Fatalf("seed err = %v", err)
+	}
+	if n.Load() != 0 {
+		t.Fatal("a seed was sent")
+	}
+	if err := c.RecordPublicKey(context.Background(), pub); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains((*seen)[0], "PublicKey="+pub) {
+		t.Fatalf("request = %q", (*seen)[0])
+	}
+}
+
+// Refusals carry only the parsed code; ServiceUnavailable is the one
+// retryable refusal, and transport failures are retryable too.
+func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
+	cases := []struct {
+		status    int
+		code      string
+		retryable bool
+	}{
+		{403, "AccessDenied", false},
+		{400, "InvalidParameterValue", false},
+		{400, "InvalidState", false},
+		{403, "SignatureDoesNotMatch", false},
+		{503, "ServiceUnavailable", true},
+		{500, "InternalFailure", false},
+	}
+	for _, tc := range cases {
+		c, _, _ := server(t, tc.status, `<ErrorResponse><Error><Code>`+tc.code+`</Code><Message>secret-ish detail</Message></Error></ErrorResponse>`)
+		_, _, err := c.Claim(context.Background())
+		api, ok := errors.AsType[*APIError](err)
+		if !ok || api.Code != tc.code || api.Retryable() != tc.retryable || api.Pending() != (tc.code == "AccessDenied") {
+			t.Fatalf("%s: err = %v", tc.code, err)
+		}
+		if strings.Contains(err.Error(), "secret-ish") {
+			t.Fatal("the error echoes the response body")
+		}
+	}
+
+	c, _, _ := server(t, 200, "")
+	c.baseURL = "https://127.0.0.1:1"
+	_, _, err := c.Claim(context.Background())
+	tr, ok := errors.AsType[*TransientError](err)
+	if !ok || !tr.Retryable() {
+		t.Fatalf("unreachable err = %v, want TransientError", err)
+	}
+}
+
+// A redirect would resend a signed request elsewhere; it is not followed.
+func TestCall_DoesNotFollowRedirects(t *testing.T) {
+	var followed atomic.Bool
+	target := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { followed.Store(true) }))
+	t.Cleanup(target.Close)
+	c, _, _ := server(t, 0, "")
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect)
+	}))
+	t.Cleanup(srv.Close)
+	c.baseURL = srv.URL
+	if _, _, err := c.Claim(context.Background()); err == nil {
+		t.Fatal("a redirect was accepted as a decision")
+	}
+	if followed.Load() {
+		t.Fatal("the signed request followed a redirect")
+	}
+}
+
+// Only "pending" with nothing else, or "bound" with an IPv4 address and one
+// valid public key, is a store state.
+func TestDescribeStore_AcceptsOnlyAValidState(t *testing.T) {
+	pub, seed := userKey(t)
+	wrap := func(inner string) string {
+		return `<DescribeStoreResponse><DescribeStoreResult>` + inner + `</DescribeStoreResult></DescribeStoreResponse>`
+	}
+	t.Run("pending", func(t *testing.T) {
+		c, _, seen := server(t, 200, wrap(`<State>pending</State>`))
+		st, err := c.DescribeStore(context.Background())
+		if err != nil || st.Bound {
+			t.Fatalf("store = %+v %v", st, err)
+		}
+		if !strings.Contains((*seen)[0], "Action=DescribeStore") {
+			t.Fatalf("request = %q", (*seen)[0])
+		}
+	})
+	t.Run("bound", func(t *testing.T) {
+		c, _, _ := server(t, 200, wrap(`<State>bound</State><Address>10.240.0.9</Address><PublicKey>`+pub+`</PublicKey>`))
+		st, err := c.DescribeStore(context.Background())
+		if err != nil || !st.Bound || st.Address != "10.240.0.9" || st.PublicKey != pub {
+			t.Fatalf("store = %+v %v", st, err)
+		}
+	})
+	for name, inner := range map[string]string{
+		"pending with address": `<State>pending</State><Address>10.240.0.9</Address>`,
+		"bound without key":    `<State>bound</State><Address>10.240.0.9</Address>`,
+		"bound with seed":      `<State>bound</State><Address>10.240.0.9</Address><PublicKey>` + seed + `</PublicKey>`,
+		"bound with hostname":  `<State>bound</State><Address>store.example</Address><PublicKey>` + pub + `</PublicKey>`,
+		"bound with IPv6":      `<State>bound</State><Address>::ffff:10.240.0.9</Address><PublicKey>` + pub + `</PublicKey>`,
+		"unknown state":        `<State>ready</State>`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			c, _, _ := server(t, 200, wrap(inner))
+			_, err := c.DescribeStore(context.Background())
+			if err == nil {
+				t.Fatal("accepted an invalid store state")
+			}
+			if strings.Contains(err.Error(), seed) {
+				t.Fatal("the error echoes a seed")
+			}
+		})
+	}
+}

internal/testkit/fakeetcd/fakeetcd.go

@@ -0,0 +1,217 @@
+// Package fakeetcd serves the etcd Cluster and Maintenance gRPC APIs over mutual
+// TLS with the file layout K3s uses, so a real etcd client can be driven in tests.
+package fakeetcd
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+// Member is one entry of the fake cluster's member list.
+type Member struct {
+	ID       uint64
+	Name     string
+	PeerURLs []string
+}
+
+// Server is a running fake etcd.
+type Server struct {
+	// Endpoint is the https URL clients dial.
+	Endpoint string
+	// CertDir holds client.crt, client.key and server-ca.crt.
+	CertDir string
+
+	mu      sync.Mutex
+	members []Member
+	selfID  uint64
+	removed []uint64
+	failAll error
+}
+
+// Start serves members with selfID as the answering member. It stops with the test.
+func Start(t *testing.T, selfID uint64, members []Member) *Server {
+	t.Helper()
+	dir := t.TempDir()
+	caCert, caKey := newCA(t)
+	writePEM(t, filepath.Join(dir, "server-ca.crt"), "CERTIFICATE", caCert.Raw)
+	serverPair := issue(t, caCert, caKey, x509.ExtKeyUsageServerAuth)
+	clientPair := issue(t, caCert, caKey, x509.ExtKeyUsageClientAuth)
+	writeKeyPair(t, dir, "client", clientPair)
+
+	pool := x509.NewCertPool()
+	pool.AddCert(caCert)
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	creds := credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{serverPair.tls},
+		ClientCAs:    pool,
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		MinVersion:   tls.VersionTLS12,
+	})
+	s := &Server{Endpoint: "https://" + lis.Addr().String(), CertDir: dir, members: members, selfID: selfID}
+	gs := grpc.NewServer(grpc.Creds(creds))
+	pb.RegisterClusterServer(gs, &cluster{s})
+	pb.RegisterMaintenanceServer(gs, &maintenance{s: s})
+	go func() { _ = gs.Serve(lis) }()
+	t.Cleanup(gs.Stop)
+	return s
+}
+
+// Removed returns the member IDs removed so far.
+func (s *Server) Removed() []uint64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]uint64(nil), s.removed...)
+}
+
+// FailWith makes every later call return err.
+func (s *Server) FailWith(err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.failAll = err
+}
+
+type cluster struct{ s *Server }
+
+func (c *cluster) MemberList(context.Context, *pb.MemberListRequest) (*pb.MemberListResponse, error) {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+	if c.s.failAll != nil {
+		return nil, c.s.failAll
+	}
+	resp := &pb.MemberListResponse{Header: &pb.ResponseHeader{MemberId: c.s.selfID}}
+	for _, m := range c.s.members {
+		resp.Members = append(resp.Members, &pb.Member{ID: m.ID, Name: m.Name, PeerURLs: m.PeerURLs})
+	}
+	return resp, nil
+}
+
+func (c *cluster) MemberRemove(_ context.Context, req *pb.MemberRemoveRequest) (*pb.MemberRemoveResponse, error) {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+	if c.s.failAll != nil {
+		return nil, c.s.failAll
+	}
+	c.s.removed = append(c.s.removed, req.ID)
+	return &pb.MemberRemoveResponse{Header: &pb.ResponseHeader{MemberId: c.s.selfID}}, nil
+}
+
+func (c *cluster) MemberAdd(context.Context, *pb.MemberAddRequest) (*pb.MemberAddResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (c *cluster) MemberUpdate(context.Context, *pb.MemberUpdateRequest) (*pb.MemberUpdateResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (c *cluster) MemberPromote(context.Context, *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+type maintenance struct {
+	pb.UnimplementedMaintenanceServer
+
+	s *Server
+}
+
+func (m *maintenance) Status(context.Context, *pb.StatusRequest) (*pb.StatusResponse, error) {
+	m.s.mu.Lock()
+	defer m.s.mu.Unlock()
+	if m.s.failAll != nil {
+		return nil, m.s.failAll
+	}
+	return &pb.StatusResponse{Header: &pb.ResponseHeader{MemberId: m.s.selfID}}, nil
+}
+
+type pair struct {
+	tls tls.Certificate
+	der []byte
+	key *ecdsa.PrivateKey
+}
+
+func newCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("ca key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "fake-etcd-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("ca cert: %v", err)
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse ca: %v", err)
+	}
+	return cert, key
+}
+
+func issue(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, usage x509.ExtKeyUsage) pair {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "fake-etcd"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{usage},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+		DNSNames:     []string{"localhost"},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("cert: %v", err)
+	}
+	return pair{tls: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, der: der, key: key}
+}
+
+func writeKeyPair(t *testing.T, dir, name string, p pair) {
+	t.Helper()
+	keyDER, err := x509.MarshalECPrivateKey(p.key)
+	if err != nil {
+		t.Fatalf("marshal key: %v", err)
+	}
+	writePEM(t, filepath.Join(dir, name+".crt"), "CERTIFICATE", p.der)
+	writePEM(t, filepath.Join(dir, name+".key"), "EC PRIVATE KEY", keyDER)
+}
+
+func writePEM(t *testing.T, path, blockType string, der []byte) {
+	t.Helper()
+	data := pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der})
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}

internal/testkit/gwfix/deps.go

@@ -0,0 +1,43 @@
+package gwfix
+
+import (
+	"sync"
+
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/nats-io/nats.go"
+)
+
+// Deps are the collaborators a test composes a gateway from. The gateway
+// config no longer carries them, so test setup keeps them beside the config.
+type Deps struct {
+	IAM   iamdomain.IAMService
+	STS   stsdomain.STSService
+	NATS  *nats.Conn
+	Nodes int
+}
+
+var (
+	depsMu sync.Mutex
+	depsOf = map[any]*Deps{}
+)
+
+// WithDeps records d for gw and returns gw.
+func WithDeps[P comparable](gw P, d Deps) P {
+	depsMu.Lock()
+	defer depsMu.Unlock()
+	depsOf[gw] = &d
+	return gw
+}
+
+// DepsOf returns the deps recorded for gw, creating an empty set when none.
+func DepsOf[P comparable](gw P) *Deps {
+	depsMu.Lock()
+	defer depsMu.Unlock()
+	d, ok := depsOf[gw]
+	if !ok {
+		d = &Deps{}
+		depsOf[gw] = d
+	}
+	return d
+}

internal/testkit/gwfix/gwfix.go

@@ -0,0 +1,266 @@
+// Package gwfix holds runtime-free fixtures for gateway-level tests: SigV4
+// signing helpers, identity/session doubles and policy builders. It must not
+// import runtime/ or operator/ so gateway tests stay free of them.
+package gwfix
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	awsv1 "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	AccessKey = "AKIAIOSFODNN7EXAMPLE"                     //nolint:gosec // AWS documentation example key
+	SecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" //nolint:gosec // AWS documentation example secret
+	Region    = "us-east-1"
+	Service   = "ec2"
+
+	SessionAKID  = "ASIATESTSESSIONAAAAA"
+	SessionToken = "wire-session-token-plain"
+	RoleARN      = "arn:aws:iam::123456789012:role/test-role"
+	AssumedARN   = "arn:aws:sts::123456789012:assumed-role/test-role/test-session"
+
+	PrincipalTypeUser        = "user"
+	PrincipalTypeAssumedRole = "assumed-role"
+	PrincipalTypeRoot        = "root"
+)
+
+// IAM implements iamdomain.IAMService for auth tests. It embeds the interface
+// so it satisfies the full contract; only the methods the SigV4 path drives are
+// wired, and any other method nil-panics to surface an unexpected call.
+type IAM struct {
+	iamdomain.IAMService
+
+	AccessKeys map[string]*iamdomain.AccessKey
+	MasterKey  []byte
+
+	// Accounts overrides the status the auth gate reads. An empty map means
+	// every account is ACTIVE.
+	Accounts map[string]*iamdomain.Account
+
+	// GetUserFn overrides the synthetic user record, for a test that cares what
+	// aws:userid resolves to or that it fails to resolve at all.
+	GetUserFn func(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error)
+}
+
+func (m *IAM) LookupAccessKey(accessKeyID string) (*iamdomain.AccessKey, error) {
+	ak, ok := m.AccessKeys[accessKeyID]
+	if !ok {
+		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	}
+	return ak, nil
+}
+
+func (m *IAM) DecryptSecret(ciphertext string) (string, error) {
+	return iamdomain.DecryptSecret(ciphertext, m.MasterKey)
+}
+
+func (m *IAM) GetAccount(accountID string) (*iamdomain.Account, error) {
+	if account, ok := m.Accounts[accountID]; ok {
+		return account, nil
+	}
+	return &iamdomain.Account{AccountID: accountID, Status: iamdomain.AccountStatusActive}, nil
+}
+
+// GetUser answers with a unique ID derived from the name, so aws:userid resolves
+// on the authorization path without every test defining a user record.
+func (m *IAM) GetUser(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error) {
+	if m.GetUserFn != nil {
+		return m.GetUserFn(accountID, input)
+	}
+	name := awsv1.StringValue(input.UserName)
+	return &iam.GetUserOutput{User: &iam.User{
+		UserName: awsv1.String(name),
+		UserId:   awsv1.String("AIDA" + strings.ToUpper(name)),
+	}}, nil
+}
+
+// PolicyIAM adds configurable user and role policy lookups to IAM.
+type PolicyIAM struct {
+	IAM
+
+	GetUserPoliciesFn func(accountID, userName string) ([]iamdomain.PolicyDocument, error)
+	GetRolePoliciesFn func(accountID, roleName string) ([]iamdomain.PolicyDocument, error)
+}
+
+func (m *PolicyIAM) GetUserPolicies(accountID, userName string) ([]iamdomain.PolicyDocument, error) {
+	if m.GetUserPoliciesFn != nil {
+		return m.GetUserPoliciesFn(accountID, userName)
+	}
+	return nil, nil
+}
+
+func (m *PolicyIAM) GetRolePolicies(accountID, roleName string) ([]iamdomain.PolicyDocument, error) {
+	if m.GetRolePoliciesFn != nil {
+		return m.GetRolePoliciesFn(accountID, roleName)
+	}
+	return nil, nil
+}
+
+// ErrorLookupIAM returns a configurable error from LookupAccessKey.
+type ErrorLookupIAM struct {
+	IAM
+
+	Err error
+}
+
+func (m *ErrorLookupIAM) LookupAccessKey(_ string) (*iamdomain.AccessKey, error) {
+	return nil, m.Err
+}
+
+// MasterKey is a fixed 32-byte key for deterministic secret encryption in tests.
+var MasterKey = bytes.Repeat([]byte{0x5a}, 32)
+
+// STS implements the session-credential half of stsdomain.STSService.
+type STS struct {
+	stsdomain.STSService
+
+	Sessions  map[string]*stsdomain.SessionCredential
+	Tokens    map[string]string // AKID → plaintext wire token for HMAC equivalence
+	LookupErr error
+	Lookups   atomic.Int32 // counts LookupSessionCredential calls for negative-side-effect assertions
+
+	PrincipalErr    error        // verdict or fault the continuity check returns
+	PrincipalChecks atomic.Int32 // counts VerifySessionPrincipal calls, to pin its ordering
+}
+
+func (m *STS) LookupSessionCredential(accessKeyID string) (*stsdomain.SessionCredential, error) {
+	m.Lookups.Add(1)
+	if m.LookupErr != nil {
+		return nil, m.LookupErr
+	}
+	if !strings.HasPrefix(accessKeyID, "ASIA") {
+		return nil, nil
+	}
+	cred, ok := m.Sessions[accessKeyID]
+	if !ok {
+		return nil, nil
+	}
+	return cred, nil
+}
+
+// VerifySessionToken uses plaintext equality rather than HMAC to keep tests
+// simple while preserving the contract shape.
+func (m *STS) VerifySessionToken(cred *stsdomain.SessionCredential, wireToken string) bool {
+	if cred == nil {
+		return false
+	}
+	want, ok := m.Tokens[cred.AccessKeyID]
+	if !ok {
+		return false
+	}
+	return want == wireToken
+}
+
+// VerifySessionPrincipal defaults to "still the same principal"; PrincipalErr
+// drives the rejection-verdict and dependency-fault branches.
+func (m *STS) VerifySessionPrincipal(cred *stsdomain.SessionCredential) (*stsdomain.SessionPrincipal, error) {
+	m.PrincipalChecks.Add(1)
+	if m.PrincipalErr != nil {
+		return nil, m.PrincipalErr
+	}
+	return StubSessionPrincipal(cred)
+}
+
+// StubSessionPrincipal is the shared "principal unchanged" answer for the
+// session doubles. It derives RoleName from the underlying role ARN rather than
+// the session name, because that is what the real implementation returns: a
+// stub echoing SessionName would let a door read the wrong field and stay green.
+func StubSessionPrincipal(cred *stsdomain.SessionCredential) (*stsdomain.SessionPrincipal, error) {
+	if cred == nil {
+		return nil, errors.New("nil session credential")
+	}
+	roleName := cred.UnderlyingRoleARN
+	if idx := strings.LastIndex(roleName, "/"); idx >= 0 {
+		roleName = roleName[idx+1:]
+	}
+	return &stsdomain.SessionPrincipal{
+		UserID:   cred.UserID,
+		RoleID:   cred.RoleID,
+		RoleName: roleName,
+		RoleARN:  cred.UnderlyingRoleARN,
+	}, nil
+}
+
+// SignRequest signs req in place with the aws-sdk-go-v2 v4 signer — the same
+// path production callers use via gwsign — so the gateway verifies a real SDK
+// signature. body must match what the middleware reads from r.Body so the payload
+// hash agrees. An optional signingTime pins the clock for skew tests.
+func SignRequest(t *testing.T, req *http.Request, body []byte, accessKey, secret string, signingTime ...time.Time) {
+	t.Helper()
+	when := time.Now().UTC()
+	if len(signingTime) > 0 {
+		when = signingTime[0]
+	}
+	SignRequestAs(t, req, body, accessKey, secret, Service, when)
+}
+
+// SignRequestAs signs with an explicit credential-scope service, so a test
+// can drive a scope the gateway does not serve.
+func SignRequestAs(t *testing.T, req *http.Request, body []byte, accessKey, secret, service string, when time.Time) {
+	t.Helper()
+	sum := sha256.Sum256(body)
+	payloadHash := hex.EncodeToString(sum[:])
+	// gwsign sets this header so the SDK signs it; sigv4 reproduces it when
+	// reconstructing the canonical request.
+	req.Header.Set("X-Amz-Content-Sha256", payloadHash)
+	require.NoError(t, v4.NewSigner().SignHTTP(context.Background(),
+		aws.Credentials{AccessKeyID: accessKey, SecretAccessKey: secret},
+		req, payloadHash, service, Region, when))
+}
+
+// SignSessionRequest sets X-Amz-Security-Token before signing so it's included
+// in SignedHeaders, matching what the AWS CLI/SDK do with session credentials.
+func SignSessionRequest(t *testing.T, req *http.Request, body []byte, accessKey, secret, sessionToken string) {
+	t.Helper()
+	req.Header.Set("X-Amz-Security-Token", sessionToken)
+	SignRequest(t, req, body, accessKey, secret)
+}
+
+// AllowPolicy returns a single-statement Allow policy granting action on "*".
+func AllowPolicy(action string) []iamdomain.PolicyDocument {
+	return AllowPolicyResource(action, "*")
+}
+
+// AllowPolicyResource returns a single-statement Allow policy for action on resource.
+func AllowPolicyResource(action, resource string) []iamdomain.PolicyDocument {
+	return []iamdomain.PolicyDocument{{
+		Version: "2012-10-17",
+		Statement: []iamdomain.Statement{{
+			Effect:   "Allow",
+			Action:   iamdomain.StringOrArr{action},
+			Resource: iamdomain.StringOrArr{resource},
+		}},
+	}}
+}
+
+// AssumedRoleSessionCred builds an assumed-role SessionCredential.
+// SecretEncrypted is filled in by the gateway builder.
+func AssumedRoleSessionCred(sessionName, underlyingRoleARN, accountID string) *stsdomain.SessionCredential {
+	return &stsdomain.SessionCredential{
+		AccessKeyID:       SessionAKID,
+		AccountID:         accountID,
+		PrincipalType:     PrincipalTypeAssumedRole,
+		AssumedRoleARN:    "arn:aws:sts::" + accountID + ":assumed-role/role/" + sessionName,
+		UnderlyingRoleARN: underlyingRoleARN,
+		SessionName:       sessionName,
+		ExpiresAt:         time.Now().UTC().Add(time.Hour),
+		CreatedAt:         time.Now().UTC().Add(-time.Minute),
+	}
+}

internal/testkit/gwfix/verifier.go

@@ -0,0 +1,44 @@
+package gwfix
+
+import (
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
+)
+
+var _ credential.Verifier = (*Verifier)(nil)
+
+// Verifier authenticates a long-lived access key as the IAM user the key names.
+// It decides nothing else: no status, session or account check, so a test that
+// asserts a credential verdict must use the real verifier instead.
+type Verifier struct {
+	IAM iamdomain.IAMService
+}
+
+func (v *Verifier) Authenticate(req credential.Request, verifySig func(secret string) error) (credential.Authenticated, *credential.Failure) {
+	ak, err := v.IAM.LookupAccessKey(req.AccessKeyID)
+	if err != nil {
+		return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorInvalidClientTokenId}
+	}
+	secret, err := v.IAM.DecryptSecret(ak.SecretAccessKey)
+	if err != nil {
+		return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorInternalError}
+	}
+	if err := verifySig(secret); err != nil {
+		return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorSignatureDoesNotMatch, SignatureMismatch: true}
+	}
+	return credential.Authenticated{AccountID: ak.AccountID, Handle: ak}, nil
+}
+
+func (v *Verifier) Resolve(_ credential.Request, auth credential.Authenticated) (credential.Principal, *credential.Failure) {
+	ak, ok := auth.Handle.(*iamdomain.AccessKey)
+	if !ok {
+		return credential.Principal{}, &credential.Failure{Code: awserrors.ErrorInternalError}
+	}
+	return credential.Principal{
+		Identity:      ak.UserName,
+		AccountID:     ak.AccountID,
+		PrincipalType: PrincipalTypeUser,
+		CallerARN:     "arn:aws:iam::" + ak.AccountID + ":user/" + ak.UserName,
+	}, nil
+}

internal/testkit/gwtest/assert.go

@@ -0,0 +1,51 @@
+package gwtest
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// AssertDenied matches on the resolved code, not the message: a policy denial
+// names the principal, action and resource ahead of the code.
+func AssertDenied(t *testing.T, err error) {
+	t.Helper()
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "unresolvable error code: %v", err)
+	assert.Equal(t, awserrors.ErrorAccessDenied, code)
+}
+
+// AssertUnauthorized asserts EC2's policy denial, UnauthorizedOperation.
+func AssertUnauthorized(t *testing.T, err error) {
+	t.Helper()
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorUnauthorizedOperation, err.Error())
+}
+
+// AssertPermitted asserts the policy gate passed. The request then fails on the
+// absent NATS connection rather than on authorization, so a denial cannot hide
+// behind a not-found.
+func AssertPermitted(t *testing.T, err error) {
+	t.Helper()
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+// AssertGatePassed asserts the policy gate ran and allowed the request, whatever
+// the handler then did with it. A denial cannot hide behind a later error.
+func (h *Harness) AssertGatePassed(t *testing.T) {
+	t.Helper()
+	require.True(t, h.GateCalled, "policy gate was never consulted")
+	require.NoError(t, h.LastGateErr)
+}
+
+// AssertNotDenied asserts the request failed, but not with a policy denial.
+func AssertNotDenied(t *testing.T, err error) {
+	t.Helper()
+	require.Error(t, err)
+	code, _ := awserrors.ResolveErrorCode(err)
+	assert.NotEqual(t, awserrors.ErrorAccessDenied, code)
+}

internal/testkit/gwtest/gwtest.go

@@ -0,0 +1,126 @@
+// Package gwtest builds the invocation a registered AWS service receives from
+// the ingress pipeline, so a service's handler tests run in the service's own
+// package with real policy evaluation and no gateway.
+package gwtest
+
+import (
+	"net/http"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
+)
+
+const (
+	// AccountID is the account the default caller belongs to.
+	AccountID = "123456789012"
+	// Region is the region the default harness serves.
+	Region = "ap-southeast-2"
+)
+
+// policyIAM resolves fixed policy documents for every user and role. Only the
+// two methods the policy evaluator reads are implemented; any other call on
+// the embedded nil interface panics, which is the point.
+type policyIAM struct {
+	iamdomain.IAMService
+
+	resolve func(accountID, name string) ([]iamdomain.PolicyDocument, error)
+}
+
+func (p policyIAM) GetUserPolicies(accountID, name string) ([]iamdomain.PolicyDocument, error) {
+	return p.resolve(accountID, name)
+}
+
+func (p policyIAM) GetRolePolicies(accountID, name string) ([]iamdomain.PolicyDocument, error) {
+	return p.resolve(accountID, name)
+}
+
+// Statement is one policy statement naming a single action and resource.
+func Statement(effect, action, resource string) iamdomain.Statement {
+	return iamdomain.Statement{
+		Effect:   effect,
+		Action:   iamdomain.StringOrArr{action},
+		Resource: iamdomain.StringOrArr{resource},
+	}
+}
+
+// Harness invokes a registration as the pipeline would for one authenticated
+// user, authorizing through the real policy evaluator.
+type Harness struct {
+	// AccountID is the caller's account as the invocation carries it.
+	AccountID string
+	Region    string
+	Policy    pipeline.PolicyEvaluator
+	// Caller overrides the default caller, an IAM user named alice.
+	Caller *credential.Principal
+	// LastGateErr is the verdict of the most recent Authorize call, and
+	// GateCalled whether the last Dispatch reached the gate at all.
+	LastGateErr error
+	GateCalled  bool
+}
+
+// Scoped returns a harness whose caller holds one policy of statements.
+func Scoped(statements ...iamdomain.Statement) *Harness {
+	return FromDocs([]iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: statements}})
+}
+
+// FromDocs returns a harness whose caller holds exactly these policy documents.
+func FromDocs(docs []iamdomain.PolicyDocument) *Harness {
+	return FromResolver(func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil })
+}
+
+// FromResolver returns a harness whose caller's policies come from resolve,
+// which sees every user or role lookup, so a test can count them.
+func FromResolver(resolve func(accountID, name string) ([]iamdomain.PolicyDocument, error)) *Harness {
+	return &Harness{
+		AccountID: AccountID,
+		Region:    Region,
+		Policy:    policyauth.New(policyIAM{resolve: resolve}),
+	}
+}
+
+// AllowAll returns a harness whose caller may do anything.
+func AllowAll() *Harness {
+	return Scoped(Statement("Allow", "*", "*"))
+}
+
+// Dispatch hands r to reg as an invocation for service, authorizing through
+// the pipeline's own gate over h's policy.
+func (h *Harness) Dispatch(reg dispatch.Registration, service string, w http.ResponseWriter, r *http.Request) error {
+	caller := credential.Principal{
+		AccountID:     h.AccountID,
+		Identity:      "alice",
+		PrincipalType: stsdomain.PrincipalTypeUser,
+		CallerARN:     "arn:aws:iam::" + h.AccountID + ":user/alice",
+	}
+	if h.Caller != nil {
+		caller = *h.Caller
+	}
+	h.LastGateErr, h.GateCalled = nil, false
+	return reg.Dispatch(w, dispatch.Invocation{
+		Request:   r,
+		AccountID: h.AccountID,
+		Service:   service,
+		Region:    h.Region,
+		Actor:     func() (dispatch.Actor, error) { return pipeline.ActorFor(caller, "") },
+		Authorize: func(svc, action string, resources []string, actionKeys iampolicy.ConditionKeys) error {
+			h.GateCalled = true
+			h.LastGateErr = pipeline.Authorize(h.Policy, r, caller, true, svc, action, resources, actionKeys)
+			return h.LastGateErr
+		},
+	})
+}
+
+// PassRoleTo is a PassRole statement on every role, conditioned on the
+// consuming service the way AWS's documented examples scope it.
+func PassRoleTo(effect, operator, service string) iamdomain.Statement {
+	s := Statement(effect, "iam:PassRole", "*")
+	s.Condition = map[string]map[string]iamdomain.ConditionValue{
+		operator: {iampolicy.KeyPassedToService: {service}},
+	}
+	return s
+}

scripts/README.md

@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 
 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
@@ -86,6 +86,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 | `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. |
 | `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). |
 | `sync-aws-models.sh` | Replaces the embedded AWS Smithy models under `internal/awsmodel/models/` with the ten service files from `aws/api-models-aws` at the given commit, and records that commit in `internal/awsmodel/model_source.go`. Takes a full 40-character SHA. |
+| `ui-playwright.sh compare\|baselines [args...]` | Runs the console's Playwright suite inside the image pinned in the frontend's `playwright.image`, the one CI's `UI End-to-End` job uses, with podman or docker. `compare` checks screenshots as CI does and `baselines` redraws them; extra arguments go to `pnpm e2e`. Installs into its own `node_modules` under `~/.cache/spinifex-ui-playwright`, leaving the host's alone. `make ui-e2e-container` and `make ui-baselines` run it. |
 | `run-bench.sh` | Launches a benchmark against running Ubuntu instances, auto-detecting the AMI for the host architecture. Tracks nbdkit and perf alongside the run. |
 | `disk-performance.sh` | Runs a `fio` random 70/30 read-write benchmark swept across 4k/16k/128k/1M. Writes JSON results to `/tmp/spinifex-disk-bench`; override with `BENCH_DIR`, `OUT_DIR`, `SIZE`, `JOBS`, `BLOCK_SIZES`. |
 | `network-performance.sh` | iperf throughput from several clients to one server, driven over SSH, writing a `summary.txt` of Gbit/s per client. `--server-ip` sets the address clients dial, so pointing it at instance private addresses measures the VPC overlay rather than the external pool. |
@@ -129,6 +130,7 @@ Installed into the EKS node AMI and run at boot or on a timer. All host communic
 | `k3s-first-boot.sh` | Runs once after the server is healthy: reads the node-token and admin kubeconfig, rewrites the kubeconfig server address to the cluster NLB endpoint, and publishes both to the host. |
 | `konnectivity-server-prestart.sh` | Waits for the K3s CA and admin kubeconfig, mints the konnectivity serving cert, and writes the apiserver replica count for the caller to source. |
 | `mulga-eks-addon-sync.sh` | Pulls staged managed-addon descriptors through the gateway, renders each bundle into the K3s auto-deploy dir, and reports per-addon delivery status back. |
+| `eks-etcd-member` | Go helper built from `cmd/eks-etcd-member` and installed on control-plane (server) images. `remove --peer-ip <ip> [--self-ip <ip>]` evicts a terminated peer's etcd member over the local client endpoint, because control planes run no agent and k3s never prunes them. First boot calls it for a replacement control plane and treats an absent member as success. |
 | `mulga-eks-etcd-snapshot.sh` | Snapshots K3s embedded etcd to predastore. Installed into two crond periodic dirs and picks its tier — `frequent` or `daily` — from the dir it was invoked from. |
 | `mulga-eks-k3s-recovery.sh` | Applies a control-plane etcd-quorum recovery directive before k3s starts. Pull-based and a no-op unless the reconciler escalated a wedged control plane. |
 | `mulga-eks-state-report.sh` | Publishes server health (`healthz`, node count) to the cluster reconciler. The apiserver binds the VPC node-ip, which the host daemon cannot probe directly. |

scripts/appliance/firstboot.sh

@@ -90,6 +90,12 @@ else
     log "WARNING: clock not synced, continuing with $(date -uIs)"
 fi
 
+# ── Connection tracking ─────────────────────────────────────────────────────
+# Before OVN opens any link, so arming the firewall later cannot drop one the
+# kernel first saw mid-stream. Non-fatal; the unit reloads it on later boots.
+/usr/local/lib/spinifex/spinifex-firewall-apply track \
+    || log "WARNING: connection tracking not loaded; arming the firewall may drop OVN links"
+
 # ── OVS and OVN ─────────────────────────────────────────────────────────────
 # Start them before setup-ovn.sh so their databases exist when it runs: on
 # physical hardware first-boot DB initialisation outlasts its own timeout.

scripts/build-sqs-store-microvm-image.sh

@@ -5,6 +5,10 @@
 # sqs-store/sqs-store-lib.sh states its fw_cfg contract. The LB image
 # (build-microvm-image.sh) is separate and unchanged.
 #
+# SQS_MICROVM_PROFILE=worker builds the shared SQS worker guest instead: the
+# same network /init and kernel, build/microvm/sqs-worker's steps and
+# $REPO_ROOT/bin/sqs-worker, with no nats-server.
+#
 # Inputs:
 #   $REPO_ROOT/build/microvm/sqs-store/init.sh, inittab
 #   $REPO_ROOT/build/microvm/sqs-store/sqs-store-{lib.sh,prepare,run,stop}
@@ -26,6 +30,18 @@ INIT_SH="$BUILD_DIR/init.sh"
 INITTAB="$BUILD_DIR/inittab"
 NATS_SERVER_BIN="$REPO_ROOT/bin/nats-server"
 
+PROFILE="${SQS_MICROVM_PROFILE:-store}"
+case "$PROFILE" in
+    store) ;;
+    worker)
+        BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
+        INITTAB="$BUILD_DIR/inittab"
+        SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
+        ;;
+    *) echo "ERROR: SQS_MICROVM_PROFILE must be store or worker, not $PROFILE" >&2; exit 1 ;;
+esac
+TAG="[build-sqs-$PROFILE-microvm-image]"
+
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
 ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
 # e2fsprogs: mkfs.ext4/e2fsck for a guest that mounts an ext4 data volume.
@@ -36,8 +52,8 @@ ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
 # button (QMP system_powerdown) into a signal busybox init handles as poweroff.
 MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
 
-echo "[build-sqs-store-microvm-image] repo root: $REPO_ROOT"
-echo "[build-sqs-store-microvm-image] build dir: $BUILD_DIR"
+echo "$TAG repo root: $REPO_ROOT"
+echo "$TAG build dir: $BUILD_DIR"
 mkdir -p "$BUILD_DIR"
 
 # --- Verify non-negotiable tools ---
@@ -52,6 +68,16 @@ for tool in cpio gzip find fakeroot; do
     fi
 done
 
+if [ "$PROFILE" = worker ]; then
+    if [ ! -x "$SQS_WORKER_BIN" ]; then
+        echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
+        exit 1
+    fi
+    if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
+        echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
+        exit 1
+    fi
+else
 # --- Verify the dedicated nats-server is the go.mod-pinned build ---
 # A stale or host-installed binary would ship an unreviewed server version.
 NATS_SERVER_PIN=$(awk '$1 == "github.com/nats-io/nats-server/v2" { print $2; exit }' "$REPO_ROOT/go.mod")
@@ -72,7 +98,8 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
     echo "ERROR: $NATS_SERVER_BIN reports $nats_server_version, go.mod pins $NATS_SERVER_PIN" >&2
     exit 1
 fi
-echo "[build-sqs-store-microvm-image] nats-server: $nats_server_version (matches go.mod)"
+echo "$TAG nats-server: $nats_server_version (matches go.mod)"
+fi
 
 # --- Create chroot ---
 CHROOT_DIR=$(mktemp -d)
@@ -88,11 +115,11 @@ STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
 CONTAINER_TOOL=""
 CONTAINER_CID=""
 cleanup() {
-    echo "[build-sqs-store-microvm-image] cleaning up chroot: $CHROOT_DIR"
+    echo "$TAG cleaning up chroot: $CHROOT_DIR"
     rm -rf "$CHROOT_DIR"
     rm -rf "${STAGE_DIR:?}"
     if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
-        echo "[build-sqs-store-microvm-image] cleaning up container: $CONTAINER_CID"
+        echo "$TAG cleaning up container: $CONTAINER_CID"
         $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
     fi
 }
@@ -101,7 +128,7 @@ trap cleanup EXIT
 # --- Populate rootfs ---
 # Strategy 1: native apk (Alpine hosts, CI)
 if command -v apk >/dev/null 2>&1; then
-    echo "[build-sqs-store-microvm-image] rootfs: native apk"
+    echo "$TAG rootfs: native apk"
     mkdir -p \
         "$CHROOT_DIR/etc/apk" \
         "$CHROOT_DIR/lib/apk/db" \
@@ -125,7 +152,7 @@ EOF
 # this grows without bound. `buildx --output type=local` writes the image
 # filesystem straight to a directory and releases every mount, so nothing leaks.
 elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
-    echo "[build-sqs-store-microvm-image] rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
+    echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
     BUILD_CTX=$(mktemp -d)
     cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
 FROM alpine:${ALPINE_VERSION}
@@ -140,7 +167,7 @@ DOCKERFILE
 # Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
 # overlay2 export-mount leak, and podman has no buildx.
 elif command -v podman >/dev/null 2>&1; then
-    echo "[build-sqs-store-microvm-image] rootfs: podman export (alpine:${ALPINE_VERSION})"
+    echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
     CONTAINER_TOOL=podman
     CONTAINER_CID=$(podman run -d \
         "alpine:${ALPINE_VERSION}" \
@@ -170,7 +197,7 @@ rm -rf "${CHROOT_DIR:?}/dev"
 mkdir "$CHROOT_DIR/dev"
 
 # --- Place init script ---
-echo "[build-sqs-store-microvm-image] installing init.sh as /init..."
+echo "$TAG installing init.sh as /init..."
 if [ ! -f "$INIT_SH" ]; then
     echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
     exit 1
@@ -183,11 +210,26 @@ if [ ! -f "$INITTAB" ]; then
     echo "ERROR: $INITTAB not found — cannot build initramfs" >&2
     exit 1
 fi
-echo "[build-sqs-store-microvm-image] installing inittab..."
+echo "$TAG installing inittab..."
 install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
 
+if [ "$PROFILE" = worker ]; then
+    echo "$TAG installing worker guest steps..."
+    for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop; do
+        if [ ! -f "$BUILD_DIR/$step" ]; then
+            echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
+            exit 1
+        fi
+    done
+    mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
+    install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
+        "$CHROOT_DIR/usr/local/lib/sqs-worker/"
+    install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
+    # The mountpoint ships empty; the worker proves its disk is mounted here.
+    mkdir -p "$CHROOT_DIR/data"
+else
 # --- Place the store's guest steps ---
-echo "[build-sqs-store-microvm-image] installing store guest steps..."
+echo "$TAG installing store guest steps..."
 mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-store"
 for step in sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop; do
     if [ ! -f "$BUILD_DIR/$step" ]; then
@@ -200,9 +242,10 @@ install -m 0755 "$BUILD_DIR/sqs-store-prepare" "$BUILD_DIR/sqs-store-run" "$BUIL
     "$CHROOT_DIR/usr/local/lib/sqs-store/"
 
 # --- Copy nats-server binary ---
-echo "[build-sqs-store-microvm-image] copying nats-server binary..."
+echo "$TAG copying nats-server binary..."
 mkdir -p "$CHROOT_DIR/usr/local/bin"
 install -m 0755 "$NATS_SERVER_BIN" "$CHROOT_DIR/usr/local/bin/nats-server"
+fi
 
 # --- Create required directories for init ---
 mkdir -p \
@@ -217,7 +260,7 @@ if [ -z "$KERNEL_IMG" ]; then
     echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
     exit 1
 fi
-echo "[build-sqs-store-microvm-image] kernel image: $KERNEL_IMG"
+echo "$TAG kernel image: $KERNEL_IMG"
 
 KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
 KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
@@ -231,9 +274,9 @@ if ! find "$CHROOT_DIR/lib/modules" \
         echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
         exit 1
     fi
-    echo "[build-sqs-store-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+    echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
 else
-    echo "[build-sqs-store-microvm-image] fw_cfg: module found in lib/modules"
+    echo "$TAG fw_cfg: module found in lib/modules"
 fi
 
 # --- Assert ext4 userspace tools (data volume) ---
@@ -243,7 +286,7 @@ for fs_tool in mkfs.ext4 e2fsck; do
         exit 1
     fi
 done
-echo "[build-sqs-store-microvm-image] e2fsprogs: mkfs.ext4 + e2fsck present"
+echo "$TAG e2fsprogs: mkfs.ext4 + e2fsck present"
 
 # --- Strip kernel modules (Phase 2 keep-list approach) ---
 # Only modules needed by a virtio-mmio microvm guest are kept:
@@ -282,7 +325,7 @@ image_kmod() {
         --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
 }
 
-echo "[build-sqs-store-microvm-image] stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
+echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
 for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     kver="$(basename "$kver_dir")"
     kernel_dir="$kver_dir/kernel"
@@ -381,8 +424,8 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
             exit 1
         fi
     done
-    echo "[build-sqs-store-microvm-image] extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
-    echo "[build-sqs-store-microvm-image] modules kept: $(echo "$kept_names" | tr '\n' ' ')"
+    echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
+    echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
 
     # Rebuild module metadata with the image's own kmod, the binary the guest's
     # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
@@ -415,12 +458,12 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
             echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
             exit 1
         done <<< "$plan"
-        echo "[build-sqs-store-microvm-image] modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
+        echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
     done
 done
 
 # --- Strip package-manager artifacts (not needed at runtime) ---
-echo "[build-sqs-store-microvm-image] stripping package manager artifacts..."
+echo "$TAG stripping package manager artifacts..."
 rm -rf \
     "$CHROOT_DIR/lib/apk" \
     "$CHROOT_DIR/var/cache/apk" \
@@ -434,7 +477,7 @@ rm -rf \
 # --- Strip debug symbols from binaries ---
 # nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
 # are the ones whose version was checked above.
-echo "[build-sqs-store-microvm-image] stripping debug symbols from binaries..."
+echo "$TAG stripping debug symbols from binaries..."
 find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
      "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
     -type f 2>/dev/null | while read -r bin; do
@@ -443,7 +486,7 @@ done
 
 # --- Copy vmlinuz out before stripping /boot from chroot ---
 VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
-echo "[build-sqs-store-microvm-image] copying kernel: $VMLINUZ_OUT"
+echo "$TAG copying kernel: $VMLINUZ_OUT"
 cp "$KERNEL_IMG" "$VMLINUZ_OUT"
 
 # Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
@@ -456,7 +499,7 @@ rm -rf "${CHROOT_DIR:?}/boot"
 
 # --- Build initramfs ---
 INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
-echo "[build-sqs-store-microvm-image] building initramfs: $INITRAMFS_OUT"
+echo "$TAG building initramfs: $INITRAMFS_OUT"
 # Create the static device nodes and pack the archive inside a single fakeroot
 # session: mknod/cpio see faked device entries and write real char-device
 # records into the cpio, while the host filesystem never gets an actual node.
@@ -470,6 +513,6 @@ fakeroot sh -c '
 # --- Gate the staged pair, then publish it ---
 # Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot
 # leave a pair the grouped make target would take as up to date.
-"$REPO_ROOT/scripts/lib/sqs-store-microvm-publish.sh" "$STAGE_DIR" "$BUILD_DIR"
+"$REPO_ROOT/scripts/lib/sqs-store-microvm-publish.sh" "$STAGE_DIR" "$BUILD_DIR" "$PROFILE"
 echo ""
-echo "[build-sqs-store-microvm-image] done."
+echo "$TAG done."

scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml

@@ -24939,11 +24939,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-applicationset-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25191,11 +25189,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-dex-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25326,11 +25322,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-notifications-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25436,11 +25430,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-redis
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25530,11 +25522,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-repo-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25895,11 +25885,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -26317,11 +26305,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-application-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml

@@ -44,12 +44,9 @@ spec:
         kubernetes.io/os: linux
       serviceAccountName: ebs-csi-controller-sa
       priorityClassName: system-cluster-critical
-      # A cluster with a GPU-only nodegroup has no untainted node: the CP carries
-      # CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
-      # worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
-      # tolerations the controller has nowhere to schedule and stays Pending
-      # forever. Real EKS never needs this: its control plane isn't a schedulable
-      # node, so the stock upstream manifest carries no such toleration.
+      # Kept for clusters created before control-plane hosts stopped registering
+      # as Nodes (they carried CriticalAddonsOnly), and for GPU-only nodegroups
+      # whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml

@@ -348,12 +348,9 @@ spec:
           readOnly: true
 {{IRSA_VOLUME_MOUNT}}
       priorityClassName: system-cluster-critical
-      # A cluster with a GPU-only nodegroup has no untainted node: the CP carries
-      # CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
-      # worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
-      # tolerations the controller has nowhere to schedule and stays Pending
-      # forever. Real EKS never needs this: its control plane isn't a schedulable
-      # node, so the stock upstream manifest carries no such toleration.
+      # Kept for clusters created before control-plane hosts stopped registering
+      # as Nodes (they carried CriticalAddonsOnly), and for GPU-only nodegroups
+      # whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/k3s-first-boot.sh

@@ -162,12 +162,9 @@ jq -n --arg k "${KUBECONFIG_REWRITTEN}" '{adminKubeconfig: $k}'      | publish_e
 jq -n --arg j "${JWKS}"                 '{jwks: $j}'                  | publish_envelope k3s-oidc-jwks
 jq -n --arg c "${CA_B64}"               '{certificateAuthority: $c}' | publish_envelope k3s-ca
 
-# 3.5 Prune a terminated control-plane peer this VM replaces. The spinifex
-# reconciler sets EKS_ETCD_PRUNE_PEER_IP to the dead member's node IP when this
-# VM is provisioned as its replacement (member-count reconcile). Deleting the
-# dead Node makes k3s embedded-etcd remove the stale member, so quorum width
-# returns to N rather than N+1-with-a-dead-peer. Best-effort: a failure leaves
-# the member for an operator and never blocks first boot.
+# 3.5 Prune the terminated CP peer this VM replaces: delete its Node if it still
+# has one, else remove its etcd member directly (CPs run no agent). Best-effort:
+# a failure is logged and never blocks first boot.
 if [ -n "${EKS_ETCD_PRUNE_PEER_IP:-}" ]; then
     log "pruning terminated CP peer at ${EKS_ETCD_PRUNE_PEER_IP} from etcd"
     dead_node=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get nodes \
@@ -181,7 +178,15 @@ if [ -n "${EKS_ETCD_PRUNE_PEER_IP:-}" ]; then
             log "WARN: delete of dead node ${dead_node} failed; etcd member left for operator"
         fi
     else
-        log "no node has internal IP ${EKS_ETCD_PRUNE_PEER_IP}; nothing to prune"
+        # The default-route source address is the address k3s advertises to etcd peers.
+        self_ip=$(ip -4 route get 1.1.1.1 2>/dev/null \
+            | awk '{for (i = 1; i < NF; i++) if ($i == "src") {print $(i + 1); exit}}') || self_ip=""
+        if member_out=$(eks-etcd-member remove --peer-ip "${EKS_ETCD_PRUNE_PEER_IP}" \
+            ${self_ip:+--self-ip "${self_ip}"} 2>&1); then
+            log "${member_out}"
+        else
+            log "WARN: etcd prune failed for peer ${EKS_ETCD_PRUNE_PEER_IP}: ${member_out}; member left for operator"
+        fi
     fi
 fi
 

scripts/images/eks-node/k3s-first-boot_test.sh

@@ -0,0 +1,123 @@
+#!/bin/sh
+# Test for the etcd peer-prune step (3.5) of k3s-first-boot.sh: kubectl and the helpers
+# are stubbed on PATH (jq required) and state paths rewritten into a temp dir.
+# Run: sh scripts/images/eks-node/k3s-first-boot_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SRC="${SCRIPT_DIR}/k3s-first-boot.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+pass() { PASS=$((PASS + 1)); echo "ok   - $1"; }
+fail() { FAIL=$((FAIL + 1)); echo "FAIL - $1"; }
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+# kubectl stub: readiness and JWKS succeed; "get nodes" returns the fixture;
+# "delete node" is recorded so the test can see which Node the script removed.
+cat > "${STUBBIN}/kubectl" <<'EOF'
+#!/bin/sh
+case "$*" in
+    *"--raw=/readyz"*)         echo ok ;;
+    *"--raw=/openid/v1/jwks"*) echo '{"keys":[]}' ;;
+    *"get nodes"*)             printf '%s' "${NODES_BODY:-}" ;;
+    *"delete node"*)           echo "$*" >> "${DELETE_LOG}" ;;
+    *) : ;;
+esac
+EOF
+
+# eks-gateway-publish stub: swallow the envelope.
+cat > "${STUBBIN}/eks-gateway-publish" <<'EOF'
+#!/bin/sh
+cat > /dev/null
+EOF
+
+# eks-etcd-member stub: record argv; exit status is configurable so the
+# best-effort contract (a helper failure never blocks first boot) can be tested.
+cat > "${STUBBIN}/eks-etcd-member" <<'EOF'
+#!/bin/sh
+echo "$*" >> "${MEMBER_LOG}"
+exit "${MEMBER_RC:-0}"
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+command -v jq >/dev/null 2>&1 || { echo "SKIP - jq not installed"; exit 0; }
+
+# Rewrite the script's absolute state paths into the sandbox.
+mkdir -p "${WORK}/state" "${WORK}/etc" "${WORK}/k3s/server" "${WORK}/rancher"
+sed -e "s|/var/lib/spinifex-eks|${WORK}/state|g" \
+    -e "s|/etc/spinifex-eks|${WORK}/etc|g" \
+    -e "s|/var/lib/rancher/k3s/server|${WORK}/k3s/server|g" \
+    -e "s|/etc/rancher/k3s|${WORK}/rancher|g" \
+    "${SRC}" > "${WORK}/first-boot.sh"
+
+cat > "${WORK}/rancher/k3s.yaml" <<'EOF'
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: Q0FEQVRB
+    server: https://127.0.0.1:6443
+EOF
+echo node-token > "${WORK}/k3s/server/node-token"
+
+# run_case <prune-ip-or-empty>: fresh sentinel + env, then run the script copy.
+run_case() {
+    prune_ip=$1
+    : > "${WORK}/state/first-boot.pending"
+    {
+        echo "EKS_GATEWAY_URL=https://gw.invalid:9999"
+        echo "EKS_ACCOUNT_ID=000000000001"
+        echo "EKS_CLUSTER_NAME=demo"
+        echo "EKS_NLB_ENDPOINT=https://demo.invalid"
+        [ -z "${prune_ip}" ] || echo "EKS_ETCD_PRUNE_PEER_IP=${prune_ip}"
+    } > "${WORK}/etc/first-boot.env"
+    : > "${WORK}/delete.log"
+    : > "${WORK}/member.log"
+    DELETE_LOG="${WORK}/delete.log" MEMBER_LOG="${WORK}/member.log" \
+        EKS_NODE_INIT=none sh "${WORK}/first-boot.sh" > "${WORK}/out.log" 2>&1
+}
+
+# --- Case 1: dead peer has no Node (all-CPs-without-agent steady state) -> the
+# --- explicit member-remove helper is called with the peer IP.
+NODES_BODY="" run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "no-node: first boot exits 0" || fail "no-node: exit ${rc}"
+[ ! -s "${WORK}/delete.log" ] && pass "no-node: no Node delete attempted" || fail "no-node: unexpected delete: $(cat "${WORK}/delete.log")"
+grep -q -- 'remove --peer-ip 10.0.0.12' "${WORK}/member.log" \
+    && pass "no-node: eks-etcd-member remove --peer-ip 10.0.0.12 called" \
+    || fail "no-node: member-remove helper not called with the peer IP (member.log: $(cat "${WORK}/member.log"))"
+
+# --- Case 2: dead peer still has a Node (mixed fleet) -> the Node is deleted so
+# --- k3s drops the member itself.
+NODES_BODY="ip-10-0-0-12 10.0.0.12" run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "node-present: first boot exits 0" || fail "node-present: exit ${rc}"
+grep -q 'delete node ip-10-0-0-12' "${WORK}/delete.log" \
+    && pass "node-present: dead Node deleted" \
+    || fail "node-present: dead Node not deleted (delete.log: $(cat "${WORK}/delete.log"))"
+
+# --- Case 3: not a replacement (no prune IP) -> neither path runs.
+NODES_BODY="" run_case "" && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "no-prune: first boot exits 0" || fail "no-prune: exit ${rc}"
+{ [ ! -s "${WORK}/delete.log" ] && [ ! -s "${WORK}/member.log" ]; } \
+    && pass "no-prune: neither delete nor member-remove ran" \
+    || fail "no-prune: unexpected prune activity"
+
+# --- Case 4: the helper fails -> best effort, first boot still completes (the
+# --- sentinel is consumed) and the helper was actually attempted.
+NODES_BODY="" MEMBER_RC=1 run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "helper-fails: first boot still exits 0" || fail "helper-fails: exit ${rc}"
+[ ! -e "${WORK}/state/first-boot.pending" ] && pass "helper-fails: sentinel consumed" || fail "helper-fails: sentinel left behind"
+[ -s "${WORK}/member.log" ] && pass "helper-fails: helper was attempted" || fail "helper-fails: helper never attempted"
+grep -Eq 'prune.*fail.*10\.0\.0\.12' "${WORK}/out.log" \
+    && pass "helper-fails: prune failure logged with the peer IP" \
+    || fail "helper-fails: no prune-failure line naming 10.0.0.12 (out.log: $(cat "${WORK}/out.log"))"
+
+echo
+echo "passed=${PASS} failed=${FAIL}"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/konnectivity/konnectivity-agent.yaml

@@ -7,14 +7,9 @@
 # the K3s auto-deploy dir by k3s.initd, which fills the __KONN_HOST__ /
 # __KONN_AGENT_IMAGE__ tokens at boot.
 #
-# Also tolerates the control-plane CriticalAddonsOnly=true:NoExecute taint and
-# the GPU nvidia.com/gpu=present:NoSchedule taint, matching every other
-# CP-critical addon in this repo (aws-load-balancer-controller, ebs-csi-driver,
-# argocd): a zero-worker or GPU-only cluster has no untainted node, and other
-# addons already schedule onto the CP, so it needs its own tunnel too — without
-# an agent there, kubectl logs/exec against CP-hosted pods and apiserver
-# aggregated-API egress (metrics-server discovery) both fail. Real EKS never
-# needs this: its control plane isn't a schedulable node.
+# Also tolerates CriticalAddonsOnly, for clusters created before control-plane
+# hosts stopped registering as Nodes, and the GPU nvidia.com/gpu taint so a
+# GPU-only nodegroup still gets a tunnel agent.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -44,11 +39,9 @@ spec:
     spec:
       serviceAccountName: konnectivity-agent
       priorityClassName: system-node-critical
-      # Without these, a zero-worker or GPU-only cluster (CP tainted
-      # CriticalAddonsOnly, workers tainted nvidia.com/gpu) shows DESIRED=0 for
-      # this DaemonSet: no tunnel exists, so kubectl logs/exec against any pod
-      # fails "No agent available" and apiserver aggregated-API egress
-      # (e.g. metrics-server discovery) fails, which stalls namespace GC.
+      # CriticalAddonsOnly is kept for clusters created before control-plane hosts
+      # stopped registering as Nodes. nvidia.com/gpu keeps the agent on GPU-only
+      # nodegroups, where logs/exec and aggregated-API egress need a tunnel.
       tolerations:
         - key: CriticalAddonsOnly
           operator: Exists

scripts/images/eks-node/manifest.conf

@@ -22,9 +22,9 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-
 # at a pinned tag (its GitHub releases ship only container images, and its go.mod
 # carries replace directives that block `go install pkg@version`). The agent half
 # is the matching proxy-agent container image, pinned in k3s.initd.
-BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-etcd-member ./cmd/eks-etcd-member && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
 
-INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
+INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/eks-etcd-member:/usr/local/bin/eks-etcd-member bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
 
 # All role services ship; none but the selector are enabled at bake time.
 # eks-node-role picks server vs agent at first boot and enables the rest.

scripts/lib/sqs-store-guest_test.sh

@@ -56,6 +56,7 @@ EOF
 cat > "$TMP/bin/nats-server" <<'EOF'
 #!/bin/sh
 echo "nats-server $*" >> "$T/calls.log"
+[ "$1" = -t ] || echo "GOMEMLIMIT=${GOMEMLIMIT:-unset}" >> "$T/env.log"
 [ "$1" = -t ] && exit "$(cat "$T/configtest.rc")"
 exit 0
 EOF
@@ -110,6 +111,8 @@ check "config listens only on the supplied address" \
 check "config admits only the supplied user NKey" 'grep -q "users = \[ { nkey: \"$NKEY\" } \]" "$T/run/nats.conf"'
 check "config syncs JetStream on the data disk" \
     'grep -q "store_dir: \"$T/data\"" "$T/run/nats.conf" && grep -q "sync_interval: always" "$T/run/nats.conf"'
+check "config admits a full SQS message (2MiB max_payload)" \
+    'grep -c "max_payload" "$T/run/nats.conf" | grep -qx 1 && grep -qx "max_payload: 2MiB" "$T/run/nats.conf"'
 check "config is tested by the server before ready" 'grep -qx "nats-server -t -c $T/run/nats.conf" "$T/calls.log"'
 check "no auth material on the console" '! grep -q "$NKEY" "$T/console"'
 check "prepare reports the disk and config, not a listener" 'grep -q "prepared: $T/dev/vdb mounted" "$T/console" && ! grep -qi "listening" "$T/console"'
@@ -188,6 +191,8 @@ run_step() {
 }
 fresh run-ready; prepare; : > "$T/calls.log"
 check "a prepared store starts the server" 'run_step && grep -qx "nats-server -c $T/run/nats.conf" "$T/calls.log"'
+fresh run-memlimit; prepare; : > "$T/env.log"
+check "the server runs under the ADR-0011:S11 heap limit" 'run_step && grep -qx "GOMEMLIMIT=512MiB" "$T/env.log"'
 fresh run-unready
 check "an unprepared store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
 fresh run-stopping; prepare; : > "$T/calls.log"; : > "$T/run/stopping"

scripts/lib/sqs-store-microvm-publish.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
-# sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR — gate a staged SQS store
-# microVM kernel + initramfs and move them into OUT_DIR only if every gate passes.
+# sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR [PROFILE] — gate a staged SQS
+# store (default) or worker microVM kernel + initramfs and move them into
+# OUT_DIR only if every gate passes.
 #
 # A failed gate leaves OUT_DIR as it was: a previous valid pair stays, and an
 # older pair stays older than the build inputs, so make rebuilds next time.
@@ -10,7 +11,15 @@ STAGE_DIR="${1:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
 OUT_DIR="${2:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
 VMLINUZ="$STAGE_DIR/vmlinuz"
 INITRAMFS="$STAGE_DIR/initramfs.cpio.gz"
-tag="[build-sqs-store-microvm-image]"
+PROFILE="${3:-store}"
+case "$PROFILE" in
+    store)
+        payload="usr/local/bin/nats-server usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop" ;;
+    worker)
+        payload="usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
+    *) echo "ERROR: profile must be store or worker, not $PROFILE" >&2; exit 1 ;;
+esac
+tag="[build-sqs-$PROFILE-microvm-image]"
 
 for f in "$VMLINUZ" "$INITRAMFS"; do
     if [ ! -s "$f" ]; then
@@ -24,9 +33,8 @@ if ! archive_list=$(zcat "$INITRAMFS" | cpio --quiet -t); then
     echo "ERROR: $INITRAMFS is not a readable gzip cpio archive" >&2
     exit 1
 fi
-for want in init etc/inittab usr/local/bin/nats-server dev/console dev/null \
-    usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
-    usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop; do
+# shellcheck disable=SC2086 # payload is a word list on purpose
+for want in init etc/inittab dev/console dev/null $payload; do
     if ! grep -qx "$want" <<< "$archive_list"; then
         echo "ERROR: $want missing from $INITRAMFS" >&2
         exit 1
@@ -51,6 +59,11 @@ if grep -qE '(^|/)(lb-agent|haproxy|nginx)$' <<< "$archive_list"; then
     echo "ERROR: LB data-plane binaries found in the SQS store image" >&2
     exit 1
 fi
+# The worker holds the store client key; it must never carry the store server.
+if [ "$PROFILE" = worker ] && grep -qE '(^|/)nats-server$' <<< "$archive_list"; then
+    echo "ERROR: nats-server found in the SQS worker image" >&2
+    exit 1
+fi
 echo "$tag content gate: PASS"
 
 echo ""

scripts/lib/sqs-worker-guest_test.sh

@@ -0,0 +1,163 @@
+#!/bin/sh
+# sqs-worker-guest_test.sh — the SQS worker guest's readiness watcher is
+# supervised by init, never leaves a store-ready marker behind a dead watcher,
+# and shutdown unmounts the worker disk only once the watcher is gone.
+#
+# Drives build/microvm/sqs-worker/sqs-worker-lib.sh with test-owned processes
+# and a stub umount. Every signal goes only to this test's own children.
+# The case settings below are read by the sourced library, not by this file.
+# shellcheck disable=SC2034
+set -eu
+
+ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
+LIB=${SQSW_GUEST_LIB:-"$ROOT/build/microvm/sqs-worker/sqs-worker-lib.sh"}
+INITTAB="$ROOT/build/microvm/sqs-worker/inittab"
+TMP=$(mktemp -d)
+CHILDREN=""
+cleanup() {
+    for p in $CHILDREN; do kill -KILL "$p" 2>/dev/null || true; done
+    rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
+
+echo "-- build/microvm/sqs-worker/sqs-worker-lib.sh"
+
+mkdir -p "$TMP/bin"
+cat > "$TMP/bin/umount" <<'EOF'
+#!/bin/sh
+echo "umount $*" >> "$T/calls.log"
+EOF
+# The fake watcher notes whether a marker was already present when it
+# started, publishes readiness as the real one does, then stays running.
+cat > "$TMP/bin/watcher" <<'EOF'
+#!/bin/sh
+[ ! -e "$T/run/store-ready" ] || echo stale >> "$T/saw-stale"
+echo "$$" >> "$T/watchers"
+echo "10.0.0.5 UPUB" > "$T/run/store-ready"
+exec sleep 60
+EOF
+chmod +x "$TMP/bin"/*
+
+fresh() {
+    T="$TMP/$1"
+    export T
+    mkdir -p "$T/run" "$T/data"
+    echo UPUBLICKEY > "$T/run/public-key"
+    for f in calls.log console mounts; do : > "$T/$f"; done
+}
+gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
+waitfor() { i=0; while [ "$i" -lt 100 ] && ! eval "$1"; do i=$((i + 1)); sleep 0.05; done; eval "$1"; }
+# start_run: one init respawn of the readiness step, in the background.
+start_run() {
+    SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK_INTERVAL="${PARK:-30}" \
+        sh -c '. "$1"; sqsw_run' sh "$LIB" & pid=$!
+    CHILDREN="$CHILDREN $pid"
+}
+# record <pid>: write the identity sqs_run would record for that process.
+# shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
+record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/ready.pid"; }
+# shutdown_step [survivor-pid]: with a pid, that process is reported alive
+# whatever happens to it, as a watcher stuck past SIGKILL would be.
+shutdown_step() {
+    (
+        SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_MOUNTS="$T/mounts"
+        UMOUNT="$TMP/bin/umount" SQSW_STOP_BUDGET_TENTHS=5
+        # shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
+        . "$LIB"
+        if [ -n "${1:-}" ]; then sqsw_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi
+        sqsw_shutdown
+    ) > /dev/null 2>&1
+}
+
+check "init respawns the readiness step rather than running it once" \
+    'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
+
+# --- A watcher killed after readiness: its marker must not outlive it ---
+fresh killed-after-ready
+start_run; first=$pid
+check "the watcher publishes store-ready" 'waitfor "[ -s \"\$T/run/store-ready\" ]"'
+kill -KILL "$first"
+check "SIGKILL leaves the marker behind, as a crash would" 'waitfor "gone $first" && [ -e "$T/run/store-ready" ]'
+start_run; second=$pid
+check "the respawned step starts a new watcher" 'waitfor "[ \"\$(wc -l < \"\$T/watchers\")\" -eq 2 ]"'
+check "the respawn removed the dead watcher's marker before the new watcher ran" \
+    '[ ! -e "$T/saw-stale" ] && grep -q "removed a store-ready marker left by an earlier watcher" "$T/console"'
+check "the respawn records the new watcher's identity" 'grep -q "^$second " "$T/run/ready.pid"'
+kill -KILL "$second" 2>/dev/null || true
+
+# --- A watcher that keeps exiting is respawned at most once per gap ---
+fresh respawn-gap
+date +%s > "$T/run/ready.last"
+GAP=2 start_run; t0=$(date +%s)
+waitfor "[ -s \"\$T/watchers\" ]" || true
+t1=$(date +%s)
+check "a start right after the previous one waits for the gap" '[ $((t1 - t0)) -ge 1 ]'
+kill -KILL "$pid" 2>/dev/null || true
+
+# --- Runs that must not start ---
+# A run that cannot start parks rather than exiting, so init does not respawn
+# it in a loop; a second is many park intervals, and it must log only once.
+fresh run-stopping; : > "$T/run/stopping"; echo stale > "$T/run/store-ready"
+PARK=0.05 start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
+sleep 1
+check "a start during shutdown parks instead of exiting and respawning" '! gone "$pid"'
+check "a parked start during shutdown starts no watcher and removes the marker" \
+    '[ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
+check "a parked start during shutdown logs once" '[ "$(grep -c "shutdown in progress" "$T/console")" -eq 1 ]'
+shutdown_step
+check "shutdown stops the parked start" 'waitfor "gone $pid"'
+
+fresh run-no-key; rm -f "$T/run/public-key"
+PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
+sleep 1
+check "a guest without a worker key parks instead of exiting and respawning" '! gone "$pid" && [ ! -e "$T/watchers" ]'
+check "a parked guest without a worker key logs once" '[ "$(grep -c "no worker key" "$T/console")" -eq 1 ]'
+echo UPUBLICKEY > "$T/run/public-key"
+check "a key that appears later starts the watcher" 'waitfor "[ -s \"\$T/watchers\" ]"'
+kill -KILL "$pid" 2>/dev/null || true
+
+fresh run-no-key-stop; rm -f "$T/run/public-key"
+PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
+shutdown_step
+check "shutdown stops a start parked for a missing key" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
+
+# --- Shutdown: unmount only once the watcher is confirmed gone ---
+fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
+start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"
+check "orderly shutdown succeeds" 'shutdown_step'
+check "orderly shutdown confirms the watcher exited, removes the marker, then unmounts" \
+    'gone "$pid" && grep -q "readiness watcher pid=$pid exited" "$T/console" && [ ! -e "$T/run/store-ready" ] &&
+     [ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
+check "shutdown blocks any later respawn" '[ -e "$T/run/stopping" ]'
+
+fresh shutdown-hung; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
+(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
+echo "10.0.0.5 UPUB" > "$T/run/store-ready"
+shutdown_step
+check "a watcher ignoring SIGTERM is killed after the budget, then the disk is unmounted" \
+    'gone "$pid" && grep -q SIGKILL "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
+
+fresh shutdown-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
+(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
+echo "10.0.0.5 UPUB" > "$T/run/store-ready"
+check "shutdown with a surviving watcher reports failure" '! shutdown_step "$pid"'
+check "a surviving watcher leaves every mount alone and is never reported stopped" \
+    '! grep -q "^umount" "$T/calls.log" && grep -q "STILL ALIVE" "$T/console" && grep -q "skipping umount -a -r" "$T/console" && ! grep -q "exited" "$T/console"'
+check "a surviving watcher still leaves no readiness marker" '[ ! -e "$T/run/store-ready" ]'
+kill -KILL "$pid" 2>/dev/null || true
+
+fresh shutdown-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
+(exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/ready.pid"
+shutdown_step
+check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running readiness watcher" "$T/console"'
+kill -KILL "$pid" 2>/dev/null || true
+
+if [ "$fails" -ne 0 ]; then
+    echo "  $fails failure(s)"
+    exit 1
+fi

scripts/lib/stage-sqs-store-microvm.sh

@@ -1,18 +1,24 @@
 #!/bin/sh
-# stage-sqs-store-microvm.sh <image-dir> <dist-root>: copy a built SQS store
-# microVM image into a release tree as <dist-root>/microvm-sqs-store. Without a
-# complete image it stages nothing and says so, as the load balancer image does.
+# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-store|sqs-worker]:
+# copy a built SQS microVM image into a release tree as <dist-root>/microvm-<name>.
+# Without a complete image it stages nothing and says so, as the LB image does.
 set -eu
-src=$1 dest=$2/microvm-sqs-store
+src=$1 name=${3:-sqs-store}
+case "$name" in
+    sqs-store) label="SQS store" ;;
+    sqs-worker) label="SQS worker" ;;
+    *) echo "[distro] unknown SQS microVM image '$name'" >&2; exit 2 ;;
+esac
+dest=$2/microvm-$name
 # An image staged by an earlier build must not ship with this one. Only the
 # two image files are removed; anything else in the directory is left alone.
 rm -f "$dest/vmlinuz" "$dest/initramfs.cpio.gz"
 if [ -s "$src/vmlinuz" ] && [ -s "$src/initramfs.cpio.gz" ]; then
-    echo "[distro] staging the SQS store microVM image into the tarball"
+    echo "[distro] staging the $label microVM image into the tarball"
     mkdir -p "$dest"
     cp "$src/vmlinuz" "$src/initramfs.cpio.gz" "$dest/"
 else
     [ ! -d "$dest" ] || rmdir "$dest" 2> /dev/null || true
-    echo "[distro] WARNING: SQS store microVM image missing; the tarball will not include it"
-    echo "[distro]          run 'make build-sqs-store-microvm-image' first"
+    echo "[distro] WARNING: $label microVM image missing; the tarball will not include it"
+    echo "[distro]          run 'make build-$name-microvm-image' first"
 fi

scripts/mkosi-build.sh

@@ -99,7 +99,8 @@ profile_binaries() {
 ./cmd/eks-gateway-publish:usr/local/bin/eks-gateway-publish \
 ./cmd/eks-gateway-fetch:usr/local/bin/eks-gateway-fetch \
 ./cmd/eks-webhook-cert:usr/local/bin/eks-webhook-cert \
-./cmd/eks-konnectivity-cert:usr/local/bin/eks-konnectivity-cert"
+./cmd/eks-konnectivity-cert:usr/local/bin/eks-konnectivity-cert \
+./cmd/eks-etcd-member:usr/local/bin/eks-etcd-member"
             ;;
         ecs) echo "./cmd/ecs-agent:usr/local/bin/ecs-agent" ;;
         *)   echo "" ;;

scripts/setup.sh

@@ -16,8 +16,8 @@
 #   VERBOSE                    Set to 1 to echo "[setup] <stage>" before each top-level step.
 #   SETUP_STAGES               Comma-separated subset of stages to run:
 #                                deps, aws, dns, users, sudoers, firewall, timesync, files,
-#                                directories, env, systemd, sysctl, modules, logrotate,
-#                                udev, fixown, migrations
+#                                directories, env, systemd, sysctl, needrestart, modules,
+#                                logrotate, udev, fixown, migrations
 #                              Unset = run every stage appropriate for the current mode.
 
 set -e
@@ -39,6 +39,7 @@ FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
 FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
 FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
 FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
+FIREWALL_TRACK="${FIREWALL_DIR}/conntrack.nft"
 FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
 FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
 
@@ -556,6 +557,11 @@ resolve_firewall_mode() {
         info "Host firewall: firewalld is active, so the Spinifex policy will be armed in its place"
         return 0
     fi
+    if [ "${ISO_BUILD:-0}" != "1" ] && ufw_active; then
+        INSTALL_SPINIFEX_FIREWALL="on"
+        info "Host firewall: ufw is active, so the Spinifex policy will be armed in its place"
+        return 0
+    fi
     # The ISO is an appliance and arms; any other host was already doing something.
     if [ "${ISO_BUILD:-0}" = "1" ]; then
         INSTALL_SPINIFEX_FIREWALL="on"
@@ -597,6 +603,29 @@ mask_firewalld() {
     fi
 }
 
+ufw_active() {
+    command -v ufw > /dev/null 2>&1 || return 1
+    ${SUDO:-} ufw status 2>/dev/null | grep -q '^Status: active'
+}
+
+# Disable ufw if it is enforcing, as some cloud images ship it (Vultr's Ubuntu
+# does, with INPUT DROP). Its rules would have to accept everything the Spinifex
+# policy does, so guest IMDS, routed NAT and the API break with nothing logged.
+# `ufw disable` is persistent and reversible; the rules are kept, not migrated.
+disable_ufw() {
+    [ "${ISO_BUILD:-0}" = "1" ] && return 0
+    ufw_active || return 0
+    $SUDO ufw disable > /dev/null \
+        || fatal "Cannot disable ufw. The Spinifex nftables policy cannot run beside it"
+    warn "ufw is now disabled: the Spinifex nftables policy is the only supported node firewall."
+    warn "  ufw's rules were not migrated. Restate any restriction you relied on in $FIREWALL_CUSTOM."
+    if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
+        warn "  Until the daemon first loads the policy, this host accepts every connection."
+    else
+        warn "  The Spinifex policy is not armed (--firewall=off), so this host now has no firewall."
+    fi
+}
+
 # The policy is written here rather than shipped in the tarball so the ISO chroot
 # and the ansible path get an identical file without a packaging step. Installed
 # but NOT enabled: the peer set is empty until spinifex-daemon resolves the
@@ -651,9 +680,12 @@ table inet spinifex_filter {
         # would expose every wildcard-bound host service (sshd, the AWS gateway,
         # northstar, the UI) to any tenant. IMDS is HTTP on .254:80; VPC DNS is
         # 53 on .253. Everything else on these ports falls through to policy drop.
-        iifname "ime-*" ip daddr 169.254.169.254 tcp dport 80 accept
-        iifname "ime-*" ip daddr 169.254.169.253 udp dport 53 accept
-        iifname "ime-*" ip daddr 169.254.169.253 tcp dport 53 accept
+        #
+        # Matched on the pre-DNAT destination: with imds_host_meta_ip set, the
+        # remap rewrites the address in prerouting, before this hook sees it.
+        iifname "ime-*" ct original ip daddr 169.254.169.254 tcp dport 80 accept
+        iifname "ime-*" ct original ip daddr 169.254.169.253 udp dport 53 accept
+        iifname "ime-*" ct original ip daddr 169.254.169.253 tcp dport 53 accept
 
         # PMTUD is not optional under a Geneve overlay.
         icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
@@ -718,6 +750,28 @@ RULES
     $SUDO chmod 0644 "$FIREWALL_RULES"
     info "  $FIREWALL_RULES"
 
+    # The kernel only tracks connections while some rule uses conntrack. Without
+    # this table a disarmed node tracks nothing, and a flow opened then (OVN's
+    # RAFT links) is first seen mid-stream at arm time, from whichever end speaks
+    # first. Seen from the server, that is a new flow to an ephemeral port, and
+    # the policy drops it. Kept loaded whether or not the policy is armed.
+    $SUDO tee "$FIREWALL_TRACK" > /dev/null << 'TRACK'
+# Spinifex connection tracking. Managed by scripts/setup.sh — edits are overwritten.
+# Accepts everything and filters nothing; it exists so conntrack keeps running
+# while inet spinifex_filter is not loaded. Never delete it on a running node.
+table inet spinifex_conntrack
+delete table inet spinifex_conntrack
+
+table inet spinifex_conntrack {
+    chain input {
+        type filter hook input priority filter; policy accept;
+        ct state invalid counter
+    }
+}
+TRACK
+    $SUDO chmod 0644 "$FIREWALL_TRACK"
+    info "  $FIREWALL_TRACK"
+
     # Host-local facts the policy needs that the daemon has no view of. Separate
     # from peers.nft because the daemon rewrites that file and would drop this.
     # Refreshed on every setup.sh run, so an sshd port change is picked up by an
@@ -771,8 +825,9 @@ CUSTOM
     info "  $FIREWALL_MODE_FILE ($INSTALL_SPINIFEX_FIREWALL)"
 
     # After the mode file, so a retry after an interrupted install still finds
-    # 'on' once firewalld is no longer active.
+    # 'on' once firewalld or ufw is no longer active.
     mask_firewalld
+    disable_ufw
 
     $SUDO tee "$FIREWALL_APPLY" > /dev/null << 'APPLY'
 #!/bin/sh
@@ -784,6 +839,7 @@ set -eu
 RULES="/etc/spinifex/firewall/spinifex.nft"
 PEERS="/etc/spinifex/firewall/peers.nft"
 OPEN="/etc/spinifex/firewall/open-ports.nft"
+TRACK="/etc/spinifex/firewall/conntrack.nft"
 
 # 0 is a sentinel that keeps the set non-empty, because nft rejects an empty
 # one. No packet can match it: the kernel has no destination port 0.
@@ -805,13 +861,33 @@ write_open_ports() {
 # Deliberately ahead of the policy-file check below: turning the firewall off is
 # the recovery path, and refusing to run it because the policy is missing leaves
 # an operator with a node they can neither arm nor disarm.
+#
+# The tracking table goes in in the same transaction as the delete, so conntrack
+# never stops: a flow that opens while disarmed is then tracked from its SYN,
+# and re-arming does not drop it. The boot unit stays enabled to reload it.
 if [ "${1:-}" = "disable" ]; then
-    nft delete table inet spinifex_filter 2>/dev/null || true
+    if [ -r "$TRACK" ]; then
+        printf 'include "%s"\ntable inet spinifex_filter\ndelete table inet spinifex_filter\n' "$TRACK" | nft -f - \
+            || nft delete table inet spinifex_filter 2>/dev/null || true
+    else
+        nft delete table inet spinifex_filter 2>/dev/null || true
+    fi
     rm -f "$PEERS"
-    systemctl disable --now spinifex-firewall.service >/dev/null 2>&1 || true
     exit 0
 fi
 
+# track loads only the tracking table. boot is the unit's start: tracking on
+# every node, and the policy on top only where the daemon has written peers.
+if [ "${1:-}" = "track" ] || [ "${1:-}" = "boot" ]; then
+    if [ -r "$TRACK" ]; then
+        nft -f "$TRACK"
+    elif [ "$1" = "track" ]; then
+        echo "no tracking table at $TRACK" >&2
+        exit 1
+    fi
+    if [ "$1" = "track" ] || [ ! -r "$PEERS" ]; then exit 0; fi
+fi
+
 # Every remaining verb writes the policy, so it has to exist. Failing here
 # changes nothing, which leaves a node whose daemon has not yet resolved cluster
 # membership reachable rather than cut off from a cluster it cannot name.
@@ -885,10 +961,9 @@ if [ "${1:-}" = "set-peers" ]; then
     fi
     rm -f "$tmp" "$bak"
 
-    # The policy is loaded, so make it survive a reboot too. `disable` turns this
-    # unit off, and forming a cluster means disabling the firewall on every node
-    # first; without this, re-arming would look complete while silently leaving
-    # the node bare after its next boot. Idempotent and quiet in the normal case.
+    # The policy is loaded, so make it survive a reboot too. An older helper's
+    # `disable` turned this unit off; without this, re-arming such a node would
+    # look complete while leaving it bare after its next boot. Idempotent.
     systemctl enable spinifex-firewall.service >/dev/null 2>&1 || true
     exit 0
 fi
@@ -905,6 +980,13 @@ APPLY
     $SUDO chmod 0755 "$FIREWALL_APPLY"
     info "  $FIREWALL_APPLY"
 
+    # Now, not at the next boot: a fresh install forms its cluster before then,
+    # and an upgrade must start tracking before the daemon next arms. A chroot
+    # would load it into the build host's kernel.
+    if [ "${ISO_BUILD:-0}" != "1" ]; then
+        $SUDO "$FIREWALL_APPLY" track || warn "  Could not load $FIREWALL_TRACK: arming later may drop live cluster connections"
+    fi
+
     # Say which state ssh is in either way. A node whose allowlist was never
     # narrowed reads as configured otherwise, and that is the whole risk here.
     _ssh_srcs="$(grep -o '{[^}]*}' "$FIREWALL_CUSTOM" 2>/dev/null | tr -d '{}' | tr -s ' ')"
@@ -1171,7 +1253,7 @@ APT_RUNTIME_PACKAGES="nbdkit
 qemu-utils gdisk ovmf qemu-efi-aarch64 less
 libvirt-daemon-system libvirt-clients
 pciutils
-jq curl iproute2 ethtool netcat-openbsd wget unzip xz-utils file
+jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
 ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
 chrony nftables"
 
@@ -1181,7 +1263,7 @@ chrony nftables"
 OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
 qemu-kvm qemu-img edk2-ovmf seabios-bin less
 pciutils
-jq curl iproute ethtool nmap-ncat wget unzip xz file
+jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
 ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
 chrony nftables iptables-nft systemd-resolved"
 
@@ -1543,24 +1625,30 @@ install_files() {
         info "  /usr/share/spinifex/microvm/*"
     fi
     install_sqs_store_image
+    install_sqs_worker_image
 }
 
-# The SQS store image ships beside the load balancer's, not inside it: an older
+# The SQS images ship beside the load balancer's, not inside it: an older
 # installer copies microvm/* without -r and would stop at a subdirectory.
-install_sqs_store_image() {
-    local src="$EXTRACT_DIR/microvm-sqs-store" f
-    local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/sqs-store"
+install_sqs_store_image() { install_sqs_microvm_image sqs-store "SQS store"; }
+install_sqs_worker_image() { install_sqs_microvm_image sqs-worker "SQS worker"; }
+
+# install_sqs_microvm_image <name> <label>
+install_sqs_microvm_image() {
+    local name=$1 label=$2 f
+    local src="$EXTRACT_DIR/microvm-$name"
+    local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/$name"
     # A previously installed image belongs to an older release, and this
     # release's daemon must not boot it. Only the two image files are removed.
     if [ ! -d "$src" ]; then
-        remove_sqs_store_image "$dest"
-        warn "This release has no SQS store microVM image; SQS store launches will be refused"
+        remove_sqs_microvm_image "$dest" "$label"
+        warn "This release has no $label microVM image; $label launches will be refused"
         return 0
     fi
     for f in vmlinuz initramfs.cpio.gz; do
         if [ ! -s "$src/$f" ]; then
-            remove_sqs_store_image "$dest"
-            fatal "The SQS store microVM image in this release is incomplete: $f is missing"
+            remove_sqs_microvm_image "$dest" "$label"
+            fatal "The $label microVM image in this release is incomplete: $f is missing"
         fi
     done
     # The initramfs goes last, so an interrupted install leaves no pair the
@@ -1572,11 +1660,11 @@ install_sqs_store_image() {
     info "  $dest/vmlinuz, $dest/initramfs.cpio.gz"
 }
 
-# Running store VMs keep their already-loaded kernel; the files are read at boot.
-remove_sqs_store_image() {
+# Running VMs keep their already-loaded kernel; the files are read at boot.
+remove_sqs_microvm_image() {
     [ -e "$1/initramfs.cpio.gz" ] || [ -e "$1/vmlinuz" ] || return 0
     $SUDO rm -f "$1/initramfs.cpio.gz" "$1/vmlinuz"
-    warn "Removed the SQS store microVM image left by an earlier install from $1"
+    warn "Removed the $2 microVM image left by an earlier install from $1"
 }
 
 # --- Create directories ---
@@ -1888,9 +1976,9 @@ EOF
     # takes effect once enabled) — without this the JetStream ENOSPC-latch
     # watchdog is inert and a full disk requires a manual restart forever.
     $SUDO systemctl enable --now spinifex-nats-watchdog.timer
-    # Enabled, not started: its ConditionPathExists holds it inert until the
-    # daemon writes a peer file, so this only decides that a node which has one
-    # gets the policy back at boot, before any service opens a socket.
+    # Enabled on every node: at boot it loads connection tracking everywhere,
+    # and the policy only on a node whose daemon has written a peer file, so it
+    # is back before any service opens a socket.
     $SUDO systemctl enable spinifex-firewall.service
     info "Systemd units installed and enabled (per-service users)"
 }
@@ -1930,6 +2018,28 @@ SYSCTL
     info "Kernel tunables installed (net.core.rmem_max/wmem_max = 16MB)"
 }
 
+# --- Keep needrestart off the Spinifex, OVS and OVN units ---
+# Guest QEMU runs in spinifex-daemon's cgroup, so needrestart charges QEMU's
+# replaced libraries to the daemon and restarts it after every dpkg run, which
+# never clears them and ends at the unit's start limit. Restarting the stack
+# after an upgrade is the operator's call; needrestart still lists the units.
+install_needrestart_conf() {
+    stage "installing needrestart override"
+    local dir="${DISTRO_ROOT:-}/etc/needrestart/conf.d"
+    $SUDO install -d "$dir"
+    $SUDO tee "$dir/90-spinifex.conf" > /dev/null << 'NEEDRESTART'
+# Managed by spinifex setup.sh — edits are overwritten.
+# Never restart these automatically; restart them by hand when it suits.
+$nrconf{override_rc}{qr(^spinifex-)} = 0;
+$nrconf{override_rc}{qr(^ovs-)} = 0;
+$nrconf{override_rc}{qr(^ovsdb-)} = 0;
+$nrconf{override_rc}{qr(^ovn-)} = 0;
+$nrconf{override_rc}{qr(^openvswitch-)} = 0;
+NEEDRESTART
+    $SUDO chmod 0644 "$dir/90-spinifex.conf"
+    info "needrestart will not restart the spinifex, OVS or OVN units"
+}
+
 # --- Install logrotate ---
 install_logrotate() {
     stage "installing logrotate config"
@@ -2124,6 +2234,7 @@ main() {
     stage_enabled fixown     && fix_file_ownership
     stage_enabled systemd    && install_systemd
     stage_enabled sysctl     && install_sysctl
+    stage_enabled needrestart && install_needrestart_conf
     stage_enabled modules    && install_kernel_modules
     stage_enabled logrotate  && install_logrotate
     stage_enabled udev       && install_udev

scripts/setup_firewall_conntrack_test.sh

@@ -0,0 +1,214 @@
+#!/bin/sh
+# setup_firewall_conntrack_test.sh — arms the real host firewall under a TCP
+# flow that opened while it was disarmed, in two network namespaces.
+#
+# The policy, the tracking table and spinifex-firewall-apply are cut out of
+# setup.sh with their paths moved under a temp dir, and the helper's verbs are
+# run inside the client namespace. Needs root or passwordless sudo, ip, nft and
+# python3; skips without them. A stub systemctl keeps the helper off the host.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SETUP="$SCRIPT_DIR/setup.sh"
+
+echo "-- setup.sh firewall: flows that outlive a disarm"
+
+if [ "$(id -u)" = 0 ]; then SUDO=""
+elif sudo -n true 2>/dev/null; then SUDO="sudo -n"
+else echo "  skip - needs root or passwordless sudo"; exit 0
+fi
+for bin in ip nft python3; do
+    if ! PATH="$PATH:/usr/sbin:/sbin" command -v "$bin" > /dev/null 2>&1; then
+        echo "  skip - $bin not installed"; exit 0
+    fi
+done
+
+TMP=$(mktemp -d)
+S="fwct-s-$$"
+C="fwct-c-$$"
+cleanup() {
+    $SUDO ip netns del "$S" 2> /dev/null || true
+    $SUDO ip netns del "$C" 2> /dev/null || true
+    rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+# heredoc <tag> — the body of setup.sh's quoted heredoc named <tag>, with the
+# firewall directory moved under $TMP.
+heredoc() {
+    sed -n "/<< '$1'\$/,/^$1\$/p" "$SETUP" | sed '1d;$d' | sed "s#/etc/spinifex/firewall#$TMP/fw#g"
+}
+
+mkdir -p "$TMP/fw" "$TMP/bin"
+heredoc RULES > "$TMP/fw/spinifex.nft"
+heredoc APPLY > "$TMP/apply"
+heredoc TRACK > "$TMP/fw/conntrack.nft"
+printf 'define spinifex_ssh_ports = { 22 }\n' > "$TMP/fw/local.nft"
+printf 'define spinifex_open_ports = { 0 }\n' > "$TMP/fw/open-ports.nft"
+printf 'define trusted_ssh_peers = { 0.0.0.0/0 }\n' > "$TMP/fw/custom.nft"
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/systemctl"
+chmod +x "$TMP/bin/systemctl"
+if [ ! -s "$TMP/fw/spinifex.nft" ] || [ ! -s "$TMP/apply" ]; then
+    echo "  FAIL - could not cut the policy or the helper out of setup.sh"; exit 1
+fi
+
+# One process owns both ends, so the order packets are sent in is exact. The
+# server is 10.99.0.1 (a peer) and also holds 10.99.0.9 (not a peer).
+cat > "$TMP/flow.py" << 'PY'
+import os, socket, subprocess, sys
+
+srv_ns, cli_ns, script = sys.argv[1], sys.argv[2], sys.argv[3]
+
+def enter(ns):
+    fd = os.open("/run/netns/" + ns, os.O_RDONLY)
+    os.setns(fd, os.CLONE_NEWNET)
+    os.close(fd)
+
+enter(srv_ns)
+listener = socket.socket()
+listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+listener.bind(("10.99.0.1", 6644))
+listener.listen(1)
+enter(cli_ns)
+
+cli = srv = None
+for step in script.split("@@"):
+    if step == "connect":
+        cli = socket.create_connection(("10.99.0.1", 6644), timeout=3)
+        srv, _ = listener.accept()
+        cli.settimeout(2)
+        srv.settimeout(2)
+    elif step in ("s2c", "c2s"):
+        tx, rx = (srv, cli) if step == "s2c" else (cli, srv)
+        tx.sendall(b"x" * 300)
+        got = b""
+        try:
+            while len(got) < 300:
+                got += rx.recv(300 - len(got))
+        except socket.timeout:
+            print("lost " + step)
+            sys.exit(1)
+    else:
+        subprocess.run(step, shell=True, check=True)
+PY
+
+# dial <client source> <server addr> <port> — a new connection from the server
+# namespace to the client, which listens on the port. 0 when it connects.
+cat > "$TMP/dial.py" << 'PY'
+import os, socket, sys
+
+srv_ns, cli_ns, src, port = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+fd = os.open("/run/netns/" + cli_ns, os.O_RDONLY)
+os.setns(fd, os.CLONE_NEWNET)
+listener = socket.socket()
+listener.bind(("10.99.0.3", port))
+listener.listen(1)
+fd = os.open("/run/netns/" + srv_ns, os.O_RDONLY)
+os.setns(fd, os.CLONE_NEWNET)
+dialer = socket.socket()
+dialer.bind((src, 0))
+dialer.settimeout(2)
+try:
+    dialer.connect(("10.99.0.3", port))
+except OSError:
+    sys.exit(1)
+PY
+
+# A new pair of namespaces, and no peer file: every case starts disarmed.
+fresh() {
+    $SUDO rm -f "$TMP/fw/peers.nft"
+    $SUDO ip netns del "$S" 2> /dev/null || true
+    $SUDO ip netns del "$C" 2> /dev/null || true
+    $SUDO ip netns add "$S"
+    $SUDO ip netns add "$C"
+    $SUDO ip link add vs netns "$S" type veth peer name vc netns "$C"
+    $SUDO ip -n "$S" addr add 10.99.0.1/24 dev vs
+    $SUDO ip -n "$S" addr add 10.99.0.9/24 dev vs
+    $SUDO ip -n "$C" addr add 10.99.0.3/24 dev vc
+    for l in "$S vs" "$C vc"; do
+        # shellcheck disable=SC2086  # "<netns> <link>"
+        set -- $l
+        $SUDO ip -n "$1" link set lo up
+        $SUDO ip -n "$1" link set "$2" up
+    done
+}
+
+# Helper verbs and raw nft, run as root inside the client namespace.
+in_c="$SUDO env PATH=$TMP/bin:$PATH:/usr/sbin:/sbin ip netns exec $C"
+verb() { echo "$in_c sh $TMP/apply $*"; }
+arm="$(verb set-peers) < $TMP/peers"
+printf '10.99.0.1\n10.99.0.1\n' > "$TMP/peers"
+
+# flow <desc> <steps> — passes when every transfer is delivered.
+flow() {
+    if out=$($SUDO python3 "$TMP/flow.py" "$S" "$C" "$2" 2>&1); then pass "$1"; else fail "$1: $out"; fi
+}
+
+# Control first: with nothing tracking, arming under an open flow must drop the
+# server's next segment, or this kernel cannot show the failure and the cases
+# below prove nothing.
+fresh
+if $SUDO python3 "$TMP/flow.py" "$S" "$C" "connect@@s2c@@c2s@@$arm@@s2c" > /dev/null 2>&1; then
+    fail "control: a flow opened with no conntrack user survived the arm, so this kernel cannot reproduce the drop"
+else
+    pass "control: a flow opened with no conntrack user loses the server's next segment once armed"
+fi
+
+fresh
+flow "a flow opened after disable survives set-peers, server speaking first" \
+    "$(verb disable)@@connect@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened after boot with no peer file survives set-peers, server speaking first" \
+    "$(verb boot)@@connect@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened while armed survives disable and a re-arm, server speaking first" \
+    "$arm@@connect@@s2c@@c2s@@$(verb disable)@@s2c@@c2s@@s2c@@$arm@@s2c@@c2s"
+
+fresh
+flow "a flow opened before track survives set-peers once it has carried traffic" \
+    "connect@@s2c@@c2s@@$(verb track)@@s2c@@c2s@@$arm@@s2c@@c2s"
+
+# The tracking table must not open anything the policy drops.
+fresh
+sh -c "$(verb boot) && $arm"
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 6644; then
+    pass "armed: a peer still reaches a cluster port"
+else
+    fail "armed: a peer cannot reach a cluster port"
+fi
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 6644; then
+    fail "armed: a non-peer reaches a cluster port"
+else
+    pass "armed: a non-peer is dropped on a cluster port"
+fi
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 7777; then
+    fail "armed: a peer reaches a port the policy does not open"
+else
+    pass "armed: a peer is dropped on a port the policy does not open"
+fi
+
+# disable leaves only the tracking table, which accepts everything.
+sh -c "$(verb disable)"
+tables=$($in_c nft list tables | tr '\n' ' ')
+case "$tables" in
+    *spinifex_filter*) fail "disable left inet spinifex_filter loaded: $tables" ;;
+    *spinifex_conntrack*) pass "disable removes the policy and keeps the tracking table" ;;
+    *) fail "disable removed the tracking table: '$tables'" ;;
+esac
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 7777; then
+    pass "disarmed: the tracking table drops nothing"
+else
+    fail "disarmed: the tracking table dropped a connection"
+fi
+
+if [ "$fails" -gt 0 ]; then
+    echo "  $fails check(s) failed"
+    exit 1
+fi
+echo "  setup_firewall_conntrack ok"

scripts/setup_host_test.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # setup_host_test.sh — tests for setup.sh's DNS handover to systemd-resolved,
-# its chrony drop-in and its nbdkit plugin directory.
+# its chrony and needrestart drop-ins and its nbdkit plugin directory.
 #
 # setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs. PATH
 # holds stub commands and links to a few real tools only, so no host resolver,
@@ -17,10 +17,10 @@ fails=0
 pass() { echo "  ok   - $1"; }
 fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 
-echo "-- setup.sh host DNS, chrony and nbdkit"
+echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
 
 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
     ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
 
@@ -249,6 +249,50 @@ case "$RC:$OUT" in
     *) fail "no chrony: rc=$RC: $OUT" ;;
 esac
 
+# --------------------------------------------------------------------------
+# needrestart override.
+# --------------------------------------------------------------------------
+# restarts <conf> <unit> — how needrestart would treat <unit> under <conf>:
+# "no" when an override_rc pattern deselects it, "yes" otherwise.
+restarts() {
+    perl -e 'our %nrconf; do $ARGV[0]; die $@ if $@;
+        for my $re (keys %{$nrconf{override_rc}}) {
+            if ($ARGV[1] =~ $re) { print $nrconf{override_rc}{$re} ? "yes" : "no"; exit }
+        }
+        print "yes"' "$1" "$2"
+}
+
+host
+run install_needrestart_conf
+conf="$TMP/root/etc/needrestart/conf.d/90-spinifex.conf"
+if [ "$RC" != 0 ] || [ ! -f "$conf" ]; then
+    fail "needrestart: rc=$RC: $OUT"
+elif ! command -v perl > /dev/null; then
+    fail "needrestart: perl is not installed, so the override cannot be checked"
+else
+    wrong=""
+    for unit in spinifex-daemon.service spinifex-viperblock.service ovs-vswitchd.service \
+        ovsdb-server.service ovn-controller.service openvswitch-switch.service; do
+        [ "$(restarts "$conf" "$unit")" = no ] || wrong="$wrong $unit"
+    done
+    for unit in ssh.service rsyslog.service polkit.service; do
+        [ "$(restarts "$conf" "$unit")" = yes ] || wrong="$wrong $unit"
+    done
+    if [ -n "$wrong" ]; then
+        fail "needrestart override treats these wrongly:$wrong"
+    else
+        pass "needrestart keeps off the stack's units and still restarts everything else"
+    fi
+fi
+
+before=$(cat "$conf" 2>/dev/null)
+run install_needrestart_conf
+if [ "$RC" = 0 ] && [ "$(cat "$conf" 2>/dev/null)" = "$before" ]; then
+    pass "needrestart override is unchanged by a second install"
+else
+    fail "needrestart override changed on re-run: rc=$RC: $OUT"
+fi
+
 # --------------------------------------------------------------------------
 # nbdkit plugin directory.
 # --------------------------------------------------------------------------

scripts/setup_sqs_store_image_test.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
-# setup_sqs_store_image_test.sh — tests for shipping the SQS store microVM
-# image: staging it into a release tree and installing it with setup.sh.
+# setup_sqs_store_image_test.sh — tests for shipping the SQS store and worker
+# microVM images: staging them into a release tree and installing with setup.sh.
 #
 # setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
 # installs into a temporary SPINIFEX_MICROVM_DIR with SUDO cleared.
@@ -16,7 +16,7 @@ fails=0
 pass() { echo "  ok   - $1"; }
 fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 
-echo "-- SQS store microVM image release and install"
+echo "-- SQS store and worker microVM image release and install"
 
 # image <dir> <tag>: a fake built image whose bytes name its tag.
 image() {
@@ -186,5 +186,59 @@ else
     fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ] && echo present || echo missing)"
 fi
 
+# --- SQS worker image: same staging and install path, its own directory ---
+install_worker() {
+    RC=0
+    OUT=$(EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
+        bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
+}
+image "$TMP/wbuilt" w1
+release "$TMP/wrel"
+sh "$STAGE" "$TMP/built" "$TMP/wrel" > /dev/null
+sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" sqs-worker > /dev/null
+if same "$TMP/wbuilt/vmlinuz" "$TMP/wrel/microvm-sqs-worker/vmlinuz" \
+    && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wrel/microvm-sqs-worker/initramfs.cpio.gz" \
+    && same "$TMP/built/vmlinuz" "$TMP/wrel/microvm-sqs-store/vmlinuz"; then
+    pass "the worker image stages beside the store image"
+else
+    fail "worker staging: $(ls -R "$TMP/wrel")"
+fi
+install_worker "$TMP/wrel" "$TMP/wmv"
+if [ "$RC" = 0 ] && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wmv/sqs-worker/initramfs.cpio.gz" \
+    && [ ! -e "$TMP/wmv/sqs-store" ]; then
+    pass "the worker image installs into its own directory only"
+else
+    fail "worker install: rc=$RC $OUT"
+fi
+release "$TMP/wrel-none"
+OUT=$(sh "$STAGE" "$TMP/no-such-build" "$TMP/wrel-none" sqs-worker)
+if [ ! -e "$TMP/wrel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'SQS worker microVM image missing'; then
+    pass "a missing worker build stages nothing and says so"
+else
+    fail "missing worker build: $OUT"
+fi
+install_worker "$TMP/wrel-none" "$TMP/wmv"
+if [ "$RC" = 0 ] && [ ! -e "$TMP/wmv/sqs-worker/vmlinuz" ] && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ] \
+    && echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
+    pass "a release without the worker image removes the earlier one and warns"
+else
+    fail "release without worker image: rc=$RC $OUT"
+fi
+cp -r "$TMP/wrel" "$TMP/wrel-bad"
+rm "$TMP/wrel-bad/microvm-sqs-worker/vmlinuz"
+install_worker "$TMP/wrel" "$TMP/wmv"
+install_worker "$TMP/wrel-bad" "$TMP/wmv"
+if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'SQS worker microVM image in this release is incomplete: vmlinuz is missing' \
+    && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ]; then
+    pass "an incomplete worker image is refused and the earlier pair is removed"
+else
+    fail "incomplete worker image: rc=$RC $OUT"
+fi
+if OUT=$(sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" lb 2>&1); then
+    fail "an unknown image name was staged"
+else
+    pass "an unknown image name is refused"
+fi
+
 [ "$fails" = 0 ] || { echo "FAIL: $fails"; exit 1; }
 echo "  setup_sqs_store_image_test ok"

scripts/terraform/vultr-spx/README.md

@@ -0,0 +1,39 @@
+# Spinifex on Vultr
+
+`deploy-vultr.sh` builds a one- or three-node Spinifex deployment on Vultr VX VMs or bare-metal servers in their own VPC, and tears it down again leaving nothing billing. The customer guide is [`docs/vultr-integration/README.md`](../../../docs/vultr-integration/README.md).
+
+```bash
+export VULTR_API_KEY=...            # the key's IP allowlist must admit this host and every node
+./deploy-vultr.sh up --channel dev                                   # one VX VM
+./deploy-vultr.sh up --channel dev --nodes 3 --plan vbm-8c-64gb-amd  # three bare-metal servers
+./deploy-vultr.sh up --name v1 --distro spinifex-<version>.tar.gz    # a named deployment from a local tarball
+./deploy-vultr.sh status
+./deploy-vultr.sh credentials
+./deploy-vultr.sh ssh [nodeN] [command]
+./deploy-vultr.sh destroy
+```
+
+Without `--name` the deployment is called `default`, and `up` writes the AWS profile `spinifex`. A named deployment writes `spinifex-vultr-<name>` instead, so several can run side by side. Either way the cluster's CA is at `~/.spinifex/vultr-<name>-ca.pem` and the profile works straight away. `up` ends by printing every node's address and the `/32`s the API key's allowlist needs.
+
+## What it creates
+
+| Object | Created by | Removed by |
+| --- | --- | --- |
+| VPC `spinifex-<name>` (`10.10.8.0/24` unless `--vpc-subnet`) | Terraform | Terraform |
+| SSH key `spinifex-<name>` | Terraform | Terraform |
+| VMs or bare-metal servers `spinifex-<name>-node1` to `node3` | Terraform | Terraform |
+| VM boot volumes `spinifex-<name>-nodeN-boot` (50 GB each) | Terraform, through the VM | `destroy`: Vultr keeps it after the VM is deleted |
+| Reserved IPs labelled `spinifex-vultr-<name>:` | Spinifex, for each public IP and Elastic IP | Spinifex on release; `destroy` sweeps any left |
+
+A `vbm-` plan selects bare metal. It boots from local NVMe, so it has no boot volume, and Spinifex keeps its data on the root disk. `destroy` fails, and lists what remains, if anything above is still in the account afterwards.
+
+## Where things live
+
+- **State** is per deployment under `~/.spinifex/vultr/<name>/`: the Terraform state, the nodes' SSH host keys, logs, and one `step.*` file per completed step. A failed `up` resumes from the step that failed.
+- **The API key** is copied to `/etc/spinifex/vultr/api-key` on each node over SSH. It is never a Terraform variable, so it is in neither the state nor the instance metadata.
+- **The host firewall** is armed before Spinifex first starts. The nodes have no Vultr firewall group, so it is the only one.
+
+## Requirements
+
+- OpenTofu 1.5 or newer, the AWS CLI v2, `python3` and an SSH key (`--ssh-key`, default `~/.ssh/id_ed25519`).
+- A VX plan with at least 16 vCPU, of which Spinifex offers 14 to instances; an 8-vCPU plan offers 7.

scripts/terraform/vultr-spx/cloud-init/user-data.yaml

@@ -0,0 +1,11 @@
+#cloud-config
+# Host preparation only. Spinifex is installed and formed over SSH by deploy.sh,
+# and the API key is copied the same way so it never sits in instance metadata.
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
+    content: |
+      network: {config: disabled}
+runcmd:
+  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
+  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
+  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/vultr-spx/compute.tf

@@ -0,0 +1,50 @@
+resource "vultr_ssh_key" "deploy" {
+  name    = "spinifex-${var.deployment_name}"
+  ssh_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+}
+
+# A vbm- plan is a bare-metal server; anything else is a VX VM.
+locals {
+  bare_metal = startswith(var.plan, "vbm-")
+}
+
+# The boot volume is labelled so destroy can find it: Vultr keeps a VX boot
+# volume, and bills it, after the instance that booted from it is deleted.
+resource "vultr_instance" "node" {
+  count       = local.bare_metal ? 0 : var.node_count
+  region      = var.region
+  plan        = var.plan
+  os_id       = var.os_id
+  label       = "spinifex-${var.deployment_name}-node${count.index + 1}"
+  hostname    = "node${count.index + 1}"
+  ssh_key_ids = [vultr_ssh_key.deploy.id]
+  vpc_ids     = [vultr_vpc.lan.id]
+  user_data   = file("${path.module}/cloud-init/user-data.yaml")
+  tags        = ["spinifex", "deployment:${var.deployment_name}"]
+  backups     = "disabled"
+
+  block_devices {
+    disk_size = var.boot_disk_gb
+    bootable  = true
+    label     = "spinifex-${var.deployment_name}-node${count.index + 1}-boot"
+  }
+}
+
+# Bare metal boots from its local NVMe, so there is no boot volume to label.
+resource "vultr_bare_metal_server" "node" {
+  count            = local.bare_metal ? var.node_count : 0
+  region           = var.region
+  plan             = var.plan
+  os_id            = var.os_id
+  label            = "spinifex-${var.deployment_name}-node${count.index + 1}"
+  hostname         = "node${count.index + 1}"
+  ssh_key_ids      = [vultr_ssh_key.deploy.id]
+  vpc_id           = vultr_vpc.lan.id
+  user_data        = file("${path.module}/cloud-init/user-data.yaml")
+  tags             = ["spinifex", "deployment:${var.deployment_name}"]
+  activation_email = false
+
+  timeouts {
+    create = "60m"
+  }
+}

scripts/terraform/vultr-spx/deploy-vultr.sh

@@ -0,0 +1,511 @@
+#!/usr/bin/env bash
+# deploy-vultr.sh — build a one- or three-node Spinifex deployment on Vultr, or tear it down.
+#
+#   deploy-vultr.sh up --channel dev                                  one VX VM
+#   deploy-vultr.sh up --channel dev --nodes 3 --plan vbm-8c-64gb-amd three bare-metal servers
+#   deploy-vultr.sh status                 nodes, addresses and what the deployment holds
+#   deploy-vultr.sh credentials            print the AWS access key and secret
+#   deploy-vultr.sh ssh [nodeN] [command]  SSH to a node, node1 by default
+#   deploy-vultr.sh stop nodeN             power a node off, as a failure would
+#   deploy-vultr.sh start nodeN            power it back on
+#   deploy-vultr.sh destroy
+#
+# VULTR_API_KEY must be set. OpenTofu, or Terraform when OpenTofu is not
+# installed, creates the VPC, the nodes and any boot volumes; this script
+# installs and forms Spinifex over SSH, and records each step under the state
+# directory so a rerun resumes rather than re-initialising.
+# Three nodes are formed by install-node.sh with cluster and Geneve traffic on
+# the VPC, after each has come up as an armed single node.
+#
+# The API key is copied to the node over SSH, so it is in neither the Terraform
+# state nor the instance metadata. Destroy also deletes what Terraform never
+# created and Vultr keeps billing: the boot volume and every reserved IP the
+# node allocated under the deployment's label prefix.
+#
+# Without --name the deployment is called "default" and its AWS profile is
+# "spinifex"; a named deployment's profile is "spinifex-vultr-<name>".
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SPINIFEX_SRC="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+VULTR_API="https://api.vultr.com/v2"
+
+CMD="${1:-}"
+[ $# -gt 0 ] && shift
+NODE_ARG=""
+NAME="default"
+REGION="ord"
+PLAN="vx1-g-16c-64g"
+DISTRO=""
+CHANNEL=""
+SPX_REGION="us-east-1"
+SPX_AZ="us-east-1a"
+SSH_KEY="$HOME/.ssh/id_ed25519"
+IMAGE=""
+VPC_SUBNET="10.10.8.0"
+STATE_ROOT="$HOME/.spinifex/vultr"
+NODES=1
+YES=0
+
+usage() {
+    sed -n '2,27p' "$0" | sed 's/^# \{0,1\}//'
+    cat <<EOF
+
+Options:
+  --name NAME          deployment name (default $NAME); labels every resource
+  --region ID          Vultr region (default $REGION)
+  --plan ID            Vultr plan (default $PLAN); vbm-8c-64gb-amd for bare metal
+  --distro TARBALL     install this local release tarball
+  --channel CHANNEL    install from a release channel (latest, dev)
+  --spx-region NAME    Spinifex region (default $SPX_REGION)
+  --spx-az NAME        Spinifex availability zone (default $SPX_AZ)
+  --ssh-key PATH       private key; PATH.pub is installed (default $SSH_KEY)
+  --image NAME         also import this AMI, such as debian-13-x86_64 (default none)
+  --vpc-subnet ADDR    the deployment VPC's /24 (default $VPC_SUBNET)
+  --nodes N            1 or 3 (default $NODES)
+  --yes                destroy without asking
+EOF
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --name) NAME="$2"; shift ;;
+        --region) REGION="$2"; shift ;;
+        --plan) PLAN="$2"; shift ;;
+        --distro) DISTRO="$2"; shift ;;
+        --channel) CHANNEL="$2"; shift ;;
+        --spx-region) SPX_REGION="$2"; shift ;;
+        --spx-az) SPX_AZ="$2"; shift ;;
+        --ssh-key) SSH_KEY="$2"; shift ;;
+        --image) IMAGE="$2"; shift ;;
+        --vpc-subnet) VPC_SUBNET="$2"; shift ;;
+        --nodes) NODES="$2"; shift ;;
+        --yes) YES=1 ;;
+        -h|--help) usage; exit 0 ;;
+        node[1-3])
+            case "$CMD" in ssh|stop|start) ;; *) usage >&2; exit 2 ;; esac
+            [ -z "$NODE_ARG" ] || { usage >&2; exit 2; }
+            NODE_ARG="$1" ;;
+        *) [ "$CMD" = ssh ] && break
+           echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
+    esac
+    shift
+done
+
+log() { printf '[deploy %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
+die() { log "ERROR: $*"; exit 1; }
+
+[[ "$NAME" =~ ^[a-z0-9][a-z0-9-]{0,23}$ ]] || die "--name must be 1-24 lowercase letters, digits or hyphens"
+[ "$NODES" = 1 ] || [ "$NODES" = 3 ] || die "--nodes must be 1 or 3"
+case "$CMD" in up|destroy|status|stop|start) [ -n "${VULTR_API_KEY:-}" ] || die "VULTR_API_KEY is not set" ;; esac
+if command -v tofu >/dev/null; then TF_BIN=tofu
+elif command -v terraform >/dev/null; then TF_BIN=terraform
+else die "neither OpenTofu (tofu) nor Terraform is installed"; fi
+
+STATE_DIR="$STATE_ROOT/$NAME"
+TF_STATE="$STATE_DIR/terraform.tfstate"
+KNOWN_HOSTS="$STATE_DIR/known_hosts"
+LABEL_PREFIX="spinifex-vultr-$NAME"
+PROFILE="spinifex-vultr-$NAME"
+[ "$NAME" = default ] && PROFILE="spinifex"
+CA_FILE="$HOME/.spinifex/vultr-$NAME-ca.pem"
+mkdir -p "$STATE_DIR"
+chmod 0700 "$STATE_DIR"
+
+# vultr METHOD PATH [BODY] prints the response body and fails on a non-2xx.
+vultr() {
+    local out code
+    out="$(curl -sS -w '\n%{http_code}' -X "$1" -H "Authorization: Bearer $VULTR_API_KEY" \
+        -H 'Content-Type: application/json' ${3:+-d "$3"} "$VULTR_API$2")"
+    code="${out##*$'\n'}"
+    out="${out%$'\n'*}"
+    case "$code" in
+        2??) printf '%s' "$out" ;;
+        *) log "Vultr $1 $2 returned $code: $out"; return 1 ;;
+    esac
+}
+
+tf() {
+    "$TF_BIN" -chdir="$SCRIPT_DIR" "$@" -state="$TF_STATE" \
+        -var "deployment_name=$NAME" -var "region=$REGION" -var "plan=$PLAN" \
+        -var "ssh_public_key_path=$SSH_KEY.pub" -var "vpc_subnet=$VPC_SUBNET" \
+        -var "node_count=$NODES"
+}
+
+done_step() { [ -f "$STATE_DIR/step.$1" ]; }
+mark_step() { date -u +%FT%TZ > "$STATE_DIR/step.$1"; }
+
+# node_field FIELD prints FIELD of every node, in node order.
+node_field() {
+    "$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes |
+        python3 -c 'import json,sys; [print(n[sys.argv[1]]) for n in json.load(sys.stdin)]' "$1"
+}
+
+# for_nodes STEP FN runs FN once per node with IP and NODE_NAME set,
+# recording STEP.<node> so a rerun skips the nodes already done.
+for_nodes() {
+    local step="$1" fn="$2" i
+    for i in "${!IPS[@]}"; do
+        IP="${IPS[$i]}" NODE_NAME="node$((i + 1))"
+        done_step "$step.$NODE_NAME" && continue
+        "$fn"
+        mark_step "$step.$NODE_NAME"
+    done
+}
+
+# vpc_addr prints the VPC NIC's name and address on the current node.
+vpc_addr() { on 'ip -4 -o addr show' | awk -v p="${VPC_SUBNET%.*}." 'index($4, p) == 1 {sub(/\/.*/, "", $4); print $2, $4; exit}'; }
+
+on() {
+    ssh -i "$SSH_KEY" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+        -o UserKnownHostsFile="$KNOWN_HOSTS" "root@$IP" "$@"
+}
+
+# ── up ───────────────────────────────────────────────────────────────────────
+
+preflight() {
+    [ -r "$SSH_KEY" ] && [ -r "$SSH_KEY.pub" ] || die "SSH key $SSH_KEY (and .pub) not readable"
+    if [ -n "$DISTRO" ]; then
+        [ -r "$DISTRO" ] || die "--distro $DISTRO not readable"
+    elif [ -z "$CHANNEL" ]; then
+        die "pass --distro TARBALL or --channel latest|dev"
+    fi
+    vultr GET /account >/dev/null || die "the API key was refused; check its IP allowlist"
+    local type=vx1
+    [[ "$PLAN" == vbm-* ]] && type=vbm
+    vultr GET "/regions/$REGION/availability?type=$type" | python3 -c '
+import json,sys
+plan=sys.argv[1]
+if plan not in json.load(sys.stdin).get("available_plans",[]):
+    sys.exit("plan %s is not available in this region" % plan)' "$PLAN" || die "plan check failed"
+}
+
+provision() {
+    log "terraform apply ($PLAN in $REGION)"
+    "$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
+    tf apply -input=false -auto-approve >"$STATE_DIR/apply.log" 2>&1 || {
+        tail -20 "$STATE_DIR/apply.log" >&2
+        die "terraform apply failed; log in $STATE_DIR/apply.log"
+    }
+}
+
+# wait_host pins the host key only once the marker cloud-init writes exists:
+# bare metal first answers SSH from Vultr's installer, with a key of its own.
+wait_host() {
+    log "waiting for SSH and cloud-init on $IP"
+    local i
+    for i in $(seq 1 120); do
+        ssh -i "$SSH_KEY" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR "root@$IP" \
+            'test -f /var/lib/cloud/spinifex-host-ready' 2>/dev/null && break
+        [ "$i" = 120 ] && die "host $IP was not ready after 20 minutes"
+        sleep 10
+    done
+    ssh-keygen -R "$IP" -f "$KNOWN_HOSTS" >/dev/null 2>&1 || true
+    on 'cloud-init status --wait >/dev/null; test -e /dev/kvm' || die "no /dev/kvm on $IP"
+    # Vultr's image writes a static resolv.conf, so the node would bypass the
+    # split DNS Spinifex sets in resolved for its LB and EKS names.
+    on 'if [ ! -L /etc/resolv.conf ] && systemctl is-active -q systemd-resolved; then
+            ln -sf ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf; fi' ||
+        die "could not point /etc/resolv.conf at systemd-resolved on $IP"
+}
+
+install_spinifex() {
+    log "installing Spinifex"
+    if [ -n "$DISTRO" ]; then
+        scp -q -i "$SSH_KEY" -o BatchMode=yes -o UserKnownHostsFile="$KNOWN_HOSTS" \
+            "$DISTRO" "$SPINIFEX_SRC/scripts/setup.sh" "root@$IP:/tmp/" || die "copy to $IP failed"
+        on "INSTALL_SPINIFEX_TARBALL=/tmp/$(basename "$DISTRO") bash /tmp/setup.sh >/root/setup.log 2>&1" ||
+            die "setup.sh failed; see /root/setup.log on $IP"
+        # A tarball built by build-install-artifacts.sh ships setup.sh beside it
+        # rather than in it, and the later steps run the installed copy.
+        on 'test -x /usr/local/share/spinifex/setup.sh || install -D -m 0755 /tmp/setup.sh /usr/local/share/spinifex/setup.sh'
+    else
+        on "curl -sfL https://install.mulgadc.com | bash -s -- --channel $CHANNEL >/root/setup.log 2>&1" ||
+            die "install failed; see /root/setup.log on $IP"
+    fi
+}
+
+# form brings a node up as its own one-node cluster, the state an ISO install
+# leaves and the one install-node.sh forms a larger cluster from.
+form() {
+    local vpc_if vpc_ip
+    read -r vpc_if vpc_ip <<<"$(vpc_addr)"
+    [ -n "$vpc_ip" ] || die "no VPC address on $IP"
+    log "initialising $NODE_NAME on $IP (VPC $vpc_if $vpc_ip, region $SPX_REGION)"
+    on "/usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink --encap-ip=$vpc_ip >/root/setup-ovn.log 2>&1" ||
+        die "setup-ovn.sh failed; see /root/setup-ovn.log on $IP"
+    on "spx admin init --node $NODE_NAME --nodes 1 --region $SPX_REGION --az $SPX_AZ --ipsec=false --external-mode=nat >/root/init.log 2>&1" ||
+        die "spx admin init failed; see /root/init.log on $IP"
+}
+
+install_key() {
+    log "installing the API key on $NODE_NAME"
+    printf '%s' "$VULTR_API_KEY" | on 'install -d -m 0700 -o spinifex-daemon -g spinifex /etc/spinifex/vultr &&
+        umask 077 && cat > /etc/spinifex/vultr/api-key &&
+        chown spinifex-daemon:spinifex /etc/spinifex/vultr/api-key && chmod 0600 /etc/spinifex/vultr/api-key'
+    on 'curl -sf -o /dev/null -H "Authorization: Bearer $(cat /etc/spinifex/vultr/api-key)" https://api.vultr.com/v2/reserved-ips' ||
+        die "the node's API calls are refused; add $IP to the key's allowlist"
+}
+
+# configure_pool adds the pool and every node's instance ID: the config is
+# validated against all [nodes.*] tables, and a takeover attaches to the
+# taking node's own ID.
+configure_pool() {
+    log "adding the vultr pool on $NODE_NAME (label prefix $LABEL_PREFIX)"
+    local ids="" i
+    for i in "${!IDS[@]}"; do ids+="node$((i + 1))=${IDS[$i]},"; done
+    on "python3 - '$REGION' '$LABEL_PREFIX' '${ids%,}'" <<'PY'
+import re, sys
+region, prefix, ids = sys.argv[1:]
+p = "/etc/spinifex/spinifex.toml"
+s = open(p).read()
+if 'source       = "vultr"' not in s:
+    pool = ('[[network.external_pools]]\nname         = "vultr"\nsource       = "vultr"\n'
+            f'vultr_region = "{region}"\nlabel_prefix = "{prefix}"\n\n[bootstrap]')
+    assert s.count("\n[bootstrap]") == 1, "expected one [bootstrap] table"
+    s = s.replace("\n[bootstrap]", "\n" + pool, 1)
+for pair in ids.split(","):
+    node, host_id = pair.split("=")
+    head = "[nodes.%s]\n" % node
+    m = re.search(r"^" + re.escape(head) + r"(.*?)(?=^\[|\Z)", s, flags=re.M | re.S)
+    assert m, "could not find [nodes.%s]" % node
+    if "provider_host_id" not in m.group(1):
+        s = s[:m.start()] + head + 'provider_host_id = "%s"\n' % host_id + s[m.start() + len(head):]
+open(p, "w").write(s)
+PY
+}
+
+start_spinifex() {
+    log "arming the firewall and starting $NODE_NAME"
+    on 'env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on >/root/firewall.log 2>&1' ||
+        die "arming the firewall failed; see /root/firewall.log on $IP"
+    on 'systemctl start spinifex.target'
+    wait_api
+}
+
+wait_api() {
+    local i
+    for i in $(seq 1 30); do
+        on 'curl -sk -o /dev/null -w "%{http_code}" https://127.0.0.1:9999/' 2>/dev/null | grep -q '^[34]' && break
+        [ "$i" = 30 ] && die "the API did not come up on $IP"
+        sleep 10
+    done
+}
+
+# form_cluster hands the armed single nodes to install-node.sh. It dials the
+# public addresses over SSH and is told to put the cluster and Geneve planes on
+# the VPC NIC, which must have the same name on every node.
+form_cluster() {
+    local hosts_file="$STATE_DIR/hosts" vpc_if="" if_here ip_here pinned
+    printf '%s\n' "${IPS[@]}" > "$hosts_file"
+    for IP in "${IPS[@]}"; do
+        read -r if_here ip_here <<<"$(vpc_addr)"
+        [ -n "$ip_here" ] || die "no VPC address on $IP"
+        [ -z "$vpc_if" ] || [ "$vpc_if" = "$if_here" ] || die "VPC NIC is $vpc_if on one node and $if_here on $IP"
+        vpc_if="$if_here"
+        # install-node.sh reads ~/.ssh/known_hosts and Vultr reuses addresses, so
+        # replace any older key for this one with the key this deployment pinned.
+        # -F, not grep: ssh_config usually hashes the entries.
+        pinned="$(ssh-keygen -F "$IP" -f "$KNOWN_HOSTS" | grep -v '^#')" || die "no pinned host key for $IP in $KNOWN_HOSTS"
+        ssh-keygen -R "$IP" >/dev/null 2>&1 || true
+        printf '%s\n' "$pinned" >> "$HOME/.ssh/known_hosts"
+    done
+    log "forming ${#IPS[@]} nodes with install-node.sh (cluster plane $vpc_if)"
+    "$SPINIFEX_SRC/scripts/install-node.sh" --hosts-file "$hosts_file" --user root --identity "$SSH_KEY" \
+        --external-mode nat --ipsec off --region "$SPX_REGION" --az "$SPX_AZ" \
+        --lan-bridge "$vpc_if" --vpc-bridge "$vpc_if" --yes >"$STATE_DIR/form.log" 2>&1 ||
+        die "formation failed; see $STATE_DIR/form.log"
+}
+
+restart_spinifex() {
+    log "restarting $NODE_NAME with the pool"
+    on 'systemctl restart spinifex.target'
+    wait_api
+    local i
+    for i in $(seq 1 30); do
+        on 'journalctl -u spinifex-daemon --since "-10min" --no-pager | grep -q "floating using Vultr reserved IPs"' && return 0
+        sleep 10
+    done
+    die "$NODE_NAME never logged its Vultr allocator; see journalctl -u spinifex-daemon on $IP"
+}
+
+import_image() {
+    log "importing $IMAGE"
+    on "spx admin images import --name $IMAGE --config /etc/spinifex/spinifex.toml >/root/image-import.log 2>&1" ||
+        die "image import failed; see /root/image-import.log on $IP"
+}
+
+write_profile() {
+    log "writing the CA to $CA_FILE and AWS profile $PROFILE"
+    mkdir -p "$(dirname "$CA_FILE")"
+    on 'cat /etc/spinifex/ca.pem' > "$CA_FILE"
+    local creds key secret
+    creds="$(on 'awk "/^\[spinifex\]/{f=1;next} /^\[/{f=0} f" ~/.aws/credentials')"
+    key="$(awk -F' *= *' '/aws_access_key_id/{print $2}' <<<"$creds")"
+    secret="$(awk -F' *= *' '/aws_secret_access_key/{print $2}' <<<"$creds")"
+    [ -n "$key" ] && [ -n "$secret" ] || die "no [spinifex] credentials on $IP"
+    aws configure set --profile "$PROFILE" aws_access_key_id "$key"
+    aws configure set --profile "$PROFILE" aws_secret_access_key "$secret"
+    aws configure set --profile "$PROFILE" region "$SPX_REGION"
+    aws configure set --profile "$PROFILE" endpoint_url "https://$IP:9999"
+    aws configure set --profile "$PROFILE" ca_bundle "$CA_FILE"
+}
+
+cmd_up() {
+    preflight
+    done_step terraform || { provision; mark_step terraform; }
+    mapfile -t IPS < <(node_field public_ip)
+    mapfile -t IDS < <(node_field provider_host_id)
+    [ "${#IPS[@]}" = "$NODES" ] || die "the state holds ${#IPS[@]} node(s), not $NODES; destroy it or pass --nodes ${#IPS[@]}"
+    for_nodes host wait_host
+    for_nodes install install_spinifex
+    for_nodes form form
+    for_nodes start start_spinifex
+    if [ "$NODES" -gt 1 ]; then
+        done_step cluster || { form_cluster; mark_step cluster; }
+    fi
+    for_nodes key install_key
+    for_nodes pool configure_pool
+    for_nodes restart restart_spinifex
+    IP="${IPS[0]}"
+    if [ -n "$IMAGE" ]; then done_step image || { import_image; mark_step image; }; fi
+    write_profile
+    log "ready: https://$IP:9999 (profile $PROFILE)"
+    printf 'endpoint=https://%s:9999\nprofile=%s\nca=%s\nconsole=https://%s:3000/\n' "$IP" "$PROFILE" "$CA_FILE" "$IP"
+    local i
+    for i in "${!IPS[@]}"; do printf 'node%d=%s\n' "$((i + 1))" "${IPS[$i]}"; done
+    printf 'api_key_allowlist=%s\n' "$(printf '%s/32,' "${IPS[@]}" | sed 's/,$//')"
+}
+
+# ── destroy ──────────────────────────────────────────────────────────────────
+
+# leftovers prints every billable object this deployment can own, one per line.
+leftovers() {
+    {
+        vultr GET "/instances?per_page=500" | python3 -c '
+import json,sys
+for i in json.load(sys.stdin)["instances"]:
+    if "deployment:"+sys.argv[1] in i.get("tags",[]): print("instance", i["id"], i["label"])' "$NAME"
+        vultr GET "/bare-metals?per_page=500" | python3 -c '
+import json,sys
+for i in json.load(sys.stdin)["bare_metals"]:
+    if "deployment:"+sys.argv[1] in i.get("tags",[]): print("bare-metal", i["id"], i["label"])' "$NAME"
+        vultr GET "/blocks?per_page=500" | python3 -c '
+import json,sys
+for b in json.load(sys.stdin)["blocks"]:
+    if b["label"].startswith("spinifex-"+sys.argv[1]+"-"): print("block", b["id"], b["label"])' "$NAME"
+        vultr GET "/reserved-ips?per_page=500" | python3 -c '
+import json,sys
+for r in json.load(sys.stdin)["reserved_ips"]:
+    if r["label"].startswith(sys.argv[1]+":"): print("reserved-ip", r["id"], r["subnet"])' "$LABEL_PREFIX"
+        vultr GET "/vpcs?per_page=500" | python3 -c '
+import json,sys
+for v in json.load(sys.stdin)["vpcs"]:
+    if v["description"] == "spinifex-"+sys.argv[1]: print("vpc", v["id"], v["description"])' "$NAME"
+    }
+}
+
+cmd_destroy() {
+    if [ "$YES" != 1 ]; then
+        log "about to destroy deployment '$NAME':"
+        leftovers >&2
+        read -r -p "Type the deployment name to confirm: " answer
+        [ "$answer" = "$NAME" ] || die "not confirmed"
+    fi
+    if [ -f "$TF_STATE" ]; then
+        log "terraform destroy"
+        "$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
+        tf destroy -input=false -auto-approve >"$STATE_DIR/destroy.log" 2>&1 || {
+            tail -20 "$STATE_DIR/destroy.log" >&2
+            die "terraform destroy failed; log in $STATE_DIR/destroy.log"
+        }
+    fi
+
+    # Reserved IPs the node created, and boot volumes the instance left behind.
+    local kind id label i
+    for i in 1 2 3 4 5 6; do
+        local left
+        left="$(leftovers)"
+        [ -z "$left" ] && break
+        while read -r kind id label; do
+            case "$kind" in
+                reserved-ip)
+                    log "deleting reserved IP $label"
+                    vultr POST "/reserved-ips/$id/detach" >/dev/null || true
+                    vultr DELETE "/reserved-ips/$id" >/dev/null || true ;;
+                block)
+                    log "deleting block volume $label"
+                    vultr DELETE "/blocks/$id" >/dev/null || true ;;
+                instance|bare-metal|vpc)
+                    log "deleting $kind $label, still listed after terraform destroy"
+                    vultr DELETE "/${kind}s/$id" >/dev/null || true ;;
+            esac
+        done <<<"$left"
+        sleep 10
+    done
+
+    local left
+    left="$(leftovers)"
+    if [ -n "$left" ]; then
+        log "these still exist and are billing:"
+        printf '%s\n' "$left" >&2
+        exit 1
+    fi
+    rm -f "$STATE_DIR"/step.*
+    log "deployment '$NAME' destroyed; nothing of it is left in the account"
+}
+
+cmd_status() {
+    local nodes
+    if [ -f "$TF_STATE" ] && nodes="$("$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes 2>/dev/null)"; then
+        python3 -c 'import json,sys; [print(n["name"], n["public_ip"]) for n in json.load(sys.stdin)]' <<<"$nodes"
+    fi
+    leftovers
+    find "$STATE_DIR" -maxdepth 1 -name 'step.*' | sort | sed 's/.*step\./done: /'
+}
+
+# node_ip NODE prints the public address of nodeN from the Terraform state.
+node_ip() {
+    [ -f "$TF_STATE" ] || die "no deployment '$NAME'; run up first"
+    local ip
+    ip="$(node_field public_ip | sed -n "${1#node}p")"
+    [ -n "$ip" ] || die "deployment '$NAME' has no $1"
+    printf '%s' "$ip"
+}
+
+cmd_credentials() {
+    IP="$(node_ip node1)"
+    on 'awk "/^\[spinifex\]/{f=1} /^\[/&&!/^\[spinifex\]/{f=0} f" ~/.aws/credentials'
+}
+
+# cmd_power ACTION halts or starts NODE_ARG through the Vultr API. A halt is a
+# hard power-off, so the node gets no chance to hand anything over first.
+cmd_power() {
+    [ -n "$NODE_ARG" ] || die "$CMD needs a node, such as node2"
+    local id path
+    IP="$(node_ip "$NODE_ARG")"
+    id="$(node_field provider_host_id | sed -n "${NODE_ARG#node}p")"
+    path="/instances/$id"
+    "$TF_BIN" -chdir="$SCRIPT_DIR" state list -state="$TF_STATE" | grep -q '^vultr_bare_metal_server\.' &&
+        path="/bare-metals/$id"
+    vultr POST "$path/$1" >/dev/null || die "Vultr refused to $1 $NODE_ARG"
+    log "$NODE_ARG ($IP): $1 requested"
+}
+
+cmd_ssh() {
+    IP="$(node_ip "${NODE_ARG:-node1}")"
+    ssh -i "$SSH_KEY" -o UserKnownHostsFile="$KNOWN_HOSTS" "root@$IP" "$@"
+}
+
+case "$CMD" in
+    up) cmd_up ;;
+    destroy) cmd_destroy ;;
+    status) cmd_status ;;
+    credentials) cmd_credentials ;;
+    ssh) cmd_ssh "$@" ;;
+    stop) cmd_power halt ;;
+    start) cmd_power start ;;
+    *) usage >&2; exit 2 ;;
+esac

scripts/terraform/vultr-spx/e2e-vultr.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# e2e-vultr.sh — prove a ref works on Vultr: deploy it with deploy-vultr.sh, run
+# the published Terraform workbooks against it from node1, then destroy it and
+# fail if anything is left billing. One entry point for a workstation and CI.
+#
+# Configured by environment variables, so a dispatch needs no arguments:
+#
+#   VULTR_API_KEY        Required. Its access control must admit this host and
+#                        every node, so a CI key needs 0.0.0.0/0 or a fixed runner.
+#   SPX_REF              Ref to prove. Default: the checkout's current branch.
+#   VULTR_SOURCE         tree (build SPX_REF, the default) or release.
+#   VULTR_CHANNEL        With VULTR_SOURCE=release: latest (default) or dev.
+#   VULTR_NODES          1 (default) or 3.
+#   VULTR_PLAN           Default vx1-g-16c-64g; vbm-8c-64gb-amd for bare metal.
+#   VULTR_REGION         Default ord.
+#   VULTR_NAME           Deployment name. Default e2e-<UTC stamp>.
+#   WORKBOOKS            Workbooks to run. Unset: the list below. "": none.
+#   VULTR_KEEP_ON_FAIL   1 leaves a failed deployment up. It keeps billing.
+#   VULTR_ARTIFACT_DIR   Logs and the verdict. Default ./.e2e-vultr-<stamp>.
+#   MULGA_ROOT           mulga checkout, for the artifact builder.
+#
+#   e2e-vultr.sh [--sweep-only]   --sweep-only destroys VULTR_NAME and stops.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SPINIFEX_ROOT="$(cd "$HERE/../../.." && pwd)"
+MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+
+REF="${SPX_REF:-$(git -C "$SPINIFEX_ROOT" rev-parse --abbrev-ref HEAD)}"
+SOURCE="${VULTR_SOURCE:-tree}"
+CHANNEL="${VULTR_CHANNEL:-latest}"
+NODES="${VULTR_NODES:-1}"
+PLAN="${VULTR_PLAN:-vx1-g-16c-64g}"
+REGION="${VULTR_REGION:-ord}"
+NAME="${VULTR_NAME:-e2e-$(date -u +%m%d%H%M%S)}"
+ARTIFACT_DIR="${VULTR_ARTIFACT_DIR:-$HERE/.e2e-vultr-$STAMP}"
+DEPLOY="$HERE/deploy-vultr.sh"
+DEFAULT_WORKBOOKS="nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart ecs-quickstart eks-quickstart eks-https-ingress eks-gitops-argocd"
+
+log() { printf '[e2e-vultr] %s\n' "$*"; }
+die() { log "ERROR: $*"; echo "run: FAIL ($*)" >> "$VERDICT"; finish 1; }
+
+mkdir -p "$ARTIFACT_DIR"
+VERDICT="$ARTIFACT_DIR/verdict.txt"
+[ "${1:-}" = --sweep-only ] && VERDICT="$ARTIFACT_DIR/sweep-verdict.txt"
+: > "$VERDICT"
+[ -n "${VULTR_API_KEY:-}" ] || { log "VULTR_API_KEY is not set"; exit 2; }
+export VULTR_API_KEY
+
+# A throwaway key per run unless one is given, so CI needs no SSH secret and a
+# key never outlives the deployment it was made for.
+SSH_KEY="${VULTR_SSH_KEY:-$ARTIFACT_DIR/ssh/id_ed25519}"
+if [ ! -f "$SSH_KEY" ]; then
+    mkdir -p "$(dirname "$SSH_KEY")" && chmod 700 "$(dirname "$SSH_KEY")"
+    ssh-keygen -q -t ed25519 -N "" -C "spinifex-e2e-$NAME" -f "$SSH_KEY"
+fi
+DEPLOY_ARGS=(--name "$NAME" --nodes "$NODES" --plan "$PLAN" --region "$REGION" --ssh-key "$SSH_KEY")
+
+# on_node1 [ssh options --] command: the options go before the destination,
+# where ssh reads them, and everything after -- is the remote command.
+on_node1() {
+    local ip opts=()
+    if [[ " $* " == *" -- "* ]]; then
+        while [ "$1" != -- ]; do opts+=("$1"); shift; done
+        shift
+    fi
+    ip="$("$DEPLOY" status "${DEPLOY_ARGS[@]}" 2>/dev/null | awk '$1 == "node1" {print $2}')"
+    [ -n "$ip" ] || { log "no node1 address in deploy-vultr.sh status"; return 1; }
+    ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+        -o BatchMode=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=6 "${opts[@]}" "root@$ip" "$@"
+}
+
+# destroy is also the last assertion: it fails, listing what remains, if anything
+# of the deployment is still in the account afterwards.
+sweep() {
+    log "destroying deployment $NAME"
+    if "$DEPLOY" destroy "${DEPLOY_ARGS[@]}" --yes > "$ARTIFACT_DIR/destroy.log" 2>&1; then
+        echo "teardown: PASS" >> "$VERDICT"
+    else
+        tail -20 "$ARTIFACT_DIR/destroy.log"
+        echo "teardown: FAIL (see destroy.log)" >> "$VERDICT"
+        return 1
+    fi
+}
+
+# collect_node_logs copies each node's install and deploy logs into the
+# artifacts before a teardown takes them, so a failed run can be diagnosed.
+collect_node_logs() {
+    local name ip
+    "$DEPLOY" status "${DEPLOY_ARGS[@]}" 2>/dev/null | awk '/^node[0-9] / {print $1, $2}' |
+        while read -r name ip; do
+            mkdir -p "$ARTIFACT_DIR/$name"
+            scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                -o BatchMode=yes -o ConnectTimeout=10 -q "root@$ip:/root/*.log" "$ARTIFACT_DIR/$name/" ||
+                log "could not copy logs from $name ($ip)"
+        done
+}
+
+finish() {
+    local rc="$1"
+    [ "$rc" = 0 ] || collect_node_logs
+    if [ "$rc" != 0 ] && [ "${VULTR_KEEP_ON_FAIL:-0}" = 1 ]; then
+        log "keeping $NAME up for inspection; it keeps billing until destroyed"
+    else
+        sweep || rc=1
+    fi
+    grep -q FAIL "$VERDICT" && rc=1
+    log "=== $REF on Vultr ($NODES x $PLAN in $REGION, $SOURCE build) ==="
+    cat "$VERDICT"
+    [ "$rc" = 0 ] && log PASS || log FAIL
+    exit "$rc"
+}
+
+if [ "${1:-}" = --sweep-only ]; then
+    sweep
+    exit $?
+fi
+
+src_args=()
+if [ "$SOURCE" = tree ]; then
+    git -C "$SPINIFEX_ROOT" rev-parse --verify --quiet "refs/remotes/origin/$REF" >/dev/null ||
+        die "$REF is not on origin: the builder clones from GitHub and would build dev instead"
+    BUILDER="$MULGA_ROOT/scripts/tofu-cluster/build-install-artifacts.sh"
+    [ -x "$BUILDER" ] || die "no artifact builder at $BUILDER (set MULGA_ROOT)"
+    log "building $REF"
+    "$BUILDER" "$REF" "$ARTIFACT_DIR/spinifex-distro.tar.gz" "$ARTIFACT_DIR/setup.sh" \
+        "$ARTIFACT_DIR/spinifex-tests.tar" --skip-e2e-build > "$ARTIFACT_DIR/build.log" 2>&1 ||
+        die "building $REF failed; see build.log"
+    src_args=(--distro "$ARTIFACT_DIR/spinifex-distro.tar.gz")
+else
+    src_args=(--channel "$CHANNEL")
+fi
+
+log "deploying $NAME: $NODES x $PLAN in $REGION"
+start=$SECONDS
+if "$DEPLOY" up "${DEPLOY_ARGS[@]}" "${src_args[@]}" > "$ARTIFACT_DIR/deploy.log" 2>&1; then
+    echo "deploy: PASS ($((SECONDS - start))s)" >> "$VERDICT"
+else
+    tail -30 "$ARTIFACT_DIR/deploy.log"
+    die "deploy-vultr.sh up failed; see deploy.log"
+fi
+
+if [ -n "${WORKBOOKS+x}" ] && [ -z "$WORKBOOKS" ]; then
+    echo "workbooks: SKIPPED (none requested)" >> "$VERDICT"
+    finish 0
+fi
+WORKBOOKS="${WORKBOOKS-$DEFAULT_WORKBOOKS}"
+
+log "importing the images the workbooks need"
+on_node1 'set -e; for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
+    spx admin images import --name "$img" >/dev/null; done' > "$ARTIFACT_DIR/images.log" 2>&1 ||
+    die "image import failed; see images.log"
+
+ip1="$("$DEPLOY" status "${DEPLOY_ARGS[@]}" 2>/dev/null | awk '$1 == "node1" {print $2}')"
+scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -q \
+    "$SPINIFEX_ROOT/tests/e2e/run-tofu-examples-e2e.sh" "root@$ip1:/root/run-tofu-examples-e2e.sh" &&
+    scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -qr \
+        "$SPINIFEX_ROOT/docs/terraform-workbooks" "root@$ip1:/root/workbooks" ||
+    die "could not copy the workbook driver and workbooks to node1"
+
+# The same driver and workbooks as the hypervisor nightly and OCI. The reverse
+# SOCKS forward gives it this host's vantage, outside Vultr, for public addresses.
+log "running the workbooks: $WORKBOOKS"
+if on_node1 -o ExitOnForwardFailure=yes -R 1080 -- \
+    "chmod +x /root/run-tofu-examples-e2e.sh; WORKBOOK_DIR=/root/workbooks $(printf 'WORKBOOKS=%q' "$WORKBOOKS") E2E_PUBLIC_PROXY=127.0.0.1:1080 /root/run-tofu-examples-e2e.sh" \
+    2>&1 | tee "$ARTIFACT_DIR/workbooks.log"; then
+    rc=0
+else
+    rc=1
+fi
+grep -E '^--- (PASS|FAIL|SKIP)' "$ARTIFACT_DIR/workbooks.log" | sed 's/^--- /workbook /' >> "$VERDICT"
+passed="$(grep -c '^--- PASS' "$ARTIFACT_DIR/workbooks.log" || true)"
+ran="$(grep -c '^=== RUN' "$ARTIFACT_DIR/workbooks.log" || true)"
+if [ "$rc" = 0 ]; then
+    echo "workbooks: PASS ($passed of $ran)" >> "$VERDICT"
+else
+    echo "workbooks: FAIL ($passed of $ran passed)" >> "$VERDICT"
+fi
+finish "$rc"

scripts/terraform/vultr-spx/network.tf

@@ -0,0 +1,8 @@
+# One VPC per deployment, single node included, so the encap address and the
+# east-west path are the same shape at one node as at three.
+resource "vultr_vpc" "lan" {
+  region         = var.region
+  description    = "spinifex-${var.deployment_name}"
+  v4_subnet      = var.vpc_subnet
+  v4_subnet_mask = var.vpc_subnet_mask
+}

scripts/terraform/vultr-spx/outputs.tf

@@ -0,0 +1,27 @@
+output "deployment_name" {
+  value = var.deployment_name
+}
+
+output "region" {
+  value = var.region
+}
+
+output "vpc_id" {
+  value = vultr_vpc.lan.id
+}
+
+# provider_host_id is what a reserved IP attaches to. The VPC address is read
+# on the host, since the provider does not report it.
+output "nodes" {
+  value = [for n in concat(vultr_instance.node, vultr_bare_metal_server.node) : {
+    name             = n.hostname
+    provider_host_id = n.id
+    public_ip        = n.main_ip
+  }]
+}
+
+# The addresses the Vultr API key's access control must allow once the
+# deployment is up, besides the machine that runs deploy-vultr.sh.
+output "api_key_allowlist" {
+  value = [for n in concat(vultr_instance.node, vultr_bare_metal_server.node) : "${n.main_ip}/32"]
+}