release: update go.mod for v1.12.1

fix(network): poll second-VPC transit SNAT in natuplink + log AddSNAT install (#541 )
Phase 8 of TestNATUplink checked the second VPC router's lr-nat-list with a single non-polling assert; the igw-default-snat row commits a beat after the gateway LRP appears, so the read raced the OVN commit and intermittently saw an empty list (flaky CI red). Wrap it in EventuallyErr, matching the sibling LRP and host-route polls in the same phase. AddSNAT logged only on the idempotent-skip path, so the first install was silent and RCA could not tell a late SNAT from a missing one. Emit an install log keyed by router + vpc_cidr + external_ip.
2026-10-24 21:23:51 +00:00 · 2026-07-15 09:29:22 +10:00 · 2026-07-15 09:26:15 +10:00 · 2026-07-15 09:00:35 +10:00 · 2026-07-15 08:31:09 +10:00 · 2026-07-15 07:54:21 +10:00
1494 changed files with 102842 additions and 101308 deletions
@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -463,8 +462,7 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }

-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report (Stage 1).
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.

 import (
@@ -0,0 +1,102 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">ECS · EC2 LAUNCH TYPE ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">container instances from spinifex-ecs-node AMI · awsvpc + ALB</text>
+
+  <!-- Internet -->
+  <rect x="120" y="52" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="220" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to ALB -->
+  <line x1="220" y1="92" x2="220" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="216,108 224,108 220,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- ALB -->
+  <g transform="translate(90, 116)">
+    <rect width="260" height="60" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">Application Load Balancer</text>
+    <line x1="16" y1="32" x2="244" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787">target group · target_type = ip</text>
+  </g>
+
+  <!-- Spinifex Gateway (control plane) -->
+  <g transform="translate(560, 110)">
+    <rect width="316" height="72" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">Spinifex Gateway :9999</text>
+    <text x="296" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">control plane</text>
+    <line x1="16" y1="32" x2="300" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">ECS agent registers · TLS + SigV4 (IMDS creds)</text>
+    <text x="20" y="63" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">task role creds → container at 169.254.170.2</text>
+  </g>
+
+  <!-- ALB fan-out to subnets (traffic) -->
+  <line x1="220" y1="176" x2="220" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="650" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="220" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="650" y1="200" x2="650" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="435" y="195" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">balances to each task's ENI</text>
+
+  <!-- agent path: subnets up to gateway -->
+  <line x1="718" y1="228" x2="718" y2="182" stroke="#F87171" stroke-width="1.1" stroke-dasharray="3,3" opacity="0.6"/>
+  <polygon points="714,186 722,186 718,178" fill="#F87171" opacity="0.8"/>
+
+  <!-- subnet_a -->
+  <g transform="translate(60, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- container instance -->
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="225" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(520, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="103" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">image pull</text>
+    <text x="225" y="117" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">registry / ECR</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="410" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">No ECS launch API — you add capacity by booting EC2 instances from the spinifex-ecs-node AMI (the console's "Provision capacity" wraps RunInstances)</text>
+  <text x="450" y="426" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">The service keeps the desired task count running and registers each task's ENI IP with the ALB target group</text>
+</svg>
@@ -0,0 +1,96 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="cp-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">EKS · MANAGED KUBERNETES ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">control plane VM + NLB · workers from eks-node AMI</text>
+
+  <!-- Internet / kubectl -->
+  <rect x="330" y="52" width="240" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet · kubectl + aws eks</text>
+
+  <!-- connector to control plane -->
+  <line x1="450" y1="92" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,108 454,108 450,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- Control plane (Spinifex-managed) -->
+  <g transform="translate(290, 116)">
+    <rect width="320" height="76" rx="8" fill="url(#cp-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">EKS Control Plane</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">Spinifex-managed</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="87" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">API NLB (public)</text>
+    <rect x="165" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="232" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">apiserver VM</text>
+  </g>
+
+  <!-- authn note -->
+  <rect x="624" y="138" width="172" height="32" rx="6" fill="#0a0a0a" stroke="#71717a" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="710" y="151" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">authn: API · access entries</text>
+  <text x="710" y="163" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">aws eks get-token (IAM → RBAC)</text>
+
+  <!-- fan-out to node-group subnets -->
+  <line x1="450" y1="192" x2="450" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="670" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="230" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="670" y1="210" x2="670" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="450" y="206" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">managed node group · workers join the control plane</text>
+
+  <!-- subnet_a -->
+  <g transform="translate(108, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- worker node -->
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(472, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">ECR</text>
+    <text x="248" y="107" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">pull</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="404" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">You provide the VPC, subnets, and IAM roles · Spinifex provisions the control plane, node-group VMs, and security groups</text>
+  <text x="450" y="420" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Node groups always boot the registered eks-node image · workers need egress (IGW / NAT) to pull images</text>
+</svg>
@@ -16,6 +16,10 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  ddil:
@@ -200,6 +204,16 @@ jobs:
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30
@@ -17,6 +17,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  matrix:
@@ -35,6 +39,7 @@ jobs:
          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
+          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }

    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
    timeout-minutes: 35
@@ -121,7 +126,12 @@ jobs:
        env:
          MATRIX_NET: ${{ matrix.net }}
        run: |
-          if [ "$MATRIX_NET" = "dhcp" ]; then
+          if [ "$MATRIX_NET" = "nat" ]; then
+            # Routed-NAT mode with a static public pool: exercises both the
+            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
+            export SPX_EXTERNAL_MODE=nat
+            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
+          elif [ "$MATRIX_NET" = "dhcp" ]; then
            unset SPX_EXTERNAL_POOL
            export SPX_EXTERNAL_SOURCE=dhcp
          else
@@ -154,6 +164,28 @@ jobs:
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

+      - name: Run NAT Uplink E2E
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        # Skips cleanly (exit 0) when the tested branch predates the natuplink
+        # suite so scheduled runs against dev stay green until the merge lands.
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
        working-directory: mulga/scripts/tofu-cluster
@@ -312,9 +344,19 @@ jobs:
            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
          fi
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
@@ -44,6 +44,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.ref_name }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # ---------------------------------------------------------------------------
@@ -51,7 +55,7 @@ jobs:
  # ---------------------------------------------------------------------------
  single_provision:
    name: Single env · provision
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 40
    outputs:
      wan_ip: ${{ steps.vm.outputs.wan_ip }}
@@ -199,7 +203,7 @@ jobs:
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      - name: Install kubectl on VM
@@ -263,10 +267,19 @@ jobs:
            "$VM":/tmp/start-net-capture.sh
          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  single_suite:
    name: Single env · suite
    needs: single_provision
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 45
    strategy:
      fail-fast: false
@@ -373,9 +386,9 @@ jobs:

  single_teardown:
    name: Single env · teardown
-    needs: [ single_provision, single_suite ]
+    needs: [single_provision, single_suite]
    if: always()
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 20
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
@@ -458,17 +471,27 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  # ---------------------------------------------------------------------------
  # Multinode env
  # ---------------------------------------------------------------------------
  multi_provision:
    name: Multi env · provision
    if: ${{ inputs.run_multinode }}
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 45
    outputs:
      node1_ip: ${{ steps.cluster.outputs.node1_ip }}
@@ -612,15 +635,24 @@ jobs:
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
          done

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  multi_suite:
    name: Multi env · suite
    needs: multi_provision
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
-        suite: [ multinode ]
+        suite: [multinode]
    env:
      NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -723,9 +755,9 @@ jobs:

  multi_teardown:
    name: Multi env · teardown
-    needs: [ multi_provision, multi_suite ]
+    needs: [multi_provision, multi_suite]
    if: always() && inputs.run_multinode
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 20
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
@@ -779,15 +811,25 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  e2e_suites_status:
    name: E2E suite blocks · summary
    if: always()
-    needs: [ single_suite, multi_suite ]
-    runs-on: ubuntu-latest
+    needs: [single_suite, multi_suite]
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Summarize
@@ -31,6 +31,10 @@ env:
  GIT_BRANCH: ${{ github.ref_name }}
  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
@@ -38,7 +42,7 @@ jobs:
  # can compare predicted vs actual before flipping it on.
  select_suites:
    name: Select Suites (shadow)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v7
@@ -224,13 +228,10 @@ jobs:
          scp "${SSH_OPTS[@]}" \
            /tmp/eks-node-image-build/eks-node-alpine.raw \
            "$VM":/tmp/eks-node-alpine.raw
-          # Drop the import progress bar (pterm redraws to stdout — thousands of
-          # lines over the SSH pipe). Errors go to stderr, so they survive; the
-          # exit code still gates the step.
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -460,10 +461,20 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
      - name: Fail job if any suite failed
        if: always()
        run: |
@@ -668,9 +679,6 @@ jobs:
            echo "=== systemctl is-active spinifex-* ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
            echo
-            echo "=== which spx + version ==="
-            ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true'
-            echo
            echo "=== spx get nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
            echo
@@ -877,10 +885,20 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
      - name: Fail job if any suite failed
        if: always()
        run: |
@@ -894,7 +912,7 @@ jobs:
    name: Report E2E Status
    if: always()
    needs: [e2e_single, e2e_multi]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
@@ -0,0 +1,369 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+  push:
+    branches:
+      - main
+      - dev
+    paths: &gpu_paths
+      - "spinifex/gpu/**"
+      - "spinifex/instancetypes/**"
+      - "spinifex/daemon/gpu_adapter.go"
+      - "spinifex/daemon/gpu_pool_reconciler.go"
+      - "spinifex/handlers/ecs/gpu_report.go"
+      - "spinifex/handlers/eks/nodegroup.go"
+      - "spinifex/handlers/eks/nodegroup_userdata.go"
+      - "spinifex/vm/vm.go"
+      - "spinifex/vm/lifecycle.go"
+      - "cmd/ecs-agent/gpu_discovery.go"
+      - "cmd/ecs-agent/gpu_ledger.go"
+      - "cmd/spinifex/cmd/admin_gpu.go"
+      - "tests/e2e/gpu/**"
+      - "scripts/images/eks-node-gpu/**"
+      - "scripts/images/ecs-agent-gpu/**"
+      - ".github/workflows/gpu-e2e.yml"
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths: *gpu_paths
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v6
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway (mulga-7suds), so the override is required
+      # here (proper fix tracked separately: tests/e2e/harness/env.go should
+      # prefer spinifex.toml's advertised bridge).
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi
@@ -1,16 +1,34 @@
-name: Publish EKS node AMI
+name: Publish EKS + ECS node AMIs

-# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
+# iso.mulgadc.com/system-ami/. Four images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+# Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger.
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.

 on:
  workflow_dispatch:
    inputs:
+      variant:
+        description: "Which images to build"
+        type: choice
+        default: all
+        options:
+          - all
+          - cpu
+          - gpu
      dry_run:
        description: "Build + convert only; skip the R2 upload"
        type: boolean
@@ -24,9 +42,37 @@ concurrency:
  cancel-in-progress: false

 jobs:
-  publish:
-    name: Build + publish eks-node AMI
+  matrix:
+    name: Resolve image matrix
    runs-on: [ self-hosted, ci-single ]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            *)   items="${cpu},${gpu}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [ self-hosted, ci-single ]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}

    steps:
      - name: Checkout Spinifex
@@ -59,7 +105,13 @@ jobs:
          cache: false

      - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        # Idempotent: the ci-single runner reuses one workspace across matrix
+        # legs and runs, and checkout never scrubs the root go.work. Remove any
+        # stale file so each leg re-inits cleanly instead of failing on
+        # "go.work already exists".
+        run: |
+          rm -f go.work go.work.sum
+          go work init ./spinifex ./viperblock ./predastore

      - name: Install build + upload tooling
        run: |
@@ -73,7 +125,7 @@ jobs:
            rm -rf /tmp/awscliv2.zip /tmp/aws-install
          fi

-      - name: Build + publish to R2
+      - name: Build + publish ${{ matrix.name }} to R2
        working-directory: spinifex
        env:
          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -82,15 +134,20 @@ jobs:
          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
          R2_BUCKET: spinifex-iso
          DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
        run: |
          set -euo pipefail
          # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
          # direct backend = no libvirt on the runner.
          export LIBGUESTFS_BACKEND=direct
          sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
          FLAGS=(--build)
          if [[ "${DRY_RUN}" == "true" ]]; then
            FLAGS+=(--dry-run)
          fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"
@@ -34,7 +34,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v6
        with:
-          go-version: "1.26.4"
+          go-version: "1.26.5"

      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
        run: cd spinifex && make build-microvm-image
@@ -57,7 +57,6 @@ linters:
      check-type-assertions: true
      exclude-functions:
        - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
-        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable

    govet:
      enable-all: true
@@ -68,6 +68,10 @@ build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
 	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go

+build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
+	@echo -e "\n....Building ecs-agent (static)"
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
@@ -90,6 +94,12 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build

+build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=ecs-agent
+
+import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -208,9 +218,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base

 install-go:
-	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -287,7 +297,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/

-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
@@ -192,6 +192,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever

 This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.

+## Trademarks
+
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
+
 ## License

 Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
@@ -0,0 +1,50 @@
+#!/bin/sh
+# JetStream liveness watchdog for spinifex-nats.
+#
+# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
+# flushing stream state) and keep the process alive serving KV as read-only. That
+# is not a process failure, so Restart=on-failure never fires and KV-backed
+# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+#
+# This probes the NATS monitoring endpoint's JS health and restarts the service
+# only when JetStream is sustainedly unhealthy while the service is running. The
+# multi-sample window avoids restarting during transient startup/quorum 503s.
+
+set -eu
+
+HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
+SERVICE="spinifex-nats.service"
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
+    exit 0
+fi
+
+# curl is the probe transport; if it is unavailable, do nothing rather than
+# false-restart a healthy server.
+if ! command -v curl >/dev/null 2>&1; then
+    echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2
+    exit 0
+fi
+
+# A single 200 means JetStream is healthy — exit fast on the common case.
+js_healthy() {
+    curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
+}
+
+i=1
+while [ "$i" -le "$SAMPLES" ]; do
+    if js_healthy; then
+        exit 0
+    fi
+    if [ "$i" -lt "$SAMPLES" ]; then
+        sleep "$SAMPLE_GAP"
+    fi
+    i=$((i + 1))
+done
+
+echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+systemctl try-restart "$SERVICE"
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-gw
@@ -6,6 +6,7 @@ After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.servi
 Wants=openvswitch-ipsec.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -34,7 +35,9 @@ DeviceAllow=/dev/net/tun rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf

 # RG-9: hardening baseline
 ProtectHome=yes
@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog
+PartOf=spinifex.target
+After=spinifex-nats.service
+
+[Service]
+Type=oneshot
+# Runs as root: the probe is a local curl to the NATS monitoring port, and the
+# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
+# the process stays alive (which Restart=on-failure cannot catch).
+ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+
+# Hardening (kept compatible with talking to PID1 over /run/systemd).
+NoNewPrivileges=yes
+ProtectHome=yes
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+LockPersonality=yes
+RestrictSUIDSGID=yes
@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog timer
+PartOf=spinifex.target
+
+[Timer]
+# Delay the first probe so NATS startup + JetStream cluster quorum settle (a
+# forming cluster reports JS not-current transiently, which is not a fault).
+OnBootSec=60s
+OnUnitActiveSec=30s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-nats
@@ -5,6 +5,10 @@ After=spinifex-nats.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -0,0 +1,56 @@
+[Unit]
+Description=Spinifex QMP Telemetry Collector (guest metrics)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-daemon.service
+Wants=spinifex-nats.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# spinifex-daemon user: the collector reads the daemon's telemetry QMP
+# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.
+User=spinifex-daemon
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service qmp-collector start
+Restart=on-failure
+RestartSec=5
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
+# Same runtime dir as the daemon: telemetry sockets + metadata live there
+Environment=XDG_RUNTIME_DIR=/run/spinifex
+
+# RG-9: read-mostly telemetry tier — no capabilities, no device access
+ProtectSystem=full
+PrivateDevices=yes
+NoNewPrivileges=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads
+# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=512
+
+[Install]
+WantedBy=spinifex.target
@@ -0,0 +1,18 @@
+[Unit]
+Description=Spinifex graceful guest drain
+PartOf=spinifex.target
+# After= all services so it stops FIRST on target/host shutdown: the drain runs
+# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).
+After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/true
+# ExecStop only runs when the target/host goes down, not on a single-service
+# restart — so a daemon restart still leaves guests running (DDIL).
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+TimeoutStopSec=180
+
+[Install]
+WantedBy=spinifex.target
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 After=spinifex-awsgw.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-ui
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-viperblock
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ope
 Wants=spinifex-nats.service ovn-controller.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-vpcd
@@ -17,26 +18,34 @@ OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/

-# RG-9: network tier. vpcd MUST share the host mount namespace — IMDS enters
-# per-VPC netns via setns(), and any filesystem-sandbox directive (ProtectSystem/
-# Home, PrivateTmp, ProtectKernel*, ProtectControlGroups/Proc, Read*Paths) forks a
-# private mount ns that traps the /run/netns bind-mount and breaks setns(2). Those,
-# plus MemoryDenyWriteExecute, are omitted by design — not regressions.
-PrivateDevices=no
-RestrictSUIDSGID=no
+# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
+# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
+# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+NoNewPrivileges=no
+ProtectSystem=full
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
 LockPersonality=yes
 RestrictRealtime=yes
+RestrictNamespaces=yes
 ProtectClock=yes
-# mnt alongside net: in-netns `ip` unshares a mount ns for its own /sys
-RestrictNamespaces=net mnt
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
 SystemCallArchitectures=native
-MemoryDenyWriteExecute=no
+MemoryDenyWriteExecute=yes
 # AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
-# Caps: NET_ADMIN (ovs/ip), NET_RAW (AF_PACKET), NET_BIND_SERVICE (:68), SETUID/SETGID, SYS_ADMIN (setns into VPC netns)
-AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-NoNewPrivileges=no
 ProtectHostname=yes
 KeyringMode=private
 RemoveIPC=yes
@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service

 [Install]
 WantedBy=multi-user.target
@@ -13,9 +13,6 @@ package main

 import (
 	"bufio"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -25,11 +22,8 @@ import (
 	"strings"
 	"time"

-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/ecr"
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/imdscreds"

 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
@@ -44,7 +38,6 @@ const (
 	defaultEnvFile   = "/etc/spinifex-eks/agent.env"
 	defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
 	defaultIMDSBase  = "http://169.254.169.254/latest"
-	imdsTokenTTL     = "21600"
 )

 // CredentialProviderRequest is the kubelet exec-provider request on STDIN.
@@ -80,7 +73,7 @@ type config struct {
 func main() {
 	cfg := loadConfig(defaultEnvFile)

-	httpClient, err := gatewayHTTPClient(cfg.GatewayCA)
+	httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
 	if err != nil {
 		slog.Error("ecr-credential-provider: build gateway client", "err", err)
 		emitEmpty(os.Stdout)
@@ -106,155 +99,35 @@ func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error
 	if req.Image == "" {
 		return fmt.Errorf("request image is empty")
 	}
-	imageHost := hostFromImage(req.Image)
+	imageHost := ecrauth.HostFromImage(req.Image)

-	creds, err := fetchIMDSCredentials(httpClient, cfg.IMDSBase)
+	creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
 	if err != nil {
 		return fmt.Errorf("fetch IMDS credentials: %w", err)
 	}

-	username, password, proxyHost, expiresAt, err := getAuthToken(cfg, httpClient, creds)
+	tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
+		creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
 	if err != nil {
 		return fmt.Errorf("get authorization token: %w", err)
 	}

-	auth := AuthConfig{Username: username, Password: password}
+	auth := AuthConfig{Username: tok.Username, Password: tok.Password}
 	authMap := map[string]AuthConfig{imageHost: auth}
-	if proxyHost != "" && proxyHost != imageHost {
-		authMap[proxyHost] = auth
+	if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
+		authMap[tok.ProxyHost] = auth
 	}

 	resp := CredentialProviderResponse{
 		APIVersion:    credProviderAPIVersion,
 		Kind:          "CredentialProviderResponse",
 		CacheKeyType:  "Registry",
-		CacheDuration: cacheDurationFrom(expiresAt),
+		CacheDuration: cacheDurationFrom(tok.ExpiresAt),
 		Auth:          authMap,
 	}
 	return json.NewEncoder(out).Encode(&resp)
 }

-// imdsCreds mirrors the IMDS security-credentials JSON shape.
-type imdsCreds struct {
-	Code            string `json:"Code"`
-	AccessKeyID     string `json:"AccessKeyId"`
-	SecretAccessKey string `json:"SecretAccessKey"`
-	Token           string `json:"Token"`
-	Expiration      string `json:"Expiration"`
-}
-
-// fetchIMDSCredentials resolves the node role name then its credentials via
-// IMDSv2 (PUT token, then GET with the token header).
-func fetchIMDSCredentials(client *http.Client, base string) (imdsCreds, error) {
-	base = strings.TrimRight(base, "/")
-	token := imdsV2Token(client, base)
-
-	roleName, err := imdsGet(client, base+"/meta-data/iam/security-credentials/", token)
-	if err != nil {
-		return imdsCreds{}, fmt.Errorf("fetch role name: %w", err)
-	}
-	roleName = strings.TrimSpace(roleName)
-	if roleName == "" {
-		return imdsCreds{}, fmt.Errorf("IMDS returned empty role name")
-	}
-
-	body, err := imdsGet(client, base+"/meta-data/iam/security-credentials/"+roleName, token)
-	if err != nil {
-		return imdsCreds{}, fmt.Errorf("fetch role credentials: %w", err)
-	}
-	var creds imdsCreds
-	if err := json.Unmarshal([]byte(body), &creds); err != nil {
-		return imdsCreds{}, fmt.Errorf("decode role credentials: %w", err)
-	}
-	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
-		return imdsCreds{}, fmt.Errorf("IMDS credentials missing access/secret key")
-	}
-	return creds, nil
-}
-
-// imdsV2Token requests an IMDSv2 session token; returns "" if the service does
-// not enforce v2 (tokenless v1 then applies).
-func imdsV2Token(client *http.Client, base string) string {
-	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
-	if err != nil {
-		return ""
-	}
-	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", imdsTokenTTL)
-	resp, err := client.Do(req)
-	if err != nil {
-		return ""
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return ""
-	}
-	body, _ := io.ReadAll(resp.Body)
-	return strings.TrimSpace(string(body))
-}
-
-func imdsGet(client *http.Client, url, token string) (string, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	body, _ := io.ReadAll(resp.Body)
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return "", fmt.Errorf("IMDS %s returned %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body)))
-	}
-	return string(body), nil
-}
-
-// getAuthToken builds an aws-sdk ECR client pointed at the gateway and calls
-// GetAuthorizationToken, decoding the AWS:<jwt> token into Basic-auth parts. It
-// returns the username, password, the ProxyEndpoint host, and the token expiry.
-func getAuthToken(cfg config, httpClient *http.Client, creds imdsCreds) (username, password, proxyHost string, expiresAt time.Time, err error) {
-	region := cfg.Region
-	if region == "" {
-		region = "us-east-1"
-	}
-	sess, serr := session.NewSession(&aws.Config{
-		Region:      aws.String(region),
-		Endpoint:    aws.String(cfg.GatewayURL),
-		Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, creds.Token),
-		HTTPClient:  httpClient,
-		DisableSSL:  aws.Bool(false),
-	})
-	if serr != nil {
-		return "", "", "", time.Time{}, fmt.Errorf("new session: %w", serr)
-	}
-
-	out, gerr := ecr.New(sess).GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
-	if gerr != nil {
-		return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken: %w", gerr)
-	}
-	if len(out.AuthorizationData) == 0 || out.AuthorizationData[0].AuthorizationToken == nil {
-		return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken returned no authorization data")
-	}
-	data := out.AuthorizationData[0]
-
-	raw, derr := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken))
-	if derr != nil {
-		return "", "", "", time.Time{}, fmt.Errorf("decode authorization token: %w", derr)
-	}
-	user, pass, ok := strings.Cut(string(raw), ":")
-	if !ok {
-		return "", "", "", time.Time{}, fmt.Errorf("authorization token not in user:password form")
-	}
-
-	if data.ExpiresAt != nil {
-		expiresAt = aws.TimeValue(data.ExpiresAt)
-	}
-	return user, pass, hostFromImage(aws.StringValue(data.ProxyEndpoint)), expiresAt, nil
-}
-
 // cacheDurationFrom derives a Go-duration string from the token expiry minus a
 // safety margin, falling back to the default when expiry is unknown or too near.
 func cacheDurationFrom(expiresAt time.Time) string {
@@ -268,30 +141,6 @@ func cacheDurationFrom(expiresAt time.Time) string {
 	return d.Round(time.Second).String()
 }

-// gatewayHTTPClient builds an HTTP client trusting the gateway CA at caPath.
-// When caPath is empty it relies on the system trust store.
-func gatewayHTTPClient(caPath string) (*http.Client, error) {
-	tlsCfg := &tls.Config{
-		MinVersion:       tls.VersionTLS13,
-		CurvePreferences: tlsconfig.Curves,
-	}
-	if caPath != "" {
-		pem, err := os.ReadFile(caPath)
-		if err != nil {
-			return nil, fmt.Errorf("read gateway CA %q: %w", caPath, err)
-		}
-		pool := x509.NewCertPool()
-		if !pool.AppendCertsFromPEM(pem) {
-			return nil, fmt.Errorf("gateway CA %q has no usable certificates", caPath)
-		}
-		tlsCfg.RootCAs = pool
-	}
-	return &http.Client{
-		Timeout:   15 * time.Second,
-		Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
-	}, nil
-}
-
 // loadConfig reads the cloud-init env file then lets real env vars override. The
 // IMDS base is fixed (not in the env file) but overridable for tests.
 func loadConfig(envFile string) config {
@@ -341,18 +190,6 @@ func parseEnvFile(path string) map[string]string {
 	return out
 }

-// hostFromImage returns the host[:port] portion of an image ref or URL. It strips
-// any scheme and trims at the first path separator.
-func hostFromImage(image string) string {
-	if i := strings.Index(image, "://"); i >= 0 {
-		image = image[i+3:]
-	}
-	if i := strings.IndexByte(image, '/'); i >= 0 {
-		image = image[:i]
-	}
-	return image
-}
-
 // emitEmpty writes a valid CredentialProviderResponse with no auth entries.
 func emitEmpty(out io.Writer) {
 	resp := CredentialProviderResponse{
@@ -122,17 +122,3 @@ func TestRun_ReturnsBasicAuth(t *testing.T) {
 		t.Errorf("password = %q, want testjwt", auth.Password)
 	}
 }
-
-func TestHostFromImage(t *testing.T) {
-	cases := map[string]string{
-		"acct.dkr.ecr.us-east-1.spinifex.internal/demo:latest": "acct.dkr.ecr.us-east-1.spinifex.internal",
-		"https://acct.dkr.ecr.us-east-1.spinifex.internal":     "acct.dkr.ecr.us-east-1.spinifex.internal",
-		"host:9999/repo:tag": "host:9999",
-		"plainhost":          "plainhost",
-	}
-	for in, want := range cases {
-		if got := hostFromImage(in); got != want {
-			t.Errorf("hostFromImage(%q) = %q, want %q", in, got, want)
-		}
-	}
-}
@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	goruntime "runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// version is the agent build version, reported in the register message.
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
+// container runtime, and an ECR resolver. It registers the host as a container
+// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
+// task assignments, and runs them through containerd, reporting state back over
+// the gateway. It never connects to NATS — the bus stays host-internal.
+type Agent struct {
+	cfg      config
+	id       identity
+	cp       controlPlane
+	puller   ctrruntime.ImagePuller
+	runner   ctrruntime.Runner
+	resolver ctrruntime.Resolver
+
+	reg *registrar
+	hb  *heartbeater
+
+	// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
+	netns *taskNetns
+
+	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
+	cred *credEndpoint
+
+	// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
+	// Always non-nil (newAgent seeds an empty one); New() reseeds it from
+	// nvidia-smi discovery.
+	gpu *gpuLedger
+
+	closers []func() error
+}
+
+// newAgent assembles an Agent from already-built seams. Tests use this directly
+// with fakes; New builds the production seams and delegates here. runner may be
+// nil when containerd is unavailable; the assign path then reports the task
+// STOPPED rather than crashing.
+func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
+	return &Agent{
+		cfg:      cfg,
+		id:       id,
+		cp:       cp,
+		puller:   puller,
+		runner:   runner,
+		resolver: resolver,
+		reg:      newRegistrar(cp, id),
+		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		netns:    newTaskNetns(execNetRunner{}),
+		gpu:      newGPULedger(id.Capacity.GPUIDs),
+	}
+}
+
+// New builds an Agent from config: it resolves the host identity from IMDS,
+// builds the gateway control-plane client, the ECR resolver and (best-effort)
+// the containerd runtime. A containerd connect failure is logged, not fatal —
+// registration and heartbeat still run so the instance is visible while the
+// runtime recovers. The ECR gateway client is built lazily on first image pull
+// (not here), so a missing or malformed gateway CA does not stop registration.
+func New(cfg config) (*Agent, error) {
+	imdsClient := &http.Client{Timeout: 5 * time.Second}
+
+	meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
+	if err != nil {
+		return nil, fmt.Errorf("instance metadata: %w", err)
+	}
+	host, _ := os.Hostname()
+	id := identity{
+		AccountID:    meta.AccountID,
+		ClusterName:  cfg.ClusterName,
+		InstanceID:   meta.InstanceID,
+		AZ:           meta.AZ,
+		Hostname:     host,
+		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
+		AgentVersion: version,
+	}
+
+	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
+
+	var puller ctrruntime.ImagePuller
+	var runner ctrruntime.Runner
+	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
+	} else {
+		puller = rt
+		runner = rt
+	}
+
+	cp, err := newGatewayControlPlane(cfg, creds)
+	if err != nil {
+		return nil, fmt.Errorf("build gateway control-plane: %w", err)
+	}
+
+	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
+		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
+	if puller != nil {
+		a.closers = append(a.closers, puller.Close)
+	}
+	return a, nil
+}
+
+// Run registers the instance, starts the heartbeat and assignment-poll loops,
+// and blocks until ctx is cancelled, then tears down.
+func (a *Agent) Run(ctx context.Context) error {
+	if err := a.reg.Register(); err != nil {
+		slog.Error("ecs-agent: initial registration failed", "err", err)
+	} else {
+		slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
+	}
+
+	if a.cred != nil {
+		if err := a.cred.Start(); err != nil {
+			slog.Error("ecs-agent: credential endpoint start failed", "err", err)
+		}
+	}
+
+	go a.hb.Run(ctx)
+
+	// Re-adopt containers still running from before this restart, then poll with
+	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
+	go a.pollAssignments(ctx, a.reconcile(ctx))
+
+	<-ctx.Done()
+	return a.Stop()
+}
+
+// Stop closes the runtime. Safe to call once.
+func (a *Agent) Stop() error {
+	var firstErr error
+	if a.cred != nil {
+		_ = a.cred.Stop()
+	}
+	for _, c := range a.closers {
+		if err := c(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
+// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
+// non-GPU host).
+func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
+	return bus.InstanceCapacity{
+		CPU:       goruntime.NumCPU() * 1024,
+		MemoryMiB: memTotalMiB(),
+		GPU:       len(gpuUUIDs),
+		GPUIDs:    gpuUUIDs,
+	}
+}
+
+// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
+func memTotalMiB() int {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			return 0
+		}
+		kb, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return 0
+		}
+		return kb / 1024
+	}
+	return 0
+}
@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
+	puller := &ctrruntime.FakePuller{}
+	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a.closers = append(a.closers, puller.Close)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	// Give it time to register + re-register (heartbeat) a few times.
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("Run returned error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+
+	// One boot register + at least one heartbeat re-register.
+	if cp.registerCount() < 2 {
+		t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
+	}
+	if !puller.Closed {
+		t.Error("Stop did not close the puller")
+	}
+}
+
+func TestDetectCapacity_Positive(t *testing.T) {
+	c := detectCapacity(nil)
+	if c.CPU <= 0 {
+		t.Errorf("CPU = %d, want > 0", c.CPU)
+	}
+	// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
+	if c.MemoryMiB < 0 {
+		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
+	}
+	if c.GPU != 0 || c.GPUIDs != nil {
+		t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
+	}
+}
+
+func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
+	c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
+	if c.GPU != 2 {
+		t.Errorf("GPU = %d, want 2", c.GPU)
+	}
+	if len(c.GPUIDs) != 2 {
+		t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
+	}
+}
@@ -0,0 +1,119 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	defaultEnvFile          = "/etc/spinifex-ecs/agent.env"
+	defaultGatewayCA        = "/etc/spinifex-ecs/gateway-ca.pem"
+	defaultIMDSBase         = "http://169.254.169.254/latest"
+	defaultContainerdSocket = "/run/containerd/containerd.sock"
+	defaultHeartbeat        = 30 * time.Second
+	defaultPollInterval     = 5 * time.Second
+)
+
+// config holds the static settings the agent reads at boot. Cluster identity
+// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
+// only the cluster *name* the instance was launched into is static. AccessKey /
+// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
+// gateway calls with instance-role credentials from IMDS; the agent never holds
+// a NATS token.
+type config struct {
+	GatewayURL       string
+	GatewayCA        string
+	Region           string
+	ClusterName      string
+	AccessKey        string
+	SecretKey        string
+	IMDSBase         string
+	ContainerdSocket string
+	Heartbeat        time.Duration
+	PollInterval     time.Duration
+	CredEndpointIP   string
+	CredEndpointPort int
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL:       get("ECS_GATEWAY_URL"),
+		GatewayCA:        get("ECS_GATEWAY_CA"),
+		Region:           get("ECS_REGION"),
+		ClusterName:      get("ECS_CLUSTER"),
+		AccessKey:        get("ECS_ACCESS_KEY"),
+		SecretKey:        get("ECS_SECRET_KEY"),
+		IMDSBase:         get("ECS_IMDS_BASE"),
+		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	if cfg.ContainerdSocket == "" {
+		cfg.ContainerdSocket = defaultContainerdSocket
+	}
+	if cfg.ClusterName == "" {
+		cfg.ClusterName = "default"
+	}
+	cfg.Heartbeat = defaultHeartbeat
+	if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.Heartbeat = d
+		}
+	}
+	cfg.PollInterval = defaultPollInterval
+	if v := get("ECS_POLL_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollInterval = d
+		}
+	}
+	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
+	if cfg.CredEndpointIP == "" {
+		cfg.CredEndpointIP = defaultCredEndpointIP
+	}
+	cfg.CredEndpointPort = defaultCredEndpointPort
+	if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.CredEndpointPort = p
+		}
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
@@ -0,0 +1,88 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	// Ensure env vars don't leak in from the host.
+	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
+		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
+		t.Setenv(k, "")
+	}
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.GatewayCA != defaultGatewayCA {
+		t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
+	}
+	if cfg.IMDSBase != defaultIMDSBase {
+		t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
+	}
+	if cfg.ContainerdSocket != defaultContainerdSocket {
+		t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
+	}
+	if cfg.ClusterName != "default" {
+		t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
+	}
+	if cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
+	}
+	if cfg.PollInterval != defaultPollInterval {
+		t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
+	}
+}
+
+func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
+	dir := t.TempDir()
+	envFile := filepath.Join(dir, "agent.env")
+	body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
+	if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
+	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
+
+	cfg := loadConfig(envFile)
+	if cfg.GatewayURL != "https://gw.env" {
+		t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
+	}
+	if cfg.Region != "us-west-2" {
+		t.Errorf("Region = %q, want from file", cfg.Region)
+	}
+	if cfg.ClusterName != "prod" {
+		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
+	}
+	if cfg.AccessKey != "AKIAENV" {
+		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
+	}
+}
+
+func TestLoadConfig_HeartbeatOverride(t *testing.T) {
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
+		t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
+	}
+	// Garbage value falls back to the default, not zero.
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
+	}
+}
+
+func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "f.env")
+	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	m := parseEnvFile(p)
+	if m["A"] != "1" || m["B"] != "2" {
+		t.Errorf("parse mismatch: %#v", m)
+	}
+	if _, ok := m["nokeyval"]; ok {
+		t.Errorf("line without = should be skipped")
+	}
+}
@@ -0,0 +1,155 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecsgw"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
+// NATS publisher: every method is a SigV4-signed gateway request, so the NATS bus
+// stays host-internal. Tests inject a fake; production uses gatewayControlPlane.
+type controlPlane interface {
+	// Register (re-)registers the container instance. Called at boot and on every
+	// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
+	// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
+	Register(id identity) error
+	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
+	SubmitTaskState(st bus.TaskState) error
+	// PollAssignments drains this instance's assignment + stop inboxes, acking the
+	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
+	// still-pending assignments and stop directives.
+	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+	// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
+	// containers, so the control plane can populate DescribeTasks gpuIds.
+	ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
+}
+
+// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
+type gatewayControlPlane struct {
+	client *ecsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// pollTimeout bounds a single PollAssignments request; larger than the register
+// timeout to leave room for a future server-side long-poll.
+const pollTimeout = 30 * time.Second
+
+// newGatewayControlPlane builds the SigV4 client signing with the instance-role
+// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
+func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
+// per-call credentials hook.
+func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
+	return func(ctx context.Context) (string, string, string, error) {
+		c, err := p.Retrieve(ctx)
+		if err != nil {
+			return "", "", "", err
+		}
+		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
+	}
+}
+
+func (g *gatewayControlPlane) Register(id identity) error {
+	in := &ecs.RegisterContainerInstanceInput{
+		Cluster:                  aws.String(id.ClusterName),
+		InstanceIdentityDocument: aws.String(id.InstanceID),
+		TotalResources: []*ecs.Resource{
+			{Name: aws.String("CPU"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.CPU))},
+			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
+		},
+		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+	}
+	if len(id.Capacity.GPUIDs) > 0 {
+		in.TotalResources = append(in.TotalResources, &ecs.Resource{
+			Name: aws.String("GPU"), Type: aws.String("STRINGSET"), StringSetValue: aws.StringSlice(id.Capacity.GPUIDs),
+		})
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal register: %w", err)
+	}
+	if _, err := g.client.Call("RegisterContainerInstance", body); err != nil {
+		return fmt.Errorf("register: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
+	in := &ecs.SubmitTaskStateChangeInput{
+		Cluster: aws.String(st.ClusterName),
+		Task:    aws.String(st.TaskID),
+		Status:  aws.String(st.LastStatus),
+		Reason:  aws.String(st.Reason),
+	}
+	for _, c := range st.Containers {
+		csc := &ecs.ContainerStateChange{
+			ContainerName: aws.String(c.Name),
+			Status:        aws.String(c.Status),
+			RuntimeId:     aws.String(c.ContainerID),
+		}
+		if c.ExitCode != nil {
+			csc.ExitCode = aws.Int64(int64(*c.ExitCode))
+		}
+		in.Containers = append(in.Containers, csc)
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal task-state: %w", err)
+	}
+	if _, err := g.client.Call("SubmitTaskStateChange", body); err != nil {
+		return fmt.Errorf("submit task-state: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	in := &handlers_ecs.PollAssignmentsInput{
+		Cluster:           cluster,
+		ContainerInstance: instance,
+		AckTaskIDs:        ackAssigns,
+		AckStopIDs:        ackStops,
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal poll: %w", err)
+	}
+	resp, err := g.client.Call("PollAssignments", body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("poll: %w", err)
+	}
+	var out handlers_ecs.PollAssignmentsOutput
+	if err := json.Unmarshal(resp, &out); err != nil {
+		return nil, nil, fmt.Errorf("decode poll: %w", err)
+	}
+	return out.Assignments, out.Stops, nil
+}
+
+func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal report-task-gpu: %w", err)
+	}
+	if _, err := g.client.Call("ReportTaskGPU", body); err != nil {
+		return fmt.Errorf("report task gpu: %w", err)
+	}
+	return nil
+}
@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+)
+
+// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// that only need a valid SigV4 signature, not real IMDS rotation.
+type fixedCredsProvider struct{}
+
+func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
+	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+}
+
+func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
+	t.Helper()
+	cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
+	if err != nil {
+		t.Fatalf("newGatewayControlPlane: %v", err)
+	}
+	return cp
+}
+
+// Register carries the discovered GPU UUIDs as a STRINGSET resource, matching
+// the control plane's RegisterContainerInstance GPU parsing (Epic C3).
+func TestGatewayControlPlane_Register_SendsGPUStringSet(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{
+		ClusterName: "default", InstanceID: "i-1",
+		Capacity: detectCapacity([]string{"GPU-aaa", "GPU-bbb"}),
+	}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	var gpu *ecs.Resource
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			gpu = r
+		}
+	}
+	if gpu == nil {
+		t.Fatal("no GPU resource in TotalResources")
+	}
+	if *gpu.Type != "STRINGSET" {
+		t.Errorf("GPU resource type = %q, want STRINGSET", *gpu.Type)
+	}
+	got := make([]string, len(gpu.StringSetValue))
+	for i, v := range gpu.StringSetValue {
+		got[i] = *v
+	}
+	if len(got) != 2 || got[0] != "GPU-aaa" || got[1] != "GPU-bbb" {
+		t.Errorf("GPU UUIDs = %v, want [GPU-aaa GPU-bbb]", got)
+	}
+}
+
+// A non-GPU host (no discovered UUIDs) registers with no GPU resource at all.
+func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{ClusterName: "default", InstanceID: "i-1", Capacity: detectCapacity(nil)}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			t.Fatalf("unexpected GPU resource on a non-GPU host: %+v", r)
+		}
+	}
+}
+
+// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
+// action, decodable as the handlers_ecs internal shape.
+func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
+	var gotTarget string
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTarget = r.Header.Get("X-Amz-Target")
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
+		{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
+	})
+	if err != nil {
+		t.Fatalf("ReportTaskGPU: %v", err)
+	}
+	if gotTarget == "" {
+		t.Fatal("no X-Amz-Target header sent")
+	}
+
+	var in handlers_ecs.ReportTaskGPUInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if in.Cluster != "web" || in.Task != "t-1" {
+		t.Errorf("cluster/task = %q/%q, want web/t-1", in.Cluster, in.Task)
+	}
+	if len(in.Containers) != 1 || in.Containers[0].Name != "trainer" || len(in.Containers[0].GPUIDs) != 1 {
+		t.Errorf("containers = %+v, want one trainer entry with one UUID", in.Containers)
+	}
+}
@@ -0,0 +1,400 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+const (
+	// defaultCredEndpointIP is the AWS-reserved link-local address tasks reach for
+	// container credentials; defaultCredEndpointPort is the standard HTTP port.
+	defaultCredEndpointIP   = "169.254.170.2"
+	defaultCredEndpointPort = 80
+	// defaultCredProxyPort is the high port the listener actually binds; a nat
+	// DNAT rule rewrites 169.254.170.2:80 to it so nothing holds a socket on :80
+	// and host/bridge tasks sharing the netns can bind :80 themselves.
+	defaultCredProxyPort = 51679
+	// credRefreshMargin refreshes a cached credential set this long before expiry
+	// so a container never starts a request with a token about to lapse.
+	credRefreshMargin = 5 * time.Minute
+	// credDummyIface is the dummy interface the endpoint IP is added to in the host
+	// netns so the agent can bind it.
+	credDummyIface = "ecs-cred"
+)
+
+// credResponse is the ECS task-credentials JSON the container's AWS SDK expects
+// at AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.
+type credResponse struct {
+	RoleArn         string `json:"RoleArn"`
+	AccessKeyID     string `json:"AccessKeyId"`
+	SecretAccessKey string `json:"SecretAccessKey"`
+	Token           string `json:"Token"`
+	Expiration      string `json:"Expiration"`
+}
+
+// credEntry caches one task's assumed-role credentials alongside the role it was
+// assumed from.
+type credEntry struct {
+	roleARN string
+	creds   stsauth.Credentials
+}
+
+// credEndpoint is the host-netns HTTP server that hands a task its IAM role
+// credentials. It maps credID -> taskRoleARN (registered as tasks start) and
+// mints/caches credentials on demand by assuming the role over the gateway with
+// the agent's instance credentials.
+type credEndpoint struct {
+	provider   credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+	ip         string
+	port       int
+	proxyPort  int
+	run        netCmdRunner
+
+	// assume mints fresh credentials for a role; defaults to AssumeRole over the
+	// gateway and is overridable in tests.
+	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+
+	mu         sync.Mutex
+	roles      map[string]string    // credID -> taskRoleARN
+	cache      map[string]credEntry // credID -> cached creds
+	httpClient *http.Client
+
+	srv *http.Server
+	ln  net.Listener
+}
+
+// newCredEndpoint builds the endpoint. An empty ip/port falls back to the
+// AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
+// adds the link-local address to the dummy interface (real or fake).
+func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+	if ip == "" {
+		ip = defaultCredEndpointIP
+	}
+	if port == 0 {
+		port = defaultCredEndpointPort
+	}
+	c := &credEndpoint{
+		provider:   provider,
+		region:     region,
+		gatewayURL: gatewayURL,
+		caPath:     caPath,
+		ip:         ip,
+		port:       port,
+		proxyPort:  defaultCredProxyPort,
+		run:        run,
+		roles:      map[string]string{},
+		cache:      map[string]credEntry{},
+	}
+	c.assume = c.assumeOverGateway
+	return c
+}
+
+// Register associates a credID with the task role it serves; Deregister removes
+// it and drops any cached credentials. Both are no-ops for an empty credID.
+func (c *credEndpoint) Register(credID, roleARN string) {
+	if credID == "" || roleARN == "" {
+		return
+	}
+	c.mu.Lock()
+	c.roles[credID] = roleARN
+	c.mu.Unlock()
+}
+
+func (c *credEndpoint) Deregister(credID string) {
+	if credID == "" {
+		return
+	}
+	c.mu.Lock()
+	delete(c.roles, credID)
+	delete(c.cache, credID)
+	c.mu.Unlock()
+}
+
+// relativeURI is the value injected into a container's
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI for the given credID.
+func credRelativeURI(credID string) string { return "/v2/credentials/" + credID }
+
+// ServeHTTP serves GET /v2/credentials/{credID}. Unknown paths/credIDs return
+// 404; non-GET returns 405; an AssumeRole failure returns 502.
+func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	credID, ok := strings.CutPrefix(r.URL.Path, "/v2/credentials/")
+	if !ok || credID == "" || strings.Contains(credID, "/") {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	creds, roleARN, err := c.fetch(r.Context(), credID)
+	if err != nil {
+		if err == errUnknownCredID {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		http.Error(w, "credentials unavailable", http.StatusBadGateway)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(credResponse{
+		RoleArn:         roleARN,
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		Token:           creds.SessionToken,
+		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+	}); err != nil {
+		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+	}
+}
+
+// errUnknownCredID signals a credID with no registered task role.
+var errUnknownCredID = fmt.Errorf("unknown credId")
+
+// fetch returns valid credentials for credID, reusing the cache until within the
+// refresh margin and otherwise assuming the task role over the gateway.
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+	c.mu.Lock()
+	roleARN, known := c.roles[credID]
+	if !known {
+		c.mu.Unlock()
+		return stsauth.Credentials{}, "", errUnknownCredID
+	}
+	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+		c.mu.Unlock()
+		return e.creds, roleARN, nil
+	}
+	c.mu.Unlock()
+
+	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	if err != nil {
+		return stsauth.Credentials{}, "", err
+	}
+
+	c.mu.Lock()
+	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.mu.Unlock()
+	return creds, roleARN, nil
+}
+
+// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
+// roleARN over the gateway — the execution-role path for ECR image pulls. It
+// caches the assumed set and re-assumes within the refresh margin. session scopes
+// the STS session name.
+func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
+}
+
+// assumeProvider adapts a credEndpoint's AssumeRole path to the agent's
+// CredentialsProvider interface, caching the assumed credentials.
+type assumeProvider struct {
+	ep      *credEndpoint
+	roleARN string
+	session string
+
+	mu     sync.Mutex
+	cached stsauth.Credentials
+}
+
+var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+
+func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !credValid(p.cached, credRefreshMargin) {
+		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
+		if err != nil {
+			return credentials.Credentials{}, err
+		}
+		p.cached = creds
+	}
+	return credentials.Credentials{
+		AccessKeyID:     p.cached.AccessKeyID,
+		SecretAccessKey: p.cached.SecretAccessKey,
+		SessionToken:    p.cached.SessionToken,
+		Expiration:      p.cached.Expiration,
+	}, nil
+}
+
+// assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
+// against the gateway with the agent's instance credentials.
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	instCreds, err := c.provider.Retrieve(ctx)
+	if err != nil {
+		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+	}
+	client, err := c.client()
+	if err != nil {
+		return stsauth.Credentials{}, err
+	}
+	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
+		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
+		roleARN, sessionName)
+}
+
+// client builds the pinned-CA gateway HTTP client on first use and caches it.
+func (c *credEndpoint) client() (*http.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.httpClient != nil {
+		return c.httpClient, nil
+	}
+	hc, err := ecrauth.GatewayHTTPClient(c.caPath)
+	if err != nil {
+		return nil, err
+	}
+	c.httpClient = hc
+	return hc, nil
+}
+
+// Start adds the endpoint IP to a dummy interface in the host netns, serves HTTP
+// on the high proxy port, and DNATs the advertised :80 to it. A bind, address,
+// or nat failure is returned so the caller can log it without aborting
+// register/heartbeat.
+func (c *credEndpoint) Start() error {
+	if err := c.addEndpointAddr(); err != nil {
+		return err
+	}
+	ln, err := net.Listen("tcp", net.JoinHostPort(c.ip, strconv.Itoa(c.listenPort())))
+	if err != nil {
+		return fmt.Errorf("listen %s:%d: %w", c.ip, c.listenPort(), err)
+	}
+	if err := c.addRedirect(); err != nil {
+		_ = ln.Close()
+		return err
+	}
+	c.ln = ln
+	c.srv = &http.Server{Handler: c, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		if err := c.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("ecs-agent: credential endpoint serve", "err", err)
+		}
+	}()
+	slog.Info("ecs-agent: credential endpoint listening", "addr", ln.Addr().String())
+	return nil
+}
+
+// Stop shuts the server down and removes the dummy interface. Safe on a nil/
+// never-started endpoint.
+func (c *credEndpoint) Stop() error {
+	if c == nil {
+		return nil
+	}
+	if c.srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = c.srv.Shutdown(ctx)
+	}
+	c.delRedirect()
+	c.delEndpointAddr()
+	return nil
+}
+
+// listenPort is the port the socket binds. Loopback test mode binds the
+// caller-supplied (ephemeral) port directly; the real endpoint binds the high
+// proxy port and reaches :80 via the DNAT rules.
+func (c *credEndpoint) listenPort() int {
+	if c.ip == "127.0.0.1" {
+		return c.port
+	}
+	return c.proxyPort
+}
+
+// addRedirect installs nat DNAT rules rewriting the advertised :80 to the proxy
+// port, for both locally generated traffic (host/bridge task containers, OUTPUT)
+// and traffic arriving over the awsvpc credential veth (PREROUTING). The dst IP
+// is left on the endpoint IP so no route_localnet handling is needed. Skipped in
+// loopback test mode, which binds the port directly.
+func (c *credEndpoint) addRedirect() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		if _, err := c.run.Run("iptables", c.natArgs("-C", chain)...); err == nil {
+			continue // rule already present
+		}
+		if _, err := c.run.Run("iptables", c.natArgs("-A", chain)...); err != nil {
+			return fmt.Errorf("add nat redirect %s: %w", chain, err)
+		}
+	}
+	return nil
+}
+
+func (c *credEndpoint) delRedirect() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		_, _ = c.run.Run("iptables", c.natArgs("-D", chain)...)
+	}
+}
+
+// natArgs builds the iptables nat rule for op (-A/-C/-D) on chain: DNAT the
+// advertised endpoint :80 to the proxy port on the same IP.
+func (c *credEndpoint) natArgs(op, chain string) []string {
+	return []string{
+		"-t", "nat", op, chain,
+		"-d", c.ip, "-p", "tcp", "--dport", strconv.Itoa(c.port),
+		"-j", "DNAT", "--to-destination", net.JoinHostPort(c.ip, strconv.Itoa(c.proxyPort)),
+	}
+}
+
+// addEndpointAddr brings up the dummy interface holding the endpoint IP. Skipped
+// for loopback (tests bind 127.0.0.1 directly, no interface plumbing needed).
+func (c *credEndpoint) addEndpointAddr() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	_, _ = c.run.Run("ip", "link", "add", credDummyIface, "type", "dummy")
+	if _, err := c.run.Run("ip", "addr", "add", c.ip+"/32", "dev", credDummyIface); err != nil &&
+		!strings.Contains(err.Error(), "exists") {
+		return fmt.Errorf("add endpoint addr %s: %w", c.ip, err)
+	}
+	if _, err := c.run.Run("ip", "link", "set", credDummyIface, "up"); err != nil {
+		return fmt.Errorf("bring up %s: %w", credDummyIface, err)
+	}
+	return nil
+}
+
+func (c *credEndpoint) delEndpointAddr() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	_, _ = c.run.Run("ip", "link", "del", credDummyIface)
+}
+
+// credValid reports whether creds are usable now with margin to spare.
+func credValid(c stsauth.Credentials, margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// STS [\w+=,.@-]{2,64} constraint.
+func sessionName(credID string) string {
+	name := "ecs-" + credID
+	if len(name) > 64 {
+		name = name[:64]
+	}
+	return name
+}
@@ -0,0 +1,197 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+// newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
+// counting calls so cache behaviour is observable. run is nil (no iface plumbing).
+func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+	var calls int32
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		atomic.AddInt32(&calls, 1)
+		return creds, nil
+	}
+	return c, &calls
+}
+
+func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
+	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	})
+	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String())
+	}
+	var got credResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || got.SecretAccessKey != "secret" || got.Token != "tok" {
+		t.Errorf("bad creds in response: %+v", got)
+	}
+	if got.RoleArn != "arn:aws:iam::111122223333:role/task" {
+		t.Errorf("RoleArn = %q", got.RoleArn)
+	}
+	if got.Expiration != exp.Format(time.RFC3339) {
+		t.Errorf("Expiration = %q, want %q", got.Expiration, exp.Format(time.RFC3339))
+	}
+}
+
+func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404", rr.Code)
+	}
+	if *calls != 0 {
+		t.Errorf("assume called %d times for unknown credID, want 0", *calls)
+	}
+}
+
+func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("status = %d, want 405", rr.Code)
+	}
+}
+
+func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	for hit := range 3 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+		if rr.Code != http.StatusOK {
+			t.Fatalf("hit %d: status %d", hit, rr.Code)
+		}
+	}
+	if *calls != 1 {
+		t.Errorf("assume called %d times, want 1 (cached)", *calls)
+	}
+}
+
+func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
+	// Expiry inside the refresh margin -> every hit re-assumes.
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	})
+	c.Register("cred-1", "arn:role")
+	for range 2 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	}
+	if *calls != 2 {
+		t.Errorf("assume called %d times, want 2 (expiry within margin)", *calls)
+	}
+}
+
+func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	c.Deregister("cred-1")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404 after deregister", rr.Code)
+	}
+}
+
+func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		return stsauth.Credentials{}, context.DeadlineExceeded
+	}
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502", rr.Code)
+	}
+}
+
+func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	c.port = 0 // bind an ephemeral port instead of privileged 80
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	url := "http://" + c.ln.Addr().String() + credRelativeURI("cred-1")
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+}
+
+func TestCredEndpoint_RedirectInstallsAndRemovesDNAT(t *testing.T) {
+	// failOn the -C existence check so addRedirect falls through to -A.
+	f := &fakeNetRunner{failOn: "-t nat -C"}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "169.254.170.2", 80, f)
+	if c.listenPort() != defaultCredProxyPort {
+		t.Fatalf("listenPort = %d, want %d (proxy)", c.listenPort(), defaultCredProxyPort)
+	}
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	target := "-d 169.254.170.2 -p tcp --dport 80 -j DNAT --to-destination 169.254.170.2:51679"
+	for _, chain := range []string{"-A OUTPUT", "-A PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing add rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+	c.delRedirect()
+	for _, chain := range []string{"-D OUTPUT", "-D PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing delete rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+}
+
+func TestCredEndpoint_RedirectSkippedOnLoopback(t *testing.T) {
+	f := &fakeNetRunner{}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, f)
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	c.delRedirect()
+	if f.sawAny("iptables") {
+		t.Errorf("loopback mode must not touch iptables; calls=%v", f.joined())
+	}
+	if c.listenPort() != c.port {
+		t.Errorf("loopback listenPort = %d, want caller port %d", c.listenPort(), c.port)
+	}
+}
+
+func TestCredRelativeURI(t *testing.T) {
+	if got := credRelativeURI("abc"); got != "/v2/credentials/abc" {
+		t.Errorf("credRelativeURI = %q", got)
+	}
+}
@@ -0,0 +1,59 @@
+package credentials
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+)
+
+// refreshMargin is how long before expiry the cache is considered stale.
+const refreshMargin = 5 * time.Minute
+
+// IMDSProvider fetches instance-role credentials from IMDS and caches them until
+// they near expiry. Safe for concurrent use.
+type IMDSProvider struct {
+	client *http.Client
+	base   string
+
+	mu     sync.Mutex
+	cached Credentials
+}
+
+var _ CredentialsProvider = (*IMDSProvider)(nil)
+
+// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
+// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	return &IMDSProvider{client: client, base: base}
+}
+
+// Retrieve returns cached credentials when still valid, else fetches fresh ones.
+func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.cached.Valid(refreshMargin) {
+		return p.cached, nil
+	}
+	if err := ctx.Err(); err != nil {
+		return Credentials{}, err
+	}
+
+	raw, err := imdscreds.Fetch(p.client, p.base)
+	if err != nil {
+		return Credentials{}, err
+	}
+	p.cached = Credentials{
+		AccessKeyID:     raw.AccessKeyID,
+		SecretAccessKey: raw.SecretAccessKey,
+		SessionToken:    raw.SessionToken,
+		Expiration:      raw.Expiration,
+	}
+	return p.cached, nil
+}
@@ -0,0 +1,104 @@
+package credentials
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits != nil {
+			atomic.AddInt32(hits, 1)
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestRetrieve_FetchesAndCaches(t *testing.T) {
+	var hits int32
+	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		"Expiration":      exp.Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	got, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("Retrieve: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
+		t.Errorf("creds mismatch: %+v", got)
+	}
+
+	// Second call within validity window must not re-hit IMDS.
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatalf("Retrieve 2: %v", err)
+	}
+	if hits != 1 {
+		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
+	}
+}
+
+func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+	var hits int32
+	// Expiry inside the refresh margin → never cached, always refetched.
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+	})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if hits != 2 {
+		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+	}
+}
+
+func TestRetrieve_CancelledContext(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := p.Retrieve(ctx); err == nil {
+		t.Fatal("expected error from cancelled context")
+	}
+}
+
+func TestCredentials_Valid(t *testing.T) {
+	if (Credentials{}).Valid(0) {
+		t.Error("empty creds should be invalid")
+	}
+	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
+	if !noExp.Valid(time.Hour) {
+		t.Error("zero expiry should be treated valid")
+	}
+	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
+	if soon.Valid(5 * time.Minute) {
+		t.Error("expiry inside margin should be invalid")
+	}
+}
@@ -0,0 +1,36 @@
+// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
+// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
+// with the host's instance-role credentials, fetched from IMDS and cached until
+// near expiry.
+package credentials
+
+import (
+	"context"
+	"time"
+)
+
+// Credentials is a SigV4 credential set with an expiry.
+type Credentials struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	SessionToken    string
+	Expiration      time.Time
+}
+
+// Valid reports whether the credentials are usable now, leaving a margin so a
+// caller does not start a request with a token about to expire mid-flight.
+func (c Credentials) Valid(margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
+// must be safe for concurrent use.
+type CredentialsProvider interface {
+	Retrieve(ctx context.Context) (Credentials, error)
+}
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func TestTaskCredID(t *testing.T) {
+	cases := []struct {
+		name string
+		as   bus.Assign
+		want string
+	}{
+		{"no role", bus.Assign{TaskID: "t1"}, ""},
+		{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
+	}
+	for _, tc := range cases {
+		if got := taskCredID(&tc.as); got != tc.want {
+			t.Errorf("%s: taskCredID = %q, want %q", tc.name, got, tc.want)
+		}
+	}
+}
+
+func TestWithCredEnv(t *testing.T) {
+	base := map[string]string{"FOO": "bar"}
+
+	// No credID -> env returned unchanged (same map).
+	if got := withCredEnv(base, ""); got["FOO"] != "bar" || got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "" {
+		t.Errorf("blank credID mutated env: %v", got)
+	}
+
+	got := withCredEnv(base, "cred-1")
+	if got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "/v2/credentials/cred-1" {
+		t.Errorf("missing relative-uri: %v", got)
+	}
+	if got["FOO"] != "bar" {
+		t.Errorf("dropped existing env: %v", got)
+	}
+	// Original map must be untouched (copy semantics).
+	if _, ok := base["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]; ok {
+		t.Errorf("source env mutated: %v", base)
+	}
+}
+
+func TestSessionName(t *testing.T) {
+	if got := sessionName("abc"); got != "ecs-abc" {
+		t.Errorf("sessionName = %q", got)
+	}
+	long := sessionName(string(make([]byte, 100)))
+	if len(long) != 64 {
+		t.Errorf("sessionName not truncated to 64: len=%d", len(long))
+	}
+}
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+)
+
+// ecrResolver implements runtime.Resolver by minting an ECR authorization token
+// per pull: it fetches the host's IMDS credentials, SigV4-signs
+// GetAuthorizationToken against the gateway, and returns the AWS:<jwt> as a
+// user/password pair for containerd.
+//
+// The gateway HTTP client is built lazily on first Authorize, not at agent boot:
+// a missing or malformed gateway CA only matters when an image is actually
+// pulled, so the agent can still register + heartbeat on a host with absent or
+// wrong ECR config.
+type ecrResolver struct {
+	creds      credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+
+	mu         sync.Mutex
+	httpClient *http.Client
+}
+
+var _ runtime.Resolver = (*ecrResolver)(nil)
+
+// newECRResolver builds a resolver with an injected HTTP client (tests). The
+// client is used as-is and never rebuilt from caPath.
+func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
+}
+
+// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
+// from caPath on first use (production path).
+func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
+}
+
+// client returns the gateway HTTP client, building it from caPath on first call
+// and caching the result. An injected client (newECRResolver) is returned as-is.
+func (e *ecrResolver) client() (*http.Client, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.httpClient != nil {
+		return e.httpClient, nil
+	}
+	c, err := ecrauth.GatewayHTTPClient(e.caPath)
+	if err != nil {
+		return nil, err
+	}
+	e.httpClient = c
+	return c, nil
+}
+
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
+	host, _, _ := strings.Cut(ref, "/")
+	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
+		return false // no registry host -> docker.io default
+	}
+	return strings.Contains(host, ".dkr.ecr.")
+}
+
+// Authorize returns the ECR token's user/password and proxy endpoint for ref.
+// Non-ECR refs resolve to anonymous pull (empty credentials).
+func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
+	if !isECRRef(ref) {
+		return "", "", "", nil
+	}
+	c, err := e.creds.Retrieve(ctx)
+	if err != nil {
+		return "", "", "", err
+	}
+	httpClient, err := e.client()
+	if err != nil {
+		return "", "", "", err
+	}
+	tok, err := ecrauth.GetAuthorizationToken(e.region, e.gatewayURL, httpClient,
+		c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
+	if err != nil {
+		return "", "", "", err
+	}
+	return tok.Username, tok.Password, tok.ProxyHost, nil
+}
@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+)
+
+// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+type stubCreds struct {
+	c   credentials.Credentials
+	err error
+}
+
+func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+	return s.c, s.err
+}
+
+func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if status != http.StatusOK {
+			w.WriteHeader(status)
+			return
+		}
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
+				"proxyEndpoint":      proxy,
+			}},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func trusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestECRResolver_Authorize(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	user, pass, endpoint, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+	if endpoint != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
+		t.Errorf("endpoint = %q", endpoint)
+	}
+}
+
+func TestECRResolver_CredsError(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("no imds")}, "us-east-1", "https://x", http.DefaultClient)
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected creds error to propagate")
+	}
+}
+
+func TestECRResolver_GatewayError(t *testing.T) {
+	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected gateway error")
+	}
+}
+
+func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	// Construction with a bogus CA path must not fail — only Authorize does.
+	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected lazy gateway-client build to fail on missing CA")
+	}
+}
+
+// TestECRResolver_NonECRAnonymous asserts that a public-registry ref pulls
+// anonymously: Authorize returns empty creds without ever touching IMDS, even
+// when the credentials provider would error.
+func TestECRResolver_NonECRAnonymous(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("imds must not be called")}, "us-east-1", "https://x", http.DefaultClient)
+	for _, ref := range []string{"nginx:alpine", "docker.io/library/nginx:alpine", "ghcr.io/org/app:1"} {
+		user, pass, endpoint, err := r.Authorize(context.Background(), ref)
+		if err != nil || user != "" || pass != "" || endpoint != "" {
+			t.Errorf("ref %q: got (%q,%q,%q,%v), want anonymous", ref, user, pass, endpoint, err)
+		}
+	}
+}
+
+func TestIsECRRef(t *testing.T) {
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	}
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+		}
+	}
+}
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// commandRunner abstracts exec.Command so discovery is unit-testable without
+// shelling out. Production uses execCommandRunner; tests inject a fake.
+type commandRunner func(name string, args ...string) ([]byte, error)
+
+// execCommandRunner runs a real host command and returns its stdout.
+func execCommandRunner(name string, args ...string) ([]byte, error) {
+	return exec.Command(name, args...).Output()
+}
+
+// discoverNvidiaGPUs enumerates local NVIDIA GPU device UUIDs via nvidia-smi.
+// A missing binary or any run failure (no GPU host) yields an empty list, not
+// an error — the agent still registers with zero GPU capacity.
+func discoverNvidiaGPUs(run commandRunner) []string {
+	out, err := run("nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader")
+	if err != nil {
+		return nil
+	}
+	return parseNvidiaSMIUUIDs(string(out))
+}
+
+// parseNvidiaSMIUUIDs splits nvidia-smi's newline-delimited UUID output,
+// dropping blank lines. Exported as a pure function so fixture strings can be
+// tested without exec.
+func parseNvidiaSMIUUIDs(out string) []string {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	uuids := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		uuids = append(uuids, line)
+	}
+	return uuids
+}
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseNvidiaSMIUUIDs(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace only", "   \n\n  ", nil},
+		{"single", "GPU-11111111-1111-1111-1111-111111111111\n", []string{"GPU-11111111-1111-1111-1111-111111111111"}},
+		{"multi", "GPU-aaa\nGPU-bbb\nGPU-ccc\n", []string{"GPU-aaa", "GPU-bbb", "GPU-ccc"}},
+		{"blank lines between", "GPU-aaa\n\nGPU-bbb\n", []string{"GPU-aaa", "GPU-bbb"}},
+		{"trailing whitespace per line", "GPU-aaa \r\nGPU-bbb\r\n", []string{"GPU-aaa", "GPU-bbb"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := parseNvidiaSMIUUIDs(c.in)
+			if !reflect.DeepEqual(got, c.want) && (len(got) != 0 || len(c.want) != 0) {
+				t.Errorf("parseNvidiaSMIUUIDs(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestDiscoverNvidiaGPUs_AbsentBinaryReturnsEmptyNoError(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return nil, errors.New(`exec: "nvidia-smi": executable file not found in $PATH`)
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list on absent nvidia-smi, got %v", got)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_ParsesFixtureOutput(t *testing.T) {
+	var gotName string
+	var gotArgs []string
+	run := func(name string, args ...string) ([]byte, error) {
+		gotName = name
+		gotArgs = args
+		return []byte("GPU-aaa\nGPU-bbb\n"), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	want := []string{"GPU-aaa", "GPU-bbb"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if gotName != "nvidia-smi" {
+		t.Errorf("command = %q, want nvidia-smi", gotName)
+	}
+	if len(gotArgs) != 2 || gotArgs[0] != "--query-gpu=uuid" || gotArgs[1] != "--format=csv,noheader" {
+		t.Errorf("args = %v, want the uuid/csv,noheader query", gotArgs)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_NoDevicesReturnsEmpty(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return []byte(""), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list, got %v", got)
+	}
+}
@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// gpuLedger is the agent's local source of truth for which of the host's
+// discovered GPU UUIDs are free vs pinned to a running task's container. C4
+// reads a task's pinned UUIDs to build the CDI device list for containerd.
+type gpuLedger struct {
+	mu     sync.Mutex
+	free   []string
+	pinned map[string][]string // key ("taskID/containerName") -> uuids
+}
+
+// newGPULedger seeds the ledger with the host's discovered device UUIDs. A nil
+// or empty slice is a valid non-GPU host: every Pin then fails, which callers
+// treat as best-effort (no device, no CDI injection later).
+func newGPULedger(uuids []string) *gpuLedger {
+	free := append([]string(nil), uuids...)
+	return &gpuLedger{free: free, pinned: make(map[string][]string)}
+}
+
+// gpuKey builds the ledger key for one container's pin/release.
+func gpuKey(taskID, containerName string) string {
+	return taskID + "/" + containerName
+}
+
+// Pin reserves n free UUIDs under key, returning them. An error means fewer
+// than n devices were free; the key is left unpinned so a retry (or a
+// best-effort caller) can proceed without a partial reservation.
+func (l *gpuLedger) Pin(key string, n int) ([]string, error) {
+	if n <= 0 {
+		return nil, nil
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.free) < n {
+		return nil, fmt.Errorf("gpu ledger: requested %d GPUs, %d free", n, len(l.free))
+	}
+	uuids := append([]string(nil), l.free[:n]...)
+	l.free = l.free[n:]
+	l.pinned[key] = uuids
+	return uuids, nil
+}
+
+// ReleaseTask frees every UUID pinned under any container key belonging to
+// taskID, called once the task's terminal STOPPED state is reported.
+func (l *gpuLedger) ReleaseTask(taskID string) {
+	prefix := taskID + "/"
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for key, uuids := range l.pinned {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		l.free = append(l.free, uuids...)
+		delete(l.pinned, key)
+	}
+}
@@ -0,0 +1,155 @@
+package main
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestGPULedger_PinReturnsDistinctUUIDs(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c"})
+	got, err := l.Pin("t-1/web", 2)
+	if err != nil {
+		t.Fatalf("Pin: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 UUIDs, got %v", got)
+	}
+	if got[0] == got[1] {
+		t.Errorf("pinned UUIDs must be distinct, got %v", got)
+	}
+	if len(l.free) != 1 {
+		t.Errorf("free = %d, want 1 remaining", len(l.free))
+	}
+}
+
+func TestGPULedger_PinInsufficientFreeErrors(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	_, err := l.Pin("t-1/web", 2)
+	if err == nil {
+		t.Fatal("want error when requesting more GPUs than free")
+	}
+	if len(l.free) != 1 {
+		t.Errorf("failed pin must not consume any device, free = %d", len(l.free))
+	}
+}
+
+func TestGPULedger_PinZeroIsNoop(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	got, err := l.Pin("t-1/web", 0)
+	if err != nil || got != nil {
+		t.Errorf("Pin(0) = %v, %v; want nil, nil", got, err)
+	}
+}
+
+func TestGPULedger_ReleaseTaskFreesOnlyThatTasksDevices(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c", "GPU-d"})
+	if _, err := l.Pin(gpuKey("t-1", "web"), 2); err != nil {
+		t.Fatalf("pin t-1: %v", err)
+	}
+	if _, err := l.Pin(gpuKey("t-2", "worker"), 1); err != nil {
+		t.Fatalf("pin t-2: %v", err)
+	}
+	if len(l.free) != 1 {
+		t.Fatalf("free = %d, want 1 before release", len(l.free))
+	}
+
+	l.ReleaseTask("t-1")
+	if len(l.free) != 3 {
+		t.Errorf("free = %d, want 3 after releasing t-1's two devices", len(l.free))
+	}
+	if _, ok := l.pinned[gpuKey("t-2", "worker")]; !ok {
+		t.Error("t-2's pin should survive releasing t-1")
+	}
+	if _, ok := l.pinned[gpuKey("t-1", "web")]; ok {
+		t.Error("t-1's pin should be gone after release")
+	}
+}
+
+// Multi-device pinning across concurrent task assigns must hand out disjoint
+// UUID sets — no two tasks may ever share a device.
+func TestGPULedger_ConcurrentPinsHandOutDistinctUUIDs(t *testing.T) {
+	const nDevices = 20
+	uuids := make([]string, nDevices)
+	for i := range uuids {
+		uuids[i] = "GPU-" + string(rune('a'+i))
+	}
+	l := newGPULedger(uuids)
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	seen := make(map[string]bool)
+	errs := 0
+	for i := range nDevices {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			key := gpuKey("t-concurrent", string(rune('A'+n)))
+			got, err := l.Pin(key, 1)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs++
+				return
+			}
+			for _, u := range got {
+				if seen[u] {
+					t.Errorf("UUID %s handed out to more than one concurrent pin", u)
+				}
+				seen[u] = true
+			}
+		}(i)
+	}
+	wg.Wait()
+	if errs != 0 {
+		t.Errorf("want all %d pins to succeed (exactly enough devices), got %d errors", nDevices, errs)
+	}
+	if len(seen) != nDevices {
+		t.Errorf("distinct UUIDs handed out = %d, want %d", len(seen), nDevices)
+	}
+	if len(l.free) != 0 {
+		t.Errorf("free = %d, want 0 after all devices pinned", len(l.free))
+	}
+}
+
+// Releasing a task's devices concurrently with new pins never double-hands a
+// UUID: a device is only ever free or pinned to exactly one key at a time.
+func TestGPULedger_ConcurrentPinAndReleaseNoDoubleAssignment(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b"})
+	var wg sync.WaitGroup
+	for round := range 50 {
+		wg.Add(2)
+		taskID := "t-" + string(rune('a'+round%26))
+		go func(id string) {
+			defer wg.Done()
+			_, _ = l.Pin(gpuKey(id, "web"), 1)
+		}(taskID)
+		go func(id string) {
+			defer wg.Done()
+			l.ReleaseTask(id)
+		}(taskID)
+	}
+	wg.Wait()
+	// Invariant: every currently-pinned UUID appears in exactly one pinned entry
+	// and never simultaneously in free.
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	freeSet := make(map[string]bool, len(l.free))
+	for _, u := range l.free {
+		if freeSet[u] {
+			t.Errorf("duplicate UUID %s in free list", u)
+		}
+		freeSet[u] = true
+	}
+	pinnedSet := make(map[string]bool)
+	for _, uuids := range l.pinned {
+		for _, u := range uuids {
+			if pinnedSet[u] {
+				t.Errorf("UUID %s pinned under more than one key", u)
+			}
+			if freeSet[u] {
+				t.Errorf("UUID %s is both free and pinned", u)
+			}
+			pinnedSet[u] = true
+		}
+	}
+}
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// heartbeater keeps the container instance alive by periodically re-registering
+// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
+// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
+// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
+type heartbeater struct {
+	cp       controlPlane
+	id       identity
+	interval time.Duration
+}
+
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	return &heartbeater{cp: cp, id: id, interval: interval}
+}
+
+// beat re-registers the instance once, refreshing its LastSeen.
+func (h *heartbeater) beat() error {
+	return h.cp.Register(h.id)
+}
+
+// Run beats every interval until ctx is cancelled. A failed beat is logged and
+// retried on the next tick rather than killing the loop.
+func (h *heartbeater) Run(ctx context.Context) {
+	ticker := time.NewTicker(h.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := h.beat(); err != nil {
+				slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
+			}
+		}
+	}
+}
@@ -0,0 +1,39 @@
+// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
+// the ECS-AMI, with containerd baked in). It registers the host with the ECS
+// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
+// re-registering, polls the gateway for task assignments, runs them through
+// containerd, and reports state back over the gateway.
+//
+// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
+// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
+// comes from IMDS at boot.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	agent, err := New(cfg)
+	if err != nil {
+		slog.Error("ecs-agent: startup failed", "err", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if err := agent.Run(ctx); err != nil {
+		slog.Error("ecs-agent: run failed", "err", err)
+		os.Exit(1)
+	}
+}
@@ -0,0 +1,97 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
+type instanceMetadata struct {
+	AccountID  string
+	InstanceID string
+	AZ         string
+}
+
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
+// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
+	token, err := imdsToken(client, base)
+	if err != nil {
+		return instanceMetadata{}, err
+	}
+
+	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
+	}
+	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
+	}
+
+	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
+	}
+	var ident struct {
+		AccountID string `json:"accountId"`
+	}
+	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
+		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
+	}
+
+	return instanceMetadata{
+		AccountID:  ident.AccountID,
+		InstanceID: strings.TrimSpace(instanceID),
+		AZ:         strings.TrimSpace(az),
+	}, nil
+}
+
+// imdsToken fetches an IMDSv2 session token.
+func imdsToken(client *http.Client, base string) (string, error) {
+	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("imds token: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
+
+// imdsGet performs a token-authenticated IMDS GET.
+func imdsGet(client *http.Client, url, token string) (string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	guard := func(w http.ResponseWriter, r *http.Request) bool {
+		if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("i-0abc123"))
+		}
+	})
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("us-east-1a"))
+		}
+	})
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
+		}
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestFetchInstanceMetadata_OK(t *testing.T) {
+	srv := metadataStub(t, true)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
+		t.Errorf("metadata mismatch: %+v", got)
+	}
+}
+
+func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
+	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
+	srv := metadataStub(t, true)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+		t.Fatalf("v2 path should succeed: %v", err)
+	}
+}
+
+func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+		t.Fatal("expected parse error on bad identity document")
+	}
+}
@@ -0,0 +1,167 @@
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// netnsRunDir is the iproute2 netns mount directory; a name here is the netns
+// path containerd joins via the OCI network namespace path.
+const netnsRunDir = "/var/run/netns"
+
+// Credentials path constants: each task netns gets a /30 veth into the host netns
+// from 169.254.172.0/22 and a host-routed /32 to the credential endpoint, keeping
+// the endpoint off the task ENI's VPC route. credTaskVeth is the task-side name
+// (unique per netns); the host side is per-task (credVethName).
+const (
+	credTaskVeth     = "ecsc0"
+	credEndpointCIDR = defaultCredEndpointIP + "/32"
+)
+
+// netCmdRunner executes a host networking command. Abstracted so unit tests can
+// assert the ip/netns sequence without touching the kernel.
+type netCmdRunner interface {
+	Run(name string, args ...string) (string, error)
+}
+
+// execNetRunner runs real commands.
+type execNetRunner struct{}
+
+func (execNetRunner) Run(name string, args ...string) (string, error) {
+	out, err := exec.Command(name, args...).CombinedOutput()
+	return string(out), err
+}
+
+// taskNetns builds and tears down per-task awsvpc network namespaces. The ENI is
+// already hot-plugged into the VM (scheduler did that); the agent finds the NIC
+// by MAC, moves it into a task netns, and DHCPs the OVN-assigned ENI IP — the
+// same way the primary NIC obtains its address.
+type taskNetns struct {
+	run     netCmdRunner
+	nicWait time.Duration
+	poll    time.Duration
+}
+
+func newTaskNetns(run netCmdRunner) *taskNetns {
+	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+}
+
+func netnsName(taskID string) string { return "ecs-" + taskID }
+func netnsPathFor(taskID string) string {
+	return netnsRunDir + "/" + netnsName(taskID)
+}
+
+// Setup creates the task netns, moves the ENI NIC (matched by MAC) into it, and
+// brings it up with a DHCP lease. It returns the netns path for containerd. On
+// any failure it tears the partial netns down so a retry starts clean.
+func (n *taskNetns) Setup(taskID, mac string) (string, error) {
+	iface, err := n.waitForNIC(mac)
+	if err != nil {
+		return "", err
+	}
+	name := netnsName(taskID)
+	if _, err := n.run.Run("ip", "netns", "add", name); err != nil {
+		return "", fmt.Errorf("netns add %s: %w", name, err)
+	}
+	hostVeth := credVethName(taskID)
+	hostIP, taskIP := credSubnet(taskID)
+	steps := [][]string{
+		{"ip", "link", "set", iface, "netns", name},
+		{"ip", "-n", name, "link", "set", "lo", "up"},
+		{"ip", "-n", name, "link", "set", iface, "up"},
+		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		// Credentials path: a veth into the host netns plus a host-routed /32 to the
+		// credential endpoint, so the container reaches 169.254.170.2 without it
+		// leaking onto the task ENI's VPC route.
+		{"ip", "link", "add", hostVeth, "type", "veth", "peer", "name", credTaskVeth, "netns", name},
+		{"ip", "addr", "add", hostIP + "/30", "dev", hostVeth},
+		{"ip", "link", "set", hostVeth, "up"},
+		{"ip", "-n", name, "addr", "add", taskIP + "/30", "dev", credTaskVeth},
+		{"ip", "-n", name, "link", "set", credTaskVeth, "up"},
+		{"ip", "-n", name, "route", "add", credEndpointCIDR, "via", hostIP},
+	}
+	for _, s := range steps {
+		if _, err := n.run.Run(s[0], s[1:]...); err != nil {
+			_ = n.Teardown(taskID)
+			return "", fmt.Errorf("netns setup %v: %w", s, err)
+		}
+	}
+	return netnsPathFor(taskID), nil
+}
+
+// credVethName is the host-side veth name for a task's credential path. Derived
+// from the taskID and kept within the 15-char interface-name limit.
+func credVethName(taskID string) string {
+	return fmt.Sprintf("ecsv%08x", crc32.ChecksumIEEE([]byte(taskID)))
+}
+
+// credSubnet derives a unique /30 in 169.254.172.0/22 for a task's credential
+// veth, returning the host-side and task-side addresses. The host side is the
+// container's gateway to the credential endpoint.
+func credSubnet(taskID string) (hostIP, taskIP string) {
+	slot := crc32.ChecksumIEEE([]byte(taskID)) % 256 // 256 /30s in the /22
+	third := 172 + (slot*4)/256
+	base := (slot * 4) % 256
+	hostIP = fmt.Sprintf("169.254.%d.%d", third, base+1)
+	taskIP = fmt.Sprintf("169.254.%d.%d", third, base+2)
+	return hostIP, taskIP
+}
+
+// Teardown deletes the task netns (releasing the moved NIC). Idempotent enough
+// for the stop path: a missing netns is reported but not fatal to the caller.
+func (n *taskNetns) Teardown(taskID string) error {
+	name := netnsName(taskID)
+	// Deleting the netns destroys the task-side veth and its host peer; remove the
+	// host side explicitly too so a leaked half from a partial setup is reaped.
+	_, _ = n.run.Run("ip", "link", "del", credVethName(taskID))
+	if _, err := n.run.Run("ip", "netns", "del", name); err != nil {
+		return fmt.Errorf("netns del %s: %w", name, err)
+	}
+	return nil
+}
+
+// waitForNIC polls until a NIC with the given MAC appears in the host netns,
+// returning its interface name. The hot-plug pipeline is async, so the device
+// may lag the assign by a few seconds.
+func (n *taskNetns) waitForNIC(mac string) (string, error) {
+	want := strings.ToLower(strings.TrimSpace(mac))
+	deadline := time.Now().Add(n.nicWait)
+	for {
+		iface, err := n.findNICByMAC(want)
+		if err == nil {
+			return iface, nil
+		}
+		if time.Now().After(deadline) {
+			return "", fmt.Errorf("nic with mac %s not found within %s", want, n.nicWait)
+		}
+		time.Sleep(n.poll)
+	}
+}
+
+// findNICByMAC parses `ip -o link` and returns the interface whose link/ether
+// matches mac.
+func (n *taskNetns) findNICByMAC(mac string) (string, error) {
+	out, err := n.run.Run("ip", "-o", "link")
+	if err != nil {
+		return "", fmt.Errorf("ip -o link: %w", err)
+	}
+	for line := range strings.SplitSeq(out, "\n") {
+		if !strings.Contains(strings.ToLower(line), mac) {
+			continue
+		}
+		// "<idx>: <ifname>: <flags> ... link/ether <mac> ..."
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		name := strings.TrimSuffix(strings.TrimSuffix(fields[1], ":"), "@")
+		if i := strings.IndexByte(name, '@'); i >= 0 {
+			name = name[:i]
+		}
+		return name, nil
+	}
+	return "", fmt.Errorf("no interface with mac %s", mac)
+}
@@ -0,0 +1,154 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeNetRunner records every command and serves canned `ip -o link` output. A
+// failOn substring forces the matching command to error so teardown paths run.
+type fakeNetRunner struct {
+	mu      sync.Mutex
+	calls   [][]string
+	linkOut string
+	failOn  string
+}
+
+func (f *fakeNetRunner) Run(name string, args ...string) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	cmd := append([]string{name}, args...)
+	f.calls = append(f.calls, cmd)
+	joined := strings.Join(cmd, " ")
+	if f.failOn != "" && strings.Contains(joined, f.failOn) {
+		return "", errors.New("boom: " + joined)
+	}
+	if name == "ip" && len(args) >= 2 && args[0] == "-o" && args[1] == "link" {
+		return f.linkOut, nil
+	}
+	return "", nil
+}
+
+func (f *fakeNetRunner) joined() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]string, len(f.calls))
+	for i, c := range f.calls {
+		out[i] = strings.Join(c, " ")
+	}
+	return out
+}
+
+func (f *fakeNetRunner) sawAny(sub string) bool {
+	for _, j := range f.joined() {
+		if strings.Contains(j, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:00:00:00 ...
+2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
+3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
+
+func newTestNetns(f *fakeNetRunner) *taskNetns {
+	n := newTaskNetns(f)
+	n.nicWait = 50 * time.Millisecond
+	n.poll = 5 * time.Millisecond
+	return n
+}
+
+func TestNetns_FindNICByMAC(t *testing.T) {
+	n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+	iface, err := n.findNICByMAC("52:54:00:de:ad:01")
+	if err != nil || iface != "eth1" {
+		t.Fatalf("want eth1, got %q err=%v", iface, err)
+	}
+	if _, err := n.findNICByMAC("52:54:00:ff:ff:ff"); err == nil {
+		t.Fatal("want error for absent mac")
+	}
+}
+
+func TestNetns_SetupSequence(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	path, err := n.Setup("t-001", "52:54:00:DE:AD:01")
+	if err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if path != netnsPathFor("t-001") {
+		t.Fatalf("want %q, got %q", netnsPathFor("t-001"), path)
+	}
+	hostIP, taskIP := credSubnet("t-001")
+	hostVeth := credVethName("t-001")
+	want := []string{
+		"ip netns add ecs-t-001",
+		"ip link set eth1 netns ecs-t-001",
+		"ip -n ecs-t-001 link set lo up",
+		"ip -n ecs-t-001 link set eth1 up",
+		"ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		"ip link add " + hostVeth + " type veth peer name ecsc0 netns ecs-t-001",
+		"ip addr add " + hostIP + "/30 dev " + hostVeth,
+		"ip -n ecs-t-001 addr add " + taskIP + "/30 dev ecsc0",
+		"ip -n ecs-t-001 route add 169.254.170.2/32 via " + hostIP,
+	}
+	for _, w := range want {
+		if !f.sawAny(w) {
+			t.Errorf("missing command %q in %v", w, f.joined())
+		}
+	}
+}
+
+func TestCredSubnet_UniquePer30(t *testing.T) {
+	hostA, taskA := credSubnet("task-a")
+	hostB, taskB := credSubnet("task-b")
+	if hostA == hostB {
+		t.Errorf("distinct tasks share a host IP: %s", hostA)
+	}
+	for _, ip := range []string{hostA, taskA, hostB, taskB} {
+		if !strings.HasPrefix(ip, "169.254.17") {
+			t.Errorf("cred IP %s outside 169.254.172.0/22", ip)
+		}
+	}
+	// host and task ends of one task sit in the same /30 (differ in last octet).
+	if hostA[:strings.LastIndexByte(hostA, '.')] != taskA[:strings.LastIndexByte(taskA, '.')] {
+		t.Errorf("host %s and task %s not in same /30", hostA, taskA)
+	}
+}
+
+func TestNetns_SetupNICNotFound(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:00:00:99"); err == nil {
+		t.Fatal("want timeout error for absent NIC")
+	}
+	if f.sawAny("netns add") {
+		t.Error("should not create netns when NIC never appears")
+	}
+}
+
+func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI, failOn: "udhcpc"}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when udhcpc fails")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown after step failure, got %v", f.joined())
+	}
+}
+
+func TestNetns_Teardown(t *testing.T) {
+	f := &fakeNetRunner{}
+	n := newTestNetns(f)
+	if err := n.Teardown("t-001"); err != nil {
+		t.Fatalf("teardown: %v", err)
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected netns del, got %v", f.joined())
+	}
+}
@@ -0,0 +1,72 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// reconcile re-adopts containers that survived an agent restart. It lists the
+// runtime's containers and, for each task still running in this cluster,
+// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
+// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// gateway's re-delivered assignments for these tasks are acked but not re-run.
+//
+// It is best-effort: a nil runner or a List error returns an empty set and the
+// agent falls back to its pre-reconcile behaviour (poll + run).
+func (a *Agent) reconcile(ctx context.Context) map[string]bool {
+	adopted := map[string]bool{}
+	if a.runner == nil {
+		return adopted
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: reconcile list failed", "err", err)
+		return adopted
+	}
+
+	// Group this cluster's running containers back into their tasks. The Assign is
+	// reconstructed from labels — enough to re-register creds and tear down netns.
+	type task struct {
+		as         *bus.Assign
+		containers []ctrruntime.Container
+	}
+	tasks := map[string]*task{}
+	for _, c := range containers {
+		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+			continue
+		}
+		taskID := c.Labels[labelTaskID]
+		if taskID == "" {
+			continue
+		}
+		t := tasks[taskID]
+		if t == nil {
+			t = &task{as: &bus.Assign{
+				TaskID:        taskID,
+				ClusterName:   a.id.ClusterName,
+				CredID:        c.Labels[labelCredID],
+				TaskRoleARN:   c.Labels[labelTaskRoleARN],
+				ENIMacAddress: c.Labels[labelENIMac],
+			}}
+			tasks[taskID] = t
+		}
+		t.containers = append(t.containers, c)
+	}
+
+	for taskID, t := range tasks {
+		a.registerTaskCreds(t.as)
+		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
+		for _, c := range t.containers {
+			name := c.Labels[labelContainerName]
+			go a.waitContainer(ctx, t.as, name, c.ID)
+			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
+		}
+		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
+		adopted[taskID] = true
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+	}
+	return adopted
+}
@@ -0,0 +1,181 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+const testTaskRole = "arn:aws:iam::111122223333:role/task"
+
+// adoptedContainer builds a runtime.Container as the reconciler would discover it,
+// stamping only the mulga.ecs.* labels that are non-empty.
+func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
+	labels := map[string]string{
+		labelTaskID:        taskID,
+		labelContainerName: name,
+		labelClusterName:   cluster,
+	}
+	if credID != "" {
+		labels[labelCredID] = credID
+	}
+	if role != "" {
+		labels[labelTaskRoleARN] = role
+	}
+	if mac != "" {
+		labels[labelENIMac] = mac
+	}
+	return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
+}
+
+func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
+	t.Helper()
+	deadline := time.After(time.Second)
+	for !slices.Contains(rt.Waits(), id) {
+		select {
+		case <-deadline:
+			t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// A running, correctly-labeled container is re-adopted: its credentials are
+// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
+func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if !adopted["t-001"] {
+		t.Fatalf("t-001 not adopted; got %v", adopted)
+	}
+	if got := a.cred.roles["t-001"]; got != testTaskRole {
+		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
+	}
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
+	}
+	waitedFor(t, rt, containerID("t-001", "web"))
+}
+
+// pollAssignments seeded with an adopted task acks its re-delivered assignment
+// but does not re-run it, while a genuinely new task still dispatches.
+func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
+	mk := func(id string) bus.Assign {
+		return bus.Assign{
+			AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
+			Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
+		}
+	}
+	adopted, fresh := mk("t-001"), mk("t-002")
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{adopted, fresh},
+		{adopted, fresh},
+		nil,
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
+
+	deadline := time.After(time.Second)
+	for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
+		select {
+		case <-deadline:
+			t.Fatal("fresh task t-002 was never dispatched")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
+	}
+	var t1Acked bool
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == "t-001" {
+				t1Acked = true
+			}
+		}
+	}
+	if !t1Acked {
+		t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
+	}
+}
+
+// Containers for another cluster or with a stopped task are not adopted.
+func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-other", "web", "other", "", "", "", true),
+			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-ok", "web", "default", "", "", "", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if len(adopted) != 1 || !adopted["t-ok"] {
+		t.Fatalf("adopted = %v, want only t-ok", adopted)
+	}
+}
+
+// A nil runner or a List error degrades to an empty set without panicking.
+func TestReconcile_DegradesGracefully(t *testing.T) {
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("nil runner: adopted = %v, want empty", got)
+	}
+
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	if got := listErr.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("list error: adopted = %v, want empty", got)
+	}
+}
+
+// taskLabels emits the optional cred/role/MAC labels only when set.
+func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
+	full := taskLabels(&bus.Assign{
+		TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
+	}, "web")
+	for k, want := range map[string]string{
+		labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
+	} {
+		if full[k] != want {
+			t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
+		}
+	}
+
+	bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
+	if len(bare) != 3 {
+		t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
+	}
+	for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
+		if _, ok := bare[k]; ok {
+			t.Errorf("bare labels unexpectedly carry %q", k)
+		}
+	}
+}
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// identity is the container instance's stable identity, assembled at boot from
+// IMDS (account, instance, AZ) and config (cluster). It is the join key the
+// scheduler uses to track this instance.
+type identity struct {
+	AccountID    string
+	ClusterName  string
+	InstanceID   string
+	AZ           string
+	Hostname     string
+	Capacity     bus.InstanceCapacity
+	AgentVersion string
+}
+
+// registrar registers the host as a container instance through the gateway.
+type registrar struct {
+	cp controlPlane
+	id identity
+}
+
+func newRegistrar(cp controlPlane, id identity) *registrar {
+	return &registrar{cp: cp, id: id}
+}
+
+// Register calls the gateway's RegisterContainerInstance. The scheduler records
+// (or refreshes) the container instance on receipt.
+func (r *registrar) Register() error {
+	if err := r.cp.Register(r.id); err != nil {
+		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
+	}
+	return nil
+}
@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// fakeCP is a test double for the gateway control-plane. It records registers,
+// task-state reports and GPU reports, and replays a scripted queue of poll
+// responses.
+type fakeCP struct {
+	mu sync.Mutex
+
+	registers    int
+	states       []bus.TaskState
+	gpuReports   []gpuReport
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int
+
+	registerErr bool
+	submitErr   bool
+}
+
+// gpuReport is one recorded ReportTaskGPU call.
+type gpuReport struct {
+	cluster    string
+	task       string
+	containers []handlers_ecs.ContainerGPUReport
+}
+
+var _ controlPlane = (*fakeCP)(nil)
+
+func (f *fakeCP) Register(_ identity) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.registerErr {
+		return errors.New("register boom")
+	}
+	f.registers++
+	return nil
+}
+
+func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.submitErr {
+		return errors.New("submit boom")
+	}
+	f.states = append(f.states, st)
+	return nil
+}
+
+func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
+	f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
+	var out []bus.Assign
+	if f.pollCalls < len(f.pollReplies) {
+		out = f.pollReplies[f.pollCalls]
+	}
+	var stops []bus.StopDirective
+	if f.pollCalls < len(f.stopReplies) {
+		stops = f.stopReplies[f.pollCalls]
+	}
+	f.pollCalls++
+	return out, stops, nil
+}
+
+func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
+	return nil
+}
+
+func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []gpuReport
+	for _, r := range f.gpuReports {
+		if r.task == taskID {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+func (f *fakeCP) registerCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.registers
+}
+
+func (f *fakeCP) taskStates() []bus.TaskState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]bus.TaskState(nil), f.states...)
+}
+
+func (f *fakeCP) acks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollAcks...)
+}
+
+func (f *fakeCP) stopAcks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollStopAcks...)
+}
+
+func testIdentity() identity {
+	return identity{
+		AccountID:    "123456789012",
+		ClusterName:  "default",
+		InstanceID:   "i-abc123",
+		AZ:           "us-east-1a",
+		Hostname:     "ecs-host",
+		Capacity:     bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
+		AgentVersion: "test",
+	}
+}
+
+// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
+// newAgent's ledger has real devices to pin from.
+func testIdentityWithGPUs(uuids ...string) identity {
+	id := testIdentity()
+	id.Capacity.GPU = len(uuids)
+	id.Capacity.GPUIDs = uuids
+	return id
+}
+
+func TestRegistrar_Register(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("registers = %d, want 1", cp.registerCount())
+	}
+}
+
+func TestRegistrar_RegisterError(t *testing.T) {
+	cp := &fakeCP{registerErr: true}
+	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+		t.Fatal("expected register error")
+	}
+}
+
+func TestHeartbeater_Beat(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("beat should re-register once, got %d", cp.registerCount())
+	}
+}
+
+func TestHeartbeater_DefaultInterval(t *testing.T) {
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	if h.interval != defaultHeartbeat {
+		t.Errorf("interval = %v, want default", h.interval)
+	}
+}
+
+func TestHeartbeater_RunStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	done := make(chan struct{})
+	go func() { h.Run(ctx); close(done) }()
+
+	time.Sleep(25 * time.Millisecond)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("Run did not stop on context cancel")
+	}
+	if cp.registerCount() == 0 {
+		t.Error("expected at least one re-register before cancel")
+	}
+}
@@ -0,0 +1,118 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/remotes"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/distribution/reference"
+)
+
+// ecsNamespace isolates ecs-agent images/containers in containerd, mirroring the
+// "moby"/"k8s.io" convention other runtimes use.
+const ecsNamespace = "ecs"
+
+// servingProbeTimeout bounds the boot-time daemon liveness check.
+const servingProbeTimeout = 3 * time.Second
+
+// containerdPuller drives a real containerd daemon over its unix socket.
+type containerdPuller struct {
+	client *containerd.Client
+}
+
+var _ ImagePuller = (*containerdPuller)(nil)
+
+// New connects to the containerd daemon at socket (e.g.
+// "/run/containerd/containerd.sock") and returns an ImagePuller. The containerd
+// client dials lazily, so New actively probes the daemon with IsServing and
+// returns an error if it is unreachable — otherwise a dead socket would only
+// surface on the first pull, and the agent's boot-time "containerd unavailable"
+// log would never fire.
+func New(socket string) (Runtime, error) {
+	client, err := containerd.New(socket, containerd.WithDefaultNamespace(ecsNamespace))
+	if err != nil {
+		return nil, fmt.Errorf("connect containerd %s: %w", socket, err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), servingProbeTimeout)
+	defer cancel()
+	serving, err := client.IsServing(ctx)
+	if err != nil || !serving {
+		_ = client.Close()
+		return nil, fmt.Errorf("containerd %s not serving: %w", socket, err)
+	}
+	return &containerdPuller{client: client}, nil
+}
+
+// Pull resolves credentials via r, then pulls and unpacks the image.
+func (p *containerdPuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Ref)
+	if err != nil {
+		return Image{}, err
+	}
+
+	resolver, err := newDockerResolver(ctx, ref, r)
+	if err != nil {
+		return Image{}, err
+	}
+
+	img, err := p.client.Pull(ctx, ref,
+		containerd.WithPullUnpack,
+		containerd.WithResolver(resolver),
+	)
+	if err != nil {
+		return Image{}, fmt.Errorf("pull %s: %w", ref, err)
+	}
+
+	target := img.Target()
+	return Image{
+		Ref:    ref,
+		Digest: target.Digest.String(),
+		Size:   target.Size,
+	}, nil
+}
+
+// normalizeRef expands a task-definition image string into the fully-qualified
+// reference containerd's resolver requires: a bare "nginx:alpine" becomes
+// "docker.io/library/nginx:alpine" and an untagged ref gets ":latest". Without
+// this, containerd parses the bare ref as "dummy://nginx:alpine" and rejects it.
+func normalizeRef(ref string) (string, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", fmt.Errorf("parse image ref %q: %w", ref, err)
+	}
+	return reference.TagNameOnly(named).String(), nil
+}
+
+// Close releases the containerd client connection.
+func (p *containerdPuller) Close() error {
+	if p.client == nil {
+		return nil
+	}
+	return p.client.Close()
+}
+
+// newDockerResolver builds a containerd remote resolver whose per-host
+// credentials come from r (the ECR token resolver). Credentials are fetched once
+// for ref and reused for every host containerd asks about during the pull.
+func newDockerResolver(ctx context.Context, ref string, r Resolver) (remotes.Resolver, error) {
+	creds := func(string) (string, string, error) { return "", "", nil }
+	if r != nil {
+		user, pass, _, err := r.Authorize(ctx, ref)
+		if err != nil {
+			return nil, fmt.Errorf("authorize %s: %w", ref, err)
+		}
+		creds = func(string) (string, string, error) { return user, pass, nil }
+	}
+
+	authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+	return docker.NewResolver(docker.ResolverOptions{
+		Hosts: docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer)),
+	}), nil
+}
@@ -0,0 +1,172 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cdi"
+	"github.com/containerd/containerd/v2/pkg/cio"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// cdiVendorGPU is the NVIDIA CDI vendor/class prefix for whole-GPU devices, per
+// the NVIDIA CDI naming convention (nvidia.com/gpu=<uuid>).
+const cdiVendorGPU = "nvidia.com/gpu"
+
+// cdiDeviceNames maps a container's pinned GPU device UUIDs to CDI device
+// names. Empty input (non-GPU container) returns nil so no CDI opt is added.
+func cdiDeviceNames(uuids []string) []string {
+	if len(uuids) == 0 {
+		return nil
+	}
+	names := make([]string, len(uuids))
+	for i, id := range uuids {
+		names[i] = cdiVendorGPU + "=" + id
+	}
+	return names
+}
+
+// cdiSpecOpts returns the oci.SpecOpts needed to inject gpuIDs (via
+// containerd's CDI machinery) as devices into a container's OCI spec. It
+// returns nil for a non-GPU container so Run adds nothing to the opts list.
+func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
+	devices := cdiDeviceNames(gpuIDs)
+	if len(devices) == 0 {
+		return nil
+	}
+	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
+}
+
+var _ Runner = (*containerdPuller)(nil)
+
+// Run creates a container from an already-pulled image and starts its task. The
+// container joins the host network namespace (bridge/CNI task networking is a
+// later sprint) and carries spec.Labels for the reboot reconciler. id must be
+// unique within the "ecs" namespace.
+func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (string, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Image)
+	if err != nil {
+		return "", err
+	}
+	image, err := p.client.GetImage(ctx, ref)
+	if err != nil {
+		return "", fmt.Errorf("get image %s: %w", ref, err)
+	}
+
+	specOpts := []oci.SpecOpts{
+		oci.WithImageConfig(image),
+		oci.WithHostHostsFile,
+		oci.WithHostResolvconf,
+	}
+	if spec.NetnsPath != "" {
+		// awsvpc: join the task ENI netns built by the agent.
+		specOpts = append(specOpts, oci.WithLinuxNamespace(specs.LinuxNamespace{
+			Type: specs.NetworkNamespace,
+			Path: spec.NetnsPath,
+		}))
+	} else {
+		// bridge/host: share the VM (host) netns.
+		specOpts = append(specOpts, oci.WithHostNamespace(specs.NetworkNamespace))
+	}
+	if len(spec.Command) > 0 {
+		specOpts = append(specOpts, oci.WithProcessArgs(spec.Command...))
+	}
+	if len(spec.Env) > 0 {
+		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
+	}
+	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
+
+	container, err := p.client.NewContainer(ctx, id,
+		containerd.WithNewSnapshot(id+"-snapshot", image),
+		containerd.WithNewSpec(specOpts...),
+		containerd.WithContainerLabels(spec.Labels),
+	)
+	if err != nil {
+		return "", fmt.Errorf("create container %s: %w", id, err)
+	}
+
+	task, err := container.NewTask(ctx, cio.NullIO)
+	if err != nil {
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("create task %s: %w", id, err)
+	}
+	if err := task.Start(ctx); err != nil {
+		_, _ = task.Delete(ctx)
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("start task %s: %w", id, err)
+	}
+	return id, nil
+}
+
+// Wait blocks until the container's task exits and returns its exit code.
+func (p *containerdPuller) Wait(ctx context.Context, containerID string) (RunStatus, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("load container %s: %w", containerID, err)
+	}
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("attach task %s: %w", containerID, err)
+	}
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("wait task %s: %w", containerID, err)
+	}
+	status := <-statusC
+	code, _, err := status.Result()
+	return RunStatus{ExitCode: int(code)}, err
+}
+
+// List enumerates every container in the ecs namespace with its labels and
+// whether its task is running, so the agent can re-adopt live containers after a
+// restart. Per-container label/task errors are treated as "not running" rather
+// than failing the whole list.
+func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	containers, err := p.client.Containers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list containers: %w", err)
+	}
+	out := make([]Container, 0, len(containers))
+	for _, c := range containers {
+		labels, _ := c.Labels(ctx)
+		running := false
+		if task, terr := c.Task(ctx, nil); terr == nil {
+			if status, serr := task.Status(ctx); serr == nil {
+				running = status.Status == containerd.Running
+			}
+		}
+		out = append(out, Container{ID: c.ID(), Labels: labels, Running: running})
+	}
+	return out, nil
+}
+
+// Remove kills and deletes the container's task, then the container + snapshot.
+func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return nil //nolint:nilerr // load failure means the container is already gone
+	}
+	if task, terr := container.Task(ctx, nil); terr == nil {
+		_, _ = task.Delete(ctx, containerd.WithProcessKill)
+	}
+	return container.Delete(ctx, containerd.WithSnapshotCleanup)
+}
+
+// envSlice renders an env map as sorted "K=V" entries for deterministic specs.
+func envSlice(env map[string]string) []string {
+	out := make([]string, 0, len(env))
+	for k, v := range env {
+		out = append(out, k+"="+v)
+	}
+	sort.Strings(out)
+	return out
+}
@@ -0,0 +1,47 @@
+package runtime
+
+import (
+	"reflect"
+	"testing"
+)
+
+// cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
+// (nvidia.com/gpu=<uuid>); Run wires the result into oci.WithCDIDevices when
+// non-empty, so this is the injection logic's unit-testable core.
+func TestCDIDeviceNames(t *testing.T) {
+	cases := []struct {
+		name  string
+		uuids []string
+		want  []string
+	}{
+		{name: "empty", uuids: nil, want: nil},
+		{name: "single", uuids: []string{"GPU-aaaaaaaa-1111-2222-3333-444444444444"},
+			want: []string{"nvidia.com/gpu=GPU-aaaaaaaa-1111-2222-3333-444444444444"}},
+		{name: "multiple preserves order", uuids: []string{"GPU-aaa", "GPU-bbb"},
+			want: []string{"nvidia.com/gpu=GPU-aaa", "nvidia.com/gpu=GPU-bbb"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := cdiDeviceNames(tc.uuids)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("cdiDeviceNames(%v) = %v, want %v", tc.uuids, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestCDISpecOpts_Wiring confirms Run's specOpts list picks up exactly one CDI
+// opt when GPUIDs is non-empty, and none for a non-GPU container — the wiring
+// Run relies on (a live oci.Spec is needed to unit test the opt's effect).
+func TestCDISpecOpts_Wiring(t *testing.T) {
+	if opts := cdiSpecOpts(nil); len(opts) != 0 {
+		t.Errorf("non-GPU container: want no spec opts, got %d", len(opts))
+	}
+	opts := cdiSpecOpts([]string{"GPU-aaa", "GPU-bbb"})
+	if len(opts) != 1 {
+		t.Fatalf("GPU container: want exactly one CDI spec opt, got %d", len(opts))
+	}
+	if opts[0] == nil {
+		t.Error("GPU container: spec opt is nil")
+	}
+}
@@ -0,0 +1,88 @@
+// Package runtime is the ecs-agent's container-runtime seam. The agent pulls
+// task images and (from Sprint 4e) runs containers through an ImagePuller; the
+// real implementation drives containerd, but the interface lets tests and the
+// scheduler-only build run against a fake.
+package runtime
+
+import "context"
+
+// PullSpec describes one image to pull.
+type PullSpec struct {
+	// Ref is the full image reference, e.g.
+	// "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:latest".
+	Ref string
+}
+
+// Image identifies a pulled image in the runtime's content store.
+type Image struct {
+	Ref    string
+	Digest string
+	Size   int64
+}
+
+// Resolver supplies registry credentials for a given image reference. The
+// ecs-agent backs this with ECR GetAuthorizationToken (see ecrresolver.go).
+type Resolver interface {
+	// Authorize returns the username, password and registry endpoint to use for
+	// ref. An empty user/pass means anonymous pull.
+	Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error)
+}
+
+// ImagePuller pulls images into the local runtime. Close releases the runtime
+// client connection.
+type ImagePuller interface {
+	Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error)
+	Close() error
+}
+
+// RunSpec describes a single container the agent must create and start. v1 runs
+// the container in the host network namespace (bridge/CNI task networking lands
+// in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
+// reconciler.
+type RunSpec struct {
+	Image   string
+	Command []string
+	Env     map[string]string
+	Labels  map[string]string
+	// NetnsPath, when set, joins the container to a pre-built network namespace
+	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
+	// bridge/host mode behaviour.
+	NetnsPath string
+	// GPU is the whole-GPU count requested for this container.
+	GPU int
+	// GPUIDs are the ledger-pinned device UUIDs assigned to this container, one
+	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
+	// The runner injects these as CDI devices into the OCI spec.
+	GPUIDs []string
+}
+
+// RunStatus is a finished container's outcome.
+type RunStatus struct {
+	ExitCode int
+}
+
+// Container is a container the runtime already knows about, discovered by List.
+// Labels carry the mulga.ecs.* task identity so the agent can re-adopt running
+// containers after a restart; Running reports whether its task is live.
+type Container struct {
+	ID      string
+	Labels  map[string]string
+	Running bool
+}
+
+// Runner creates and starts containers from already-pulled images. id is a
+// caller-unique container ID; Wait blocks until the container exits; Remove
+// tears down the container + its task; List enumerates known containers so the
+// reboot reconciler can re-adopt the ones still running.
+type Runner interface {
+	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
+	Wait(ctx context.Context, containerID string) (RunStatus, error)
+	Remove(ctx context.Context, containerID string) error
+	List(ctx context.Context) ([]Container, error)
+}
+
+// Runtime is the full container runtime the ecs-agent drives: pull + run.
+type Runtime interface {
+	ImagePuller
+	Runner
+}
@@ -0,0 +1,125 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+// FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
+// It records pull requests and replays a programmed result or error.
+type FakePuller struct {
+	mu     sync.Mutex
+	Pulls  []PullSpec
+	Authzd []string
+	Result Image
+	Err    error
+	Closed bool
+	OnPull func(PullSpec) (Image, error)
+
+	// Run bookkeeping.
+	Runs     []RunSpec
+	RunErr   error
+	WaitCode int
+	WaitErr  error
+	Waited   []string
+	Removed  []string
+	OnRun    func(string, RunSpec) (string, error)
+
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
+	// Listed records that List was called.
+	Containers []Container
+	ListErr    error
+	Listed     bool
+}
+
+var (
+	_ ImagePuller = (*FakePuller)(nil)
+	_ Runner      = (*FakePuller)(nil)
+)
+
+// Pull records the spec, exercises the resolver, and returns the programmed
+// result. If OnPull is set it takes precedence over Result/Err.
+func (f *FakePuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	f.mu.Lock()
+	f.Pulls = append(f.Pulls, spec)
+	f.mu.Unlock()
+
+	if r != nil {
+		user, _, _, err := r.Authorize(ctx, spec.Ref)
+		if err != nil {
+			return Image{}, fmt.Errorf("authorize %s: %w", spec.Ref, err)
+		}
+		f.mu.Lock()
+		f.Authzd = append(f.Authzd, user)
+		f.mu.Unlock()
+	}
+
+	if f.OnPull != nil {
+		return f.OnPull(spec)
+	}
+	if f.Err != nil {
+		return Image{}, f.Err
+	}
+	res := f.Result
+	if res.Ref == "" {
+		res.Ref = spec.Ref
+	}
+	return res, nil
+}
+
+// Close marks the puller closed.
+func (f *FakePuller) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Closed = true
+	return nil
+}
+
+// Run records the spec and returns id (or the programmed result/error).
+func (f *FakePuller) Run(_ context.Context, id string, spec RunSpec) (string, error) {
+	f.mu.Lock()
+	f.Runs = append(f.Runs, spec)
+	f.mu.Unlock()
+	if f.OnRun != nil {
+		return f.OnRun(id, spec)
+	}
+	if f.RunErr != nil {
+		return "", f.RunErr
+	}
+	return id, nil
+}
+
+// Wait records the container ID and returns the programmed exit code/error.
+func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, error) {
+	f.mu.Lock()
+	f.Waited = append(f.Waited, containerID)
+	f.mu.Unlock()
+	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
+}
+
+// Remove records the container ID.
+func (f *FakePuller) Remove(_ context.Context, containerID string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Removed = append(f.Removed, containerID)
+	return nil
+}
+
+// Waits returns a copy of the container IDs passed to Wait, for assertions.
+func (f *FakePuller) Waits() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.Waited...)
+}
+
+// List replays the programmed container set (or error).
+func (f *FakePuller) List(_ context.Context) ([]Container, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Listed = true
+	if f.ListErr != nil {
+		return nil, f.ListErr
+	}
+	return f.Containers, nil
+}
@@ -0,0 +1,89 @@
+package runtime
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// staticResolver is a test Resolver returning fixed creds or an error.
+type staticResolver struct {
+	user, pass, endpoint string
+	err                  error
+}
+
+func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
+	return s.user, s.pass, s.endpoint, s.err
+}
+
+func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
+	f := &FakePuller{}
+	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
+	if err != nil {
+		t.Fatalf("Pull: %v", err)
+	}
+	if img.Ref != "repo/app:1" {
+		t.Errorf("Ref defaulted wrong: %q", img.Ref)
+	}
+	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
+		t.Errorf("Pulls = %+v", f.Pulls)
+	}
+	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
+		t.Errorf("Authzd = %+v", f.Authzd)
+	}
+}
+
+func TestFakePuller_ResolverError(t *testing.T) {
+	f := &FakePuller{}
+	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
+	if err == nil {
+		t.Fatal("expected resolver error to propagate")
+	}
+}
+
+func TestFakePuller_ProgrammedError(t *testing.T) {
+	f := &FakePuller{Err: errors.New("pull failed")}
+	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
+		t.Fatal("expected programmed error")
+	}
+}
+
+func TestFakePuller_OnPullOverrides(t *testing.T) {
+	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
+	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
+		return want, nil
+	}}
+	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
+	if err != nil || got != want {
+		t.Fatalf("got %+v, %v; want %+v", got, err, want)
+	}
+}
+
+func TestFakePuller_Close(t *testing.T) {
+	f := &FakePuller{}
+	if err := f.Close(); err != nil || !f.Closed {
+		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
+	}
+}
+
+func TestNormalizeRef(t *testing.T) {
+	cases := map[string]string{
+		"nginx:alpine":      "docker.io/library/nginx:alpine",
+		"nginx":             "docker.io/library/nginx:latest",
+		"ghcr.io/org/app:1": "ghcr.io/org/app:1",
+		"123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1": "123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1",
+	}
+	for in, want := range cases {
+		got, err := normalizeRef(in)
+		if err != nil {
+			t.Errorf("normalizeRef(%q): %v", in, err)
+			continue
+		}
+		if got != want {
+			t.Errorf("normalizeRef(%q) = %q, want %q", in, got, want)
+		}
+	}
+	if _, err := normalizeRef("Bad::Ref"); err == nil {
+		t.Error("normalizeRef accepted invalid ref")
+	}
+}
@@ -0,0 +1,340 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"maps"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// pollAssignments drains the instance's assignment inbox on a ticker until ctx is
+// cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
+// on a poll are acked on the next poll so the gateway can drop them — a crash
+// before ack re-delivers (at-least-once), matching ACS. A failed poll is logged
+// and retried on the next tick rather than killing the loop. dispatched is seeded
+// by the startup reconcile with tasks already adopted from running containers, so
+// their re-delivered assignments are acked but not re-run.
+func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool) {
+	ticker := time.NewTicker(a.cfg.PollInterval)
+	defer ticker.Stop()
+
+	stopping := map[string]bool{}
+	var ackAssigns, ackStops []string
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			assigns, stops, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackAssigns, ackStops)
+			if err != nil {
+				slog.Warn("ecs-agent: poll assignments failed", "err", err)
+				continue
+			}
+			// Dispatch stops first so a task stopped in the same poll it was assigned
+			// is never started.
+			ackStops = ackStops[:0]
+			for i := range stops {
+				sd := stops[i]
+				if !stopping[sd.TaskID] {
+					stopping[sd.TaskID] = true
+					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
+					go a.stopTask(ctx, sd)
+				}
+				ackStops = append(ackStops, sd.TaskID)
+			}
+			ackAssigns = ackAssigns[:0]
+			for i := range assigns {
+				as := assigns[i]
+				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
+					dispatched[as.TaskID] = true
+					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
+					go a.runTask(ctx, &as)
+				}
+				ackAssigns = append(ackAssigns, as.TaskID)
+			}
+		}
+	}
+}
+
+// stopTask reaps a task's containers on a control-plane stop directive: it lists
+// the runtime's containers, removes the ones labeled with this task (kill +
+// delete, releasing host ports/netns), then reports the task STOPPED with the
+// directive's reason. Idempotent — a task with no live containers still reports
+// STOPPED so the scheduler releases its capacity.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
+	if a.runner == nil {
+		return
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
+		return
+	}
+	statuses := make([]bus.ContainerStatus, 0)
+	for _, c := range containers {
+		if c.Labels[labelTaskID] != sd.TaskID {
+			continue
+		}
+		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
+			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	reason := sd.Reason
+	if reason == "" {
+		reason = "Task stopped"
+	}
+	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
+	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+}
+
+// runTask pulls each container image, starts the containers, and reports task
+// state on the bus. A missing runtime or any per-container failure reports the
+// task STOPPED with a reason; success reports RUNNING and waits for exit.
+func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
+	if a.puller == nil || a.runner == nil {
+		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
+		return
+	}
+
+	// awsvpc: build the task netns from the hot-plugged ENI before any container.
+	netnsPath, err := a.setupTaskNetns(as)
+	if err != nil {
+		slog.Error("ecs-agent: task netns setup failed", "task", as.TaskID, "err", err)
+		a.reportTaskState(as, bus.TaskStatusStopped, "network setup failed: "+err.Error(), nil)
+		return
+	}
+
+	credID := a.registerTaskCreds(as)
+	resolver := a.pullResolver(as)
+
+	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
+	for _, c := range as.Containers {
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
+			return
+		}
+
+		cid := containerID(as.TaskID, c.Name)
+		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
+		spec := ctrruntime.RunSpec{
+			Image:     c.Image,
+			Command:   c.Command,
+			Env:       withCredEnv(c.Environment, credID),
+			Labels:    taskLabels(as, c.Name),
+			NetnsPath: netnsPath,
+			GPU:       c.GPU,
+			GPUIDs:    gpuIDs,
+		}
+		id, err := a.runner.Run(ctx, cid, spec)
+		if err != nil {
+			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
+			return
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
+		})
+		go a.waitContainer(ctx, as, c.Name, id)
+	}
+
+	a.reportTaskState(as, bus.TaskStatusRunning, "", statuses)
+	slog.Info("ecs-agent: task running", "task", as.TaskID, "containers", len(statuses))
+}
+
+// waitContainer blocks until a container exits, then reports the task STOPPED
+// with the exit code. v1 stops the whole task when its first container exits.
+func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
+	status, err := a.runner.Wait(ctx, containerID)
+	if err != nil {
+		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
+		return
+	}
+	exit := status.ExitCode
+	a.teardownTaskNetns(as)
+	statuses := []bus.ContainerStatus{{Name: name, Status: bus.TaskStatusStopped, ContainerID: containerID, ExitCode: &exit}}
+	a.reportTaskState(as, bus.TaskStatusStopped, fmt.Sprintf("container %s exited (%d)", name, exit), statuses)
+	slog.Info("ecs-agent: task stopped", "task", as.TaskID, "container", name, "exitCode", exit)
+}
+
+// setupTaskNetns builds the awsvpc task netns from the hot-plugged ENI. Bridge/
+// host tasks (no ENI MAC) and a nil controller are no-ops returning an empty path.
+func (a *Agent) setupTaskNetns(as *bus.Assign) (string, error) {
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return "", nil
+	}
+	return a.netns.Setup(as.TaskID, as.ENIMacAddress)
+}
+
+// teardownTaskNetns releases the task's IAM credential registration, then removes
+// the awsvpc task netns (the latter is a no-op for bridge/host tasks). It runs at
+// every task-stop path so credentials never outlive the task that owns them.
+func (a *Agent) teardownTaskNetns(as *bus.Assign) {
+	if a.cred != nil {
+		a.cred.Deregister(taskCredID(as))
+	}
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return
+	}
+	if err := a.netns.Teardown(as.TaskID); err != nil {
+		slog.Warn("ecs-agent: task netns teardown", "task", as.TaskID, "err", err)
+	}
+}
+
+// pullResolver returns the ECR resolver used for a task's image pulls. When the
+// assign carries an execution role, it authorizes pulls as that role (assumed
+// over the gateway); an empty role or a nil credential endpoint (unit tests)
+// falls back to the instance-role resolver, so a pull is never worse off.
+func (a *Agent) pullResolver(as *bus.Assign) ctrruntime.Resolver {
+	if as.ExecutionRoleARN == "" || a.cred == nil {
+		return a.resolver
+	}
+	prov := a.cred.AssumeProvider(as.ExecutionRoleARN, sessionName(as.TaskID))
+	return newLazyECRResolver(prov, a.cfg.Region, a.cfg.GatewayURL, a.cfg.GatewayCA)
+}
+
+// registerTaskCreds registers the task's credID -> role mapping with the
+// credential endpoint and returns the credID (empty when the task has no role).
+func (a *Agent) registerTaskCreds(as *bus.Assign) string {
+	credID := taskCredID(as)
+	if credID == "" || a.cred == nil {
+		return credID
+	}
+	a.cred.Register(credID, as.TaskRoleARN)
+	return credID
+}
+
+// taskCredID is the credential ID a task's containers fetch credentials under:
+// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
+// carries no IAM role.
+func taskCredID(as *bus.Assign) string {
+	if as.TaskRoleARN == "" {
+		return ""
+	}
+	if as.CredID != "" {
+		return as.CredID
+	}
+	return as.TaskID
+}
+
+// withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for
+// credID, copying the map so the assign's container env is left untouched. A
+// blank credID (no task role) returns env unchanged.
+func withCredEnv(env map[string]string, credID string) map[string]string {
+	if credID == "" {
+		return env
+	}
+	out := make(map[string]string, len(env)+1)
+	maps.Copy(out, env)
+	out["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = credRelativeURI(credID)
+	return out
+}
+
+// pinContainerGPUs reserves n device UUIDs for a container from the local
+// ledger, before the container is started so the runner can inject them as
+// CDI devices. A short ledger (fewer free devices than requested, e.g.
+// discovery found none) is logged and treated as best-effort: the container
+// still runs without pinned UUIDs (and without GPU access) rather than
+// failing the task.
+func (a *Agent) pinContainerGPUs(taskID, containerName string, n int) []string {
+	if n <= 0 || a.gpu == nil {
+		return nil
+	}
+	uuids, err := a.gpu.Pin(gpuKey(taskID, containerName), n)
+	if err != nil {
+		slog.Warn("ecs-agent: gpu pin short", "task", taskID, "container", containerName, "err", err)
+	}
+	return uuids
+}
+
+// reportTaskState reports a task transition through the gateway's
+// SubmitTaskStateChange action, then (RUNNING with pinned GPUs) reports the
+// per-container device UUIDs, and (STOPPED) releases them back to the ledger.
+func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
+	if a.cp == nil {
+		return
+	}
+	msg := bus.TaskState{
+		AccountID:   a.id.AccountID,
+		ClusterName: a.id.ClusterName,
+		InstanceID:  a.id.InstanceID,
+		TaskID:      as.TaskID,
+		LastStatus:  status,
+		Containers:  containers,
+		Reason:      reason,
+		ReportedAt:  time.Now().UTC(),
+	}
+	if err := a.cp.SubmitTaskState(msg); err != nil {
+		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
+	}
+	a.reportTaskGPUs(as.TaskID, containers)
+	if status == bus.TaskStatusStopped && a.gpu != nil {
+		a.gpu.ReleaseTask(as.TaskID)
+	}
+}
+
+// reportTaskGPUs sends the control plane the pinned device UUIDs for any
+// container in the report that has them, so DescribeTasks can surface gpuIds.
+// A no-op when nothing was pinned (the common non-GPU-task case).
+func (a *Agent) reportTaskGPUs(taskID string, containers []bus.ContainerStatus) {
+	reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
+	for _, c := range containers {
+		if len(c.GPUIDs) > 0 {
+			reports = append(reports, handlers_ecs.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
+		}
+	}
+	if len(reports) == 0 {
+		return
+	}
+	if err := a.cp.ReportTaskGPU(a.id.ClusterName, taskID, reports); err != nil {
+		slog.Error("ecs-agent: report task-gpu failed", "task", taskID, "err", err)
+	}
+}
+
+// containerID composes a containerd-valid container ID from a task + container.
+func containerID(taskID, name string) string {
+	return fmt.Sprintf("%s-%s", taskID, name)
+}
+
+// mulga.ecs.* label keys stamped on every container. The reboot reconciler reads
+// them back to re-associate running containers with their task on restart; the
+// cred/role/MAC labels make a container self-describing enough to re-register its
+// credentials and tear down its netns without the original assignment.
+const (
+	labelTaskID        = "mulga.ecs.taskID"
+	labelContainerName = "mulga.ecs.containerName"
+	labelClusterName   = "mulga.ecs.clusterName"
+	labelCredID        = "mulga.ecs.credID"
+	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
+	labelENIMac        = "mulga.ecs.eniMac"
+)
+
+// taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC
+// labels are omitted when empty so a container carries only what it needs.
+func taskLabels(as *bus.Assign, name string) map[string]string {
+	labels := map[string]string{
+		labelTaskID:        as.TaskID,
+		labelContainerName: name,
+		labelClusterName:   as.ClusterName,
+	}
+	if credID := taskCredID(as); credID != "" {
+		labels[labelCredID] = credID
+	}
+	if as.TaskRoleARN != "" {
+		labels[labelTaskRoleARN] = as.TaskRoleARN
+	}
+	if as.ENIMacAddress != "" {
+		labels[labelENIMac] = as.ENIMacAddress
+	}
+	return labels
+}
@@ -0,0 +1,557 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func testAssign() *bus.Assign {
+	return &bus.Assign{
+		AccountID:   "123456789012",
+		ClusterName: "default",
+		InstanceID:  "i-abc123",
+		TaskID:      "t-001",
+		Containers: []bus.AssignContainer{
+			{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"},
+				Environment: map[string]string{"FOO": "bar"}},
+		},
+	}
+}
+
+func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
+	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+}
+
+// runTask with no runtime reports the task STOPPED instead of crashing.
+func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED state, got %+v", st)
+	}
+	if st[0].Reason == "" {
+		t.Error("STOPPED state missing reason")
+	}
+}
+
+// Happy path: pull + run reports RUNNING with the container ID; Wait blocking
+// (forced error) means no STOPPED overwrite, so we can assert the RUNNING frame.
+func TestRunTask_PullRunReportsRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if len(rt.Pulls) != 1 || rt.Pulls[0].Ref != "registry/web:1" {
+		t.Fatalf("expected one pull of registry/web:1, got %+v", rt.Pulls)
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].Image != "registry/web:1" {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].Labels["mulga.ecs.taskID"] != "t-001" {
+		t.Errorf("missing task label: %+v", rt.Runs[0].Labels)
+	}
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	if len(st[len(st)-1].Containers) != 1 || st[len(st)-1].Containers[0].ContainerID != "t-001-web" {
+		t.Errorf("RUNNING state container wrong: %+v", st[len(st)-1].Containers)
+	}
+}
+
+// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
+// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// container with no GPU request keeps RunSpec.GPU at zero (regression).
+func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].GPU != 0 {
+		t.Errorf("web container: want GPU=0, got %d", rt.Runs[0].GPU)
+	}
+	if rt.Runs[1].GPU != 2 {
+		t.Errorf("trainer container: want GPU=2, got %d", rt.Runs[1].GPU)
+	}
+}
+
+// TestRunTask_GPUPinsAndReportsUUIDs covers C3: a container's GPU request is
+// pinned from the agent's local ledger, carried on the RUNNING report's
+// ContainerStatus, and forwarded to the control plane via ReportTaskGPU.
+func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	last := st[len(st)-1]
+	var trainer *bus.ContainerStatus
+	for i := range last.Containers {
+		if last.Containers[i].Name == "trainer" {
+			trainer = &last.Containers[i]
+		}
+	}
+	if trainer == nil || len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container missing pinned GPU UUID: %+v", last.Containers)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+
+	reports := cp.gpuReportsFor(as.TaskID)
+	if len(reports) != 1 {
+		t.Fatalf("want one ReportTaskGPU call, got %d", len(reports))
+	}
+	if reports[0].cluster != as.ClusterName {
+		t.Errorf("report cluster = %q, want %q", reports[0].cluster, as.ClusterName)
+	}
+	if len(reports[0].containers) != 1 || reports[0].containers[0].Name != "trainer" {
+		t.Errorf("report containers = %+v, want one trainer entry", reports[0].containers)
+	}
+}
+
+// TestRunTask_GPUIDsReachRunSpec covers C4: the ledger's pinned UUIDs for a
+// container reach RunSpec.GPUIDs (not just the RunSpec.GPU count), so the
+// runner can inject them as CDI devices; a non-GPU container's RunSpec
+// carries no GPUIDs.
+func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if len(rt.Runs[0].GPUIDs) != 0 {
+		t.Errorf("web container: want no GPUIDs, got %+v", rt.Runs[0].GPUIDs)
+	}
+	trainer := rt.Runs[1]
+	if len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container: want one pinned GPUID, got %+v", trainer.GPUIDs)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+}
+
+// A container with no GPU request never triggers a ReportTaskGPU call.
+func TestRunTask_NoGPUSkipsReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report for a non-GPU task, got %+v", reports)
+	}
+}
+
+// A GPU request exceeding the ledger's free devices (discovery found none, or
+// fewer than requested) is best-effort: the container still runs, it just
+// carries no pinned UUIDs and so no CDI devices are injected.
+func TestRunTask_GPUShortLedgerStillRuns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt) // testIdentity() has no discovered GPU UUIDs
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected both containers to run despite the short ledger, got %+v", rt.Runs)
+	}
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report when nothing was pinned, got %+v", reports)
+	}
+}
+
+// stopTask releases the task's pinned GPU UUIDs back to the ledger so a later
+// task on the same instance can reuse them.
+func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-trainer", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
+		}},
+	}}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
+		t.Fatalf("seed pin: %v", err)
+	}
+	if len(a.gpu.free) != 0 {
+		t.Fatalf("setup: want ledger fully pinned, free = %v", a.gpu.free)
+	}
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(a.gpu.free) != 1 {
+		t.Errorf("want the GPU released back to the ledger, free = %v", a.gpu.free)
+	}
+}
+
+// A pull failure reports STOPPED and never starts the container.
+func TestRunTask_PullFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Err: errors.New("pull denied")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 0 {
+		t.Errorf("container should not run after pull failure, got %+v", rt.Runs)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// A run failure reports STOPPED.
+func TestRunTask_RunFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{RunErr: errors.New("start boom")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// On container exit, waitContainer reports STOPPED with the exit code.
+func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitCode: 7}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	deadline := time.After(time.Second)
+	for {
+		st := cp.taskStates()
+		if len(st) > 0 && st[len(st)-1].LastStatus == bus.TaskStatusStopped {
+			last := st[len(st)-1]
+			if len(last.Containers) != 1 || last.Containers[0].ExitCode == nil || *last.Containers[0].ExitCode != 7 {
+				t.Fatalf("want exit code 7, got %+v", last.Containers)
+			}
+			return
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("no STOPPED state after exit; got %+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// pollAssignments dispatches each assign once and acks it on the next poll.
+func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{*as}, // poll 1: one pending assign
+		{*as}, // poll 2: still present (not yet acked when poll 2 was issued)
+		nil,   // poll 3: gateway dropped it after the ack
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	// runTask reports RUNNING through the (mutex-guarded) control plane. With the
+	// assign returned on two consecutive polls, dedup means exactly one dispatch →
+	// exactly one RUNNING report.
+	deadline := time.After(time.Second)
+	for {
+		if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("assign was never dispatched (no RUNNING report)")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 1 {
+		t.Errorf("task dispatched %d times, want exactly 1 (dedup on taskID)", running)
+	}
+	// The taskID must appear in a later poll's ack list.
+	acked := false
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("task %s was never acked; acks=%v", as.TaskID, cp.acks())
+	}
+}
+
+// stopTask reaps the task's labeled containers and reports STOPPED with the
+// directive reason and container status.
+func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		}},
+		{ID: "t-999-other", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-999", "mulga.ecs.containerName": "web",
+		}},
+	}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "bye" {
+		t.Errorf("want reason %q, got %q", "bye", st[0].Reason)
+	}
+	if len(st[0].Containers) != 1 || st[0].Containers[0].Status != bus.TaskStatusStopped {
+		t.Errorf("want one STOPPED container, got %+v", st[0].Containers)
+	}
+}
+
+// A stop for a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity.
+func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "gone"})
+
+	if len(rt.Removed) != 0 {
+		t.Errorf("nothing to remove, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+}
+
+// A stop delivered by poll is dispatched once, acked in the stop-ack list, and
+// suppresses a same-poll assign for the same task.
+func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
+		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for {
+		if stopped := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusStopped); stopped >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("stop never reaped; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	// The assign for the stopping task must never have started a container.
+	if len(rt.Runs) != 0 {
+		t.Errorf("assign for a stopping task should not run, got %+v", rt.Runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopping task should never report RUNNING, got %d", running)
+	}
+	acked := false
+	for _, ack := range cp.stopAcks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("stop %s was never acked; stopAcks=%v", as.TaskID, cp.stopAcks())
+	}
+}
+
+// stopTask with no runtime is a no-op: nothing to reap, no state reported.
+func TestStopTask_NilRunnerNoOp(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A List failure aborts the reap without reporting a (false) STOPPED state.
+func TestStopTask_ListErrorNoReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("list boom")}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("list error should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A directive with no reason reaps the matching container and reports STOPPED with
+// the default reason.
+func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "Task stopped" {
+		t.Errorf("want default reason %q, got %q", "Task stopped", st[0].Reason)
+	}
+}
+
+func countStatus(states []bus.TaskState, taskID, status string) int {
+	n := 0
+	for _, st := range states {
+		if st.TaskID == taskID && st.LastStatus == status {
+			n++
+		}
+	}
+	return n
+}
+
+// awsvpc: an assign carrying an ENI MAC builds the task netns and passes its
+// path to the container RunSpec.
+func TestRunTask_AwsvpcBuildsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	as := testAssign()
+	as.ENIMacAddress = "52:54:00:de:ad:01"
+	a.runTask(context.Background(), as)
+
+	if !f.sawAny("ip netns add ecs-t-001") {
+		t.Fatalf("expected task netns build, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != netnsPathFor("t-001") {
+		t.Fatalf("want NetnsPath %q on RunSpec, got %+v", netnsPathFor("t-001"), rt.Runs)
+	}
+}
+
+// bridge/host: no ENI MAC means no netns is built and the container runs in the
+// host (VM) netns (empty NetnsPath).
+func TestRunTask_NoMacSkipsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	a.runTask(context.Background(), testAssign())
+
+	if len(f.joined()) != 0 {
+		t.Fatalf("expected no netns commands, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != "" {
+		t.Fatalf("want empty NetnsPath, got %+v", rt.Runs)
+	}
+}
+
+func TestContainerID(t *testing.T) {
+	if got := containerID("t-1", "web"); got != "t-1-web" {
+		t.Errorf("containerID = %q, want t-1-web", got)
+	}
+}
+
+// sentinelResolver is an identity-comparable Resolver for pullResolver tests.
+type sentinelResolver struct{}
+
+func (sentinelResolver) Authorize(context.Context, string) (string, string, string, error) {
+	return "", "", "", nil
+}
+
+// TestPullResolver covers siv-459: an assign with no execution role (or no
+// credential endpoint) uses the instance resolver; with both, a distinct
+// exec-role resolver is built.
+func TestPullResolver(t *testing.T) {
+	fake := sentinelResolver{}
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+
+	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
+		t.Fatalf("no execution role: want instance resolver fallback")
+	}
+	execAssign := &bus.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
+	if got := a.pullResolver(execAssign); got != fake {
+		t.Fatalf("nil cred endpoint: want instance resolver fallback")
+	}
+
+	a.cred = &credEndpoint{}
+	got := a.pullResolver(execAssign)
+	if got == nil || got == fake {
+		t.Fatalf("execution role + cred endpoint: want distinct exec-role resolver, got %v", got)
+	}
+}
@@ -17,10 +17,14 @@
 // Usage:
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
+//	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//
+// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
+// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
-// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
+// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
 package main

 import (
@@ -68,6 +72,7 @@ func main() {
 		accountID   string
 		clusterName string
 		resource    string
+		instanceID  string
 	)

 	flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
@@ -77,7 +82,8 @@ func main() {
 	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
 	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
-	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons")
+	flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
+	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()

 	switch {
@@ -85,15 +91,29 @@ func main() {
 		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case clusterName == "":
 		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case resource != "addons":
-		fatal("--resource must be addons")
+	case resource != "addons" && resource != "recovery":
+		fatal("--resource must be addons or recovery")
+	case resource == "recovery" && instanceID == "":
+		fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}

 	client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
 	if err != nil {
 		fatal(fmt.Sprintf("build gateway client: %v", err))
 	}
-	path := "/clusters/" + clusterName + "/internal-addons/" + accountID
+
+	var (
+		path string
+		emit func(io.Writer, []byte) error
+	)
+	switch resource {
+	case "recovery":
+		path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
+		emit = emitRecoveryTSV
+	default:
+		path = "/clusters/" + clusterName + "/internal-addons/" + accountID
+		emit = emitAddonsTSV
+	}

 	var lastErr error
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
@@ -106,7 +126,7 @@ func main() {
 			}
 			continue
 		}
-		if werr := emitAddonsTSV(os.Stdout, body); werr != nil {
+		if werr := emit(os.Stdout, body); werr != nil {
 			fatal(fmt.Sprintf("render response: %v", werr))
 		}
 		return
@@ -133,6 +153,43 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 	return w.Flush()
 }

+// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
+// here to keep this tiny VM binary free of the handler package's dependency graph.
+type recoveryDirective struct {
+	Epoch            int64  `json:"epoch"`
+	Action           string `json:"action"`
+	Snapshot         string `json:"snapshot"`
+	SnapshotRequired bool   `json:"snapshotRequired"`
+}
+
+type internalRecoveryResponse struct {
+	Directive recoveryDirective `json:"directive"`
+}
+
+// emitRecoveryTSV decodes the recovery directive and writes a single
+// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
+// consumes without a JSON parser. An unset action defaults to "none" (steady
+// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
+// aborts boot on fetch failure) else 0.
+func emitRecoveryTSV(out io.Writer, body []byte) error {
+	var resp internalRecoveryResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-recovery response: %w", err)
+	}
+	d := resp.Directive
+	if d.Action == "" {
+		d.Action = "none"
+	}
+	required := "0"
+	if d.SnapshotRequired {
+		required = "1"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
+		return err
+	}
+	return nil
+}
+
 func fatal(msg string) {
 	slog.Error("eks-gateway-fetch: " + msg)
 	os.Exit(1)
@@ -0,0 +1,245 @@
+// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
+// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
+//
+// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
+// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
+// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
+// serving cert with that CA is therefore what lets the agent validate the
+// reverse-tunnel endpoint with no separate CA distribution.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
+// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
+// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
+// and cache it under -dir so the OpenRC service is byte-stable across restarts.
+//
+// Usage:
+//
+//	eks-konnectivity-cert -dir /var/lib/spinifex-eks/konnectivity \
+//	    -ca-cert /var/lib/rancher/k3s/server/tls/server-ca.crt \
+//	    -ca-key  /var/lib/rancher/k3s/server/tls/server-ca.key \
+//	    -cn konnectivity-server -sans 10.32.100.4,203.0.113.9,eks-toc.example
+//
+// Writes <dir>/tls.crt and <dir>/tls.key; prints the two paths (tab-separated).
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
+	caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
+	cn := flag.String("cn", "konnectivity-server", "certificate common name")
+	sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
+	flag.Parse()
+
+	if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
+		fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
+	}
+	dns, ips := splitSANs(*sansCSV)
+	if len(dns) == 0 && len(ips) == 0 {
+		fatal("eks-konnectivity-cert: -sans lists no usable names")
+	}
+
+	certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
+	if err != nil {
+		fatal("eks-konnectivity-cert: " + err.Error())
+	}
+	fmt.Printf("%s\t%s\n", certPath, keyPath)
+}
+
+// ensureCert returns the cached cert/key paths under dir, minting and persisting
+// a fresh CA-signed pair when none is present, readable, and still SAN-current.
+func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if certCoversSANs(certPath, keyPath, dns, ips) {
+		return certPath, keyPath, nil
+	}
+
+	caCert, caKey, err := loadCA(caCertPath, caKeyPath)
+	if err != nil {
+		return "", "", err
+	}
+	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	if err != nil {
+		return "", "", err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPath, keyPath, nil
+}
+
+// certCoversSANs reports whether the cached cert exists, parses, has a key, and
+// already carries every requested SAN — so a cert minted before the endpoint
+// set changed is re-issued rather than served stale.
+func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
+	c, err := os.ReadFile(certPath)
+	if err != nil {
+		return false
+	}
+	if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
+		return false
+	}
+	der := decodeFirst(c)
+	if der == nil {
+		return false
+	}
+	crt, err := x509.ParseCertificate(der)
+	if err != nil {
+		return false
+	}
+	have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
+	for _, n := range crt.DNSNames {
+		have[n] = struct{}{}
+	}
+	for _, ip := range crt.IPAddresses {
+		have[ip.String()] = struct{}{}
+	}
+	for _, n := range dns {
+		if _, ok := have[n]; !ok {
+			return false
+		}
+	}
+	for _, ip := range ips {
+		if _, ok := have[ip.String()]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
+func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	certPEM, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
+	}
+	der := decodeFirst(certPEM)
+	if der == nil {
+		return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
+	}
+	caCert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parse ca cert: %w", err)
+	}
+
+	keyPEM, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
+	}
+	keyDER := decodeFirst(keyPEM)
+	if keyDER == nil {
+		return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
+	}
+	caKey, err := parseECKey(keyDER)
+	if err != nil {
+		return nil, nil, err
+	}
+	return caCert, caKey, nil
+}
+
+// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
+func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
+	if k, err := x509.ParseECPrivateKey(der); err == nil {
+		return k, nil
+	}
+	k8, err := x509.ParsePKCS8PrivateKey(der)
+	if err != nil {
+		return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
+	}
+	ec, ok := k8.(*ecdsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
+	}
+	return ec, nil
+}
+
+// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
+func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
+// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
+func splitSANs(s string) (dns []string, ips []net.IP) {
+	for p := range strings.SplitSeq(s, ",") {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		if ip := net.ParseIP(p); ip != nil {
+			ips = append(ips, ip)
+			continue
+		}
+		dns = append(dns, p)
+	}
+	return dns, ips
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,153 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
+// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
+func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
+	t.Helper()
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen ca key: %v", err)
+	}
+	tmpl := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "k3s-server-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatalf("create ca cert: %v", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		t.Fatalf("marshal ca key: %v", err)
+	}
+	certPath = filepath.Join(dir, "server-ca.crt")
+	keyPath = filepath.Join(dir, "server-ca.key")
+	if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		t.Fatalf("write ca cert: %v", err)
+	}
+	if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
+		t.Fatalf("write ca key: %v", err)
+	}
+	return certPath, keyPath
+}
+
+func leafFrom(t *testing.T, certPath string) *x509.Certificate {
+	t.Helper()
+	b, err := os.ReadFile(certPath)
+	if err != nil {
+		t.Fatalf("read leaf: %v", err)
+	}
+	crt, err := x509.ParseCertificate(decodeFirst(b))
+	if err != nil {
+		t.Fatalf("parse leaf: %v", err)
+	}
+	return crt
+}
+
+func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
+	certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if _, err := os.Stat(keyPath); err != nil {
+		t.Fatalf("key not written: %v", err)
+	}
+
+	leaf := leafFrom(t, certPath)
+	if leaf.IsCA {
+		t.Error("konnectivity serving cert must be a leaf, not a CA")
+	}
+	// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
+	// so the IP must be a SAN or the tunnel TLS handshake fails.
+	if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
+		t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
+	}
+
+	// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
+	// agent trusts, so no separate CA distribution is needed.
+	caPEM, _ := os.ReadFile(caCert)
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(caPEM) {
+		t.Fatal("append CA failed")
+	}
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   "eks-toc.example",
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("leaf does not chain to the K3s server CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,eks-toc.example")
+	c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	first, _ := os.ReadFile(c1)
+
+	// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
+	// the konnectivity-server on every restart).
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if second, _ := os.ReadFile(c1); string(first) != string(second) {
+		t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
+	}
+
+	// A new endpoint in the SAN set must force a re-issue.
+	dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
+		t.Fatalf("third ensureCert: %v", err)
+	}
+	if third, _ := os.ReadFile(c1); string(first) == string(third) {
+		t.Error("ensureCert served a stale cert that does not cover the new SAN")
+	}
+	if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
+		t.Errorf("re-issued cert missing new SAN: %v", err)
+	}
+}
+
+func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
+	dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
+	if len(dns) != 1 || dns[0] != "host.example" {
+		t.Errorf("dns = %v, want [host.example]", dns)
+	}
+	wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
+	if len(ips) != len(wantIPs) {
+		t.Fatalf("ips = %v, want %v", ips, wantIPs)
+	}
+	for i, ip := range ips {
+		if !ip.Equal(wantIPs[i]) {
+			t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
+		}
+	}
+}
@@ -18,14 +18,10 @@ import (
 	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )

-// tlsCertificate is the webhook's loopback serving certificate.
-type tlsCertificate struct {
-	cert tls.Certificate
-}
-
-func (c tlsCertificate) serverTLSConfig() *tls.Config {
+// serverTLSConfig is the webhook's loopback serving TLS config.
+func serverTLSConfig(cert tls.Certificate) *tls.Config {
 	return &tls.Config{
-		Certificates:     []tls.Certificate{c.cert},
+		Certificates:     []tls.Certificate{cert},
 		MinVersion:       tls.VersionTLS13,
 		CurvePreferences: tlsconfig.Curves,
 	}
@@ -36,30 +32,30 @@ func (c tlsCertificate) serverTLSConfig() *tls.Config {
 // across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
-func ensureServingCert(certPath, keyPath string) (tlsCertificate, []byte, error) {
+func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
 	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
 		cert, err := tls.X509KeyPair(certPEM, keyPEM)
 		if err == nil {
-			return tlsCertificate{cert: cert}, certPEM, nil
+			return cert, certPEM, nil
 		}
 		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}

 	certPEM, keyPEM, err := generateSelfSigned()
 	if err != nil {
-		return tlsCertificate{}, nil, err
+		return tls.Certificate{}, nil, err
 	}
 	if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
 	}
 	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
 	}
 	cert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
+		return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
 	}
-	return tlsCertificate{cert: cert}, certPEM, nil
+	return cert, certPEM, nil
 }

 func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
@@ -10,6 +10,9 @@
 // On startup it writes the apiserver webhook kubeconfig (with its self-signed
 // serving CA) so k3s can be pointed at it via
 // --authentication-token-webhook-config-file.
+//
+// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
 package main

 import (
@@ -56,11 +59,13 @@ func loadConfig() (config, error) {
 		keyPath:     envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
 		kubeconfig:  envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
 	}
+	// Static SigV4 creds are optional: when absent, eksgw.New signs with the
+	// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
+	// sibling helpers use. The CP VM launches on an instance profile, so
+	// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
 	switch {
 	case c.gatewayURL == "":
 		return c, errors.New("EKS_GATEWAY_URL not set")
-	case c.accessKey == "" || c.secretKey == "":
-		return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
 	case c.accountID == "":
 		return c, errors.New("EKS_ACCOUNT_ID not set")
 	case c.clusterName == "":
@@ -125,7 +130,12 @@ func run(cfg config) error {
 	})
 	mux.HandleFunc("/authenticate", authr.handle)

-	server := newServer(cfg.addr, mux, tlsCert)
+	server := &http.Server{
+		Addr:              cfg.addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		TLSConfig:         serverTLSConfig(tlsCert),
+	}
 	slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
 	if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("serve: %w", err)
@@ -214,14 +224,3 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
 		slog.Error("encode TokenReview response", "err", err)
 	}
 }
-
-// newServer builds the loopback TLS server. Split out so tests can construct it
-// without binding a port.
-func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
-	return &http.Server{
-		Addr:              addr,
-		Handler:           handler,
-		ReadHeaderTimeout: 5 * time.Second,
-		TLSConfig:         cert.serverTLSConfig(),
-	}
-}
@@ -99,7 +99,7 @@ func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
 	c1, pem1, err := ensureServingCert(certPath, keyPath)
 	require.NoError(t, err)
 	require.NotEmpty(t, pem1)
-	require.NotNil(t, c1.serverTLSConfig())
+	require.NotNil(t, serverTLSConfig(c1))

 	// Second call reuses the persisted pair (same cert bytes).
 	_, pem2, err := ensureServingCert(certPath, keyPath)
@@ -121,3 +121,41 @@ func TestWriteAPIServerKubeconfig(t *testing.T) {
 	assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
 	assert.Contains(t, body, "current-context: webhook")
 }
+
+// The CP VM launches on an instance profile, so cloud-init omits the static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
+// must accept absent static keys; gating on them aborts every IMDS-mode boot,
+// leaving the apiserver webhook kubeconfig unwritten.
+func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
+	setEnv := func(t *testing.T, kv map[string]string) {
+		t.Helper()
+		for _, k := range []string{
+			"EKS_GATEWAY_URL", "EKS_ACCESS_KEY", "EKS_SECRET_KEY",
+			"EKS_ACCOUNT_ID", "EKS_CLUSTER_NAME",
+		} {
+			t.Setenv(k, "")
+		}
+		for k, v := range kv {
+			t.Setenv(k, v)
+		}
+	}
+
+	// IMDS mode: required vars set, static creds absent → accepted.
+	setEnv(t, map[string]string{
+		"EKS_GATEWAY_URL":  "https://gw:9999",
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	cfg, err := loadConfig()
+	require.NoError(t, err)
+	assert.Empty(t, cfg.accessKey)
+	assert.Empty(t, cfg.secretKey)
+
+	// Genuinely-required vars are still enforced.
+	setEnv(t, map[string]string{
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	_, err = loadConfig()
+	require.Error(t, err)
+}
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package autoinstall reads headless install parameters from environment
 // variables exported by spinifex-init.sh from the kernel cmdline.
 // Parameters are set in the GRUB "Headless" menu entry in grub.cfg.
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package branding centralises Mulga/Spinifex visual identity constants used
 // by the installer TUI.
 //
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package firstboot writes the oneshot systemd service and configuration that
 // completes Spinifex provisioning on the first real boot after installation.
 package firstboot
@@ -91,6 +74,31 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}

+	// Pre-start OVS and OVN central so their databases are initialised before
+	// setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
+	// longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
+	// here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
+	// the moment it starts — no races, no timeout failures.
+	ovnPrestart := `systemctl start openvswitch-switch
+systemctl start ovn-central
+echo "Waiting for OVN NB DB to initialise..."
+for _i in $(seq 1 120); do
+    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+        echo "OVN NB DB ready (${_i}s)"
+        break
+    fi
+    sleep 1
+done`
+
+	// When a provisioning controller owns formation (SkipFormation) it also owns
+	// the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must
+	// not start a standalone ovn-central here: the single-node .db it would leave
+	// blocks ovn-ctl's create/join (which require a clean DB). Defer both.
+	if cfg.SkipFormation {
+		ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"`
+		setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"`
+	}
+
 	callbackBlock := ""
 	if cfg.InstallCallback != "" {
 		callbackBlock = fmt.Sprintf(
@@ -123,21 +131,7 @@ fi
 # Set hostname
 hostnamectl set-hostname %s

-# Pre-start OVS and OVN central so their databases are initialised before
-# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
-# longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
-# here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
-# the moment it starts — no races, no timeout failures.
-systemctl start openvswitch-switch
-systemctl start ovn-central
-echo "Waiting for OVN NB DB to initialise..."
-for _i in $(seq 1 120); do
-    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
-        echo "OVN NB DB ready (${_i}s)"
-        break
-    fi
-    sleep 1
-done
+%s

 # Create default nameservers if DHCP server returns blank
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
@@ -229,7 +223,7 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
        echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
    fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)
+`, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock)

 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)
@@ -66,6 +66,54 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	}
 }

+func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script should run setup-ovn --management when firstboot owns formation")
+	}
+	if !strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script should pre-start ovn-central when firstboot owns formation")
+	}
+}
+
+func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script must not run setup-ovn --management when a controller owns OVN")
+	}
+	if strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script must not pre-start ovn-central when a controller owns OVN")
+	}
+	if !strings.Contains(content, "setup-ovn deferred") {
+		t.Error("script should note setup-ovn is deferred under SkipFormation")
+	}
+}
+
+func readScript(t *testing.T, root string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	return string(b)
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package install performs the disk installation steps after the UI has
 // collected configuration.
 package install
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 package install

 import (
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main

 import (
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package systemd writes systemd unit files and drop-ins into an installed
 // system root during the Spinifex ISO installation process.
 package systemd
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package ui presents the interactive installer TUI using bubbletea and lipgloss.
 package ui

@@ -49,12 +49,13 @@ func main() {
 		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
 		os.Exit(1)
 	}
-	if accessKey == "" || secretKey == "" {
-		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
-		os.Exit(1)
+	// Empty keys select IMDS instance-role credentials (the SDK default chain
+	// falls back to the in-VM IMDS endpoint); static keys sign as-is.
+	credSource := "imds"
+	if accessKey != "" && secretKey != "" {
+		credSource = "static"
 	}
-
-	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL, "credSource", credSource)

 	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
 	if err != nil {
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCABakeCmd_Args(t *testing.T) {
+	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
+	args := cmd.Args
+
+	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
+	assert.Contains(t, args, "-a")
+	assert.Contains(t, args, "/img/disk.raw")
+	assert.Contains(t, args, "--upload")
+	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
+	assert.Contains(t, args, "--run-command")
+
+	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
+	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
+	assert.Contains(t, args, "--no-selinux-relabel")
+
+	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
+	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
+	// ca-certificates bundle when no updater is present. The RHEL branch relabels
+	// only the touched trust paths so the cert is labelled without a full relabel.
+	joined := strings.Join(args, " ")
+	assert.Contains(t, joined, "update-ca-certificates")
+	assert.Contains(t, joined, "update-ca-trust")
+	assert.Contains(t, joined, "restorecon")
+	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
+}
+
+func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
+	defer func() { caBakeRunner = orig }()
+
+	// A missing CA must not invoke virt-customize and must not panic.
+	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
+	assert.False(t, called, "runner must not be called when the CA is absent")
+}
+
+func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
+	caPath := filepath.Join(t.TempDir(), "ca.pem")
+	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
+
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) {
+		called = true
+		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
+	}
+	defer func() { caBakeRunner = orig }()
+
+	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
+	bakeCACertIntoImage("/img/disk.raw", caPath)
+	assert.True(t, called, "runner must be invoked when the CA is present")
+}
@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+	"github.com/spf13/cobra"
+)
+
+var adminEksCmd = &cobra.Command{
+	Use:   "eks",
+	Short: "Manage EKS control-plane disaster recovery",
+}
+
+var adminEksRestoreSnapshotCmd = &cobra.Command{
+	Use:   "restore-snapshot",
+	Short: "Rebuild a single-CP cluster's control plane from an etcd snapshot",
+	Long: `restore-snapshot drives the single-CP total-loss DR path: it launches a fresh
+control-plane VM as a cluster-init seed, directs its boot-time recovery agent to
+run k3s server --cluster-reset (restoring the given etcd snapshot, or the latest
+one found in predastore when --snapshot is omitted), and re-points the cluster
+NLB target groups at the new CP.
+
+Only single-CP clusters are supported. An HA cluster has a potentially
+surviving etcd quorum and should be recovered via quorum reformation instead
+of this destructive single-node rebuild.`,
+	Run: runAdminEksRestoreSnapshot,
+}
+
+func init() {
+	adminCmd.AddCommand(adminEksCmd)
+	adminEksCmd.AddCommand(adminEksRestoreSnapshotCmd)
+
+	adminEksRestoreSnapshotCmd.Flags().String("cluster", "", "Cluster name to restore (required)")
+	adminEksRestoreSnapshotCmd.Flags().String("snapshot", "", "Etcd snapshot key to restore; defaults to the latest snapshot found in predastore")
+	adminEksRestoreSnapshotCmd.Flags().String("account", "", "Account ID that owns the cluster; defaults to the bootstrap account")
+	_ = adminEksRestoreSnapshotCmd.MarkFlagRequired("cluster")
+}
+
+func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
+	clusterName, _ := cmd.Flags().GetString("cluster")
+	snapshot, _ := cmd.Flags().GetString("snapshot")
+	accountID, _ := cmd.Flags().GetString("account")
+	if accountID == "" {
+		accountID = admin.DefaultAccountID()
+	}
+
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	svc := handlers_eks.NewNATSEKSService(nc)
+	// The server side launches a VM (30-60s) then fences the old CP with retries,
+	// so the RPC needs generous headroom over a bare launch.
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	fmt.Printf("Restoring control plane for cluster %q (account %s)...\n", clusterName, accountID)
+	out, err := svc.RestoreSnapshot(ctx, &handlers_eks.RestoreSnapshotInput{
+		ClusterName: clusterName,
+		Snapshot:    snapshot,
+	}, accountID)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
+	fmt.Printf("Status: %s\n", out.Status)
+}
@@ -0,0 +1,247 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `external_mode = "nat"`)
+	assert.Contains(t, content, `bridge_mode = "nat"`)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `gateway     = "100.127.0.1"`)
+	assert.NotContains(t, content, "range_start", "nat mode has no public IP range")
+}
+
+func TestSpinifexTomlTemplate_GwLrpRange(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat", BridgeMode: "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+			GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254",
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `gw_lrp_range_start = "100.127.0.16"`)
+	assert.Contains(t, content, `gw_lrp_range_end   = "100.127.0.254"`)
+}
+
+func TestSpinifexTomlTemplate_GwLrpRangeOmittedWhenUnset(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "pool", ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "gw_lrp_range", "unset range must not render")
+}
+
+func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode:  "pool",
+		ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "bridge_mode", "bridged modes keep runtime auto-detect")
+}
+
+func TestIsNonBridgeableUplink(t *testing.T) {
+	for _, name := range []string{"wlan0", "wlp3s0", "wlx001122334455", "wwan0", "wwp0s20f0u6", "ppp0"} {
+		assert.True(t, isNonBridgeableUplink(name), name)
+	}
+	for _, name := range []string{"eth0", "enp0s3", "eno1", "br-wan", "veth-wan-br"} {
+		assert.False(t, isNonBridgeableUplink(name), name)
+	}
+}
+
+func TestBridgeModeFor(t *testing.T) {
+	assert.Equal(t, "nat", bridgeModeFor("nat"))
+	assert.Equal(t, "", bridgeModeFor("pool"))
+	assert.Equal(t, "", bridgeModeFor(""))
+}
+
+func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24},
+			{
+				Name: "wan", Source: "dhcp", BindBridge: "wlan0",
+				DHCPMAC: "interface", PrefixLen: 24,
+			},
+		},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `name        = "wan"`)
+	assert.Contains(t, content, `source      = "dhcp"`)
+	assert.Contains(t, content, `bind_bridge = "wlan0"`)
+	assert.Contains(t, content, `dhcp_mac    = "interface"`)
+	transitIdx := strings.Index(content, `name        = "nat-transit"`)
+	wanIdx := strings.Index(content, `name        = "wan"`)
+	assert.Less(t, transitIdx, wanIdx, "transit pool must render first (IGW allocator picks it)")
+}
+
+func TestResolvePublicPoolFlags(t *testing.T) {
+	tests := []struct {
+		name              string
+		source, poolRange string
+		bindBridge, gw    string
+		defaultBridge     string
+		wantSource        string
+		wantStart         string
+		wantEnd           string
+		wantBridge        string
+		wantErr           string
+	}{
+		{
+			name: "static pool range", source: "", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1",
+			wantSource: "static", wantStart: "192.168.1.150", wantEnd: "192.168.1.250",
+		},
+		{
+			name: "dhcp defaults bind bridge", source: "dhcp", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "wlan0",
+		},
+		{
+			name: "dhcp explicit bind bridge", source: "dhcp", bindBridge: "br-wan", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "br-wan",
+		},
+		{
+			name: "dhcp rejects pool range", source: "dhcp", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-pool not allowed",
+		},
+		{
+			name: "static requires gateway", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-gateway is required",
+		},
+		{
+			name: "static rejects bind bridge", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1", bindBridge: "br-wan",
+			wantErr: "--external-bind-bridge only valid",
+		},
+		{
+			name: "static requires range", source: "static", gw: "192.168.1.1",
+			wantErr: "--external-pool is required",
+		},
+		{
+			name: "bad range", poolRange: "not-a-range", gw: "192.168.1.1",
+			wantErr: "start-end IPs",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src, start, end, bb, err := resolvePublicPoolFlags(tt.source, tt.poolRange, tt.bindBridge, tt.gw, tt.defaultBridge)
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantSource, src)
+			assert.Equal(t, tt.wantStart, start)
+			assert.Equal(t, tt.wantEnd, end)
+			assert.Equal(t, tt.wantBridge, bb)
+		})
+	}
+}
@@ -9,9 +9,9 @@ import (
 	"strings"
 	"testing"

-	s3 "github.com/mulgadc/predastore/s3"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	toml "github.com/pelletier/go-toml/v2"
 	"github.com/spf13/cobra"
@@ -256,23 +256,6 @@ func TestWritePredastoreEncryptionKey(t *testing.T) {
 	assert.NotEqual(t, key1, key2, "per-node keys must differ")
 }

-// The viperblock key gates at-rest encryption for every volume on a fresh
-// install. masterkey.LoadShared rejects anything that isn't exactly 32 bytes at
-// 0640-or-stricter, so a regression in size/mode would silently fall back to
-// cleartext (empty path) or fail service startup.
-func TestWriteViperblockEncryptionKey(t *testing.T) {
-	configDir := t.TempDir()
-
-	keyPath, err := writeViperblockEncryptionKey(configDir)
-	require.NoError(t, err)
-	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
-
-	info, err := os.Stat(keyPath)
-	require.NoError(t, err)
-	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
-	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
-}
-
 // Joiners persist the leader's shared key via saveViperblockEncryptionKey
 // against a configDir with no viperblock/ subdir yet. The bytes round-trip
 // verbatim — the whole cluster shares one key, so any mutation would orphan
@@ -293,6 +276,181 @@ func TestSaveViperblockEncryptionKey_RoundTrip(t *testing.T) {
 	assert.Equal(t, key, got, "shared key must be written verbatim")
 }

+// A re-init must not rotate the per-node predastore key: rotating it would orphan
+// every fragment already sealed under the old key. The helper preserves an
+// existing key byte-for-byte.
+func TestWritePredastoreEncryptionKey_PreservesExisting(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	orig, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+
+	keyPath2, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, keyPath, keyPath2)
+	after, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.Equal(t, orig, after, "existing predastore key must be preserved on re-init")
+}
+
+// ensureViperblockEncryptionKey generates a shared 32-byte 0600 key on a fresh
+// dir and returns bytes that match the on-disk file; a second call preserves it
+// so a re-init keeps every encrypted volume readable.
+func TestEnsureViperblockEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	key, keyPath, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
+	assert.Len(t, key, 32, "viperblock key must be 32 bytes")
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
+
+	onDisk, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, key, onDisk, "returned bytes must match the file")
+
+	key2, _, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, key, key2, "existing viperblock key must be preserved on re-init")
+}
+
+// A present-but-corrupt viperblock key must fail loud rather than regenerate,
+// which would orphan every already-encrypted volume in the cluster.
+func TestEnsureViperblockEncryptionKey_CorruptFailsLoud(t *testing.T) {
+	configDir := t.TempDir()
+	keyDir := filepath.Join(configDir, "viperblock")
+	require.NoError(t, os.MkdirAll(keyDir, 0750))
+	keyPath := filepath.Join(keyDir, "encryption.key")
+	require.NoError(t, os.WriteFile(keyPath, []byte("bad"), 0600))
+
+	_, _, err := ensureViperblockEncryptionKey(configDir)
+	require.Error(t, err, "a truncated viperblock key must error, not regenerate")
+
+	after, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("bad"), after, "corrupt viperblock key must not be overwritten")
+}
+
+// ensureMasterKey generates a fresh in-memory key without touching disk (the
+// bootstrap writer persists it), and preserves an existing on-disk key so a
+// re-init never rotates the key that encrypts IAM secrets in NATS KV.
+func TestEnsureMasterKey(t *testing.T) {
+	t.Run("FreshGeneratesWithoutWriting", func(t *testing.T) {
+		configDir := t.TempDir()
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.False(t, existed, "no master.key present yet")
+		assert.Len(t, key, 32)
+		assert.NoFileExists(t, filepath.Join(configDir, "master.key"),
+			"ensureMasterKey must not persist on the fresh path")
+	})
+
+	t.Run("PreservesExisting", func(t *testing.T) {
+		configDir := t.TempDir()
+		seed := make([]byte, 32)
+		for i := range seed {
+			seed[i] = byte(200 - i)
+		}
+		require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), seed))
+
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.True(t, existed, "existing master.key must be detected")
+		assert.Equal(t, seed, key, "existing master key must be loaded verbatim")
+	})
+
+	// A present-but-corrupt master.key must fail loud, never be silently
+	// regenerated: rotating it would orphan every IAM secret in NATS KV.
+	t.Run("CorruptFailsLoud", func(t *testing.T) {
+		configDir := t.TempDir()
+		keyPath := filepath.Join(configDir, "master.key")
+		require.NoError(t, os.WriteFile(keyPath, []byte("too-short"), 0600))
+
+		_, _, err := ensureMasterKey(configDir)
+		require.Error(t, err, "a truncated master.key must error, not regenerate")
+
+		after, err := os.ReadFile(keyPath)
+		require.NoError(t, err)
+		assert.Equal(t, []byte("too-short"), after, "corrupt master key must not be overwritten")
+	})
+}
+
+// The identity bundle is preserved holistically on re-init: the load helpers
+// recover the exact system credentials and never rewrite bootstrap.json. Admin
+// credentials are not recovered from disk — awsgw consumes and deletes
+// bootstrap.json after first boot, so the preserve path must not read it back.
+func TestPreservedIdentityBundle(t *testing.T) {
+	configDir := t.TempDir()
+	bootstrapDir := filepath.Join(configDir, "awsgw")
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	const (
+		sysAccess   = "AKIASYSTEM0000000000"
+		sysSecret   = "system-secret-value"
+		adminAccess = "AKIAADMIN00000000000"
+		adminSecret = "admin-secret-value"
+		accountID   = "123456789012"
+	)
+	require.NoError(t, writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey,
+		sysAccess, sysSecret, accountID, adminAccess, adminSecret))
+	require.NoError(t, writeSystemCredentials(configDir, sysAccess, sysSecret))
+
+	bootstrapPath := filepath.Join(bootstrapDir, "bootstrap.json")
+	bootstrapBefore, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+
+	// master.key round-trips and is flagged as pre-existing.
+	loadedKey, existed, err := ensureMasterKey(configDir)
+	require.NoError(t, err)
+	assert.True(t, existed)
+	assert.Equal(t, masterKey, loadedKey)
+
+	// System credentials come back verbatim (they must match the NATS KV seed).
+	gotSysAccess, gotSysSecret, err := loadSystemCredentials(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, sysAccess, gotSysAccess)
+	assert.Equal(t, sysSecret, gotSysSecret)
+
+	// The preserve helpers must not rewrite the seed file.
+	bootstrapAfter, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+	assert.Equal(t, bootstrapBefore, bootstrapAfter, "bootstrap.json must not be rewritten on re-init")
+}
+
+// loadSystemCredentials surfaces a clear error when the file is absent rather
+// than returning empty credentials that would silently break SigV4 auth.
+func TestLoadSystemCredentials_Missing(t *testing.T) {
+	_, _, err := loadSystemCredentials(t.TempDir())
+	require.Error(t, err)
+}
+
+// A present-but-empty or malformed system-credentials.json must error rather
+// than yield blank credentials that would silently break SigV4 between services.
+func TestLoadSystemCredentials_Invalid(t *testing.T) {
+	t.Run("EmptyValues", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte(`{"access_key_id":"","secret_access_key":""}`), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+
+	t.Run("MalformedJSON", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte("not json"), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+}
+
 // A fresh install must render encryption_key_file so viperblockd enables
 // at-rest encryption; an empty field (upgrade / legacy) must omit the line
 // entirely so the volume stays in cleartext legacy mode.
@@ -383,20 +541,19 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "spinifex.toml")
 	settings := admin.ConfigSettings{
-		Node:           "node1",
-		Az:             "ap-southeast-2a",
-		Port:           "4432",
-		Region:         "ap-southeast-2",
-		BindIP:         "10.11.12.1",
-		ConfigDir:      dir,
-		OVNNBAddr:      "tcp:127.0.0.1:6641",
-		OVNSBAddr:      "tcp:127.0.0.1:6642",
-		ExternalMode:   "pool",
-		ExternalIface:  "eth0",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		Node:          "node1",
+		Az:            "ap-southeast-2a",
+		Port:          "4432",
+		Region:        "ap-southeast-2",
+		BindIP:        "10.11.12.1",
+		ConfigDir:     dir,
+		OVNNBAddr:     "tcp:127.0.0.1:6641",
+		OVNSBAddr:     "tcp:127.0.0.1:6642",
+		ExternalMode:  "pool",
+		ExternalIface: "eth0",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -425,12 +582,11 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 		OVNSBAddr:     "tcp:127.0.0.1:6642",
 		ExternalMode:  "pool",
 		ExternalIface: "eth0",
-		PoolName:      "wan",
-		PoolSource:    "static",
-		PoolStart:     "192.168.1.150",
-		PoolEnd:       "192.168.1.250",
-		PoolGateway:   "192.168.1.1",
-		PoolPrefixLen: 24,
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -455,8 +611,9 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 		PoolPrefixLen:  24,
 	}
 	applyNetworkConfig(settings, nc)
-	assert.Equal(t, "dhcp", settings.PoolSource)
-	assert.Equal(t, "br-wan", settings.PoolBindBridge)
+	require.Len(t, settings.Pools, 1)
+	assert.Equal(t, "dhcp", settings.Pools[0].Source)
+	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
 }

 func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
@@ -503,7 +660,7 @@ func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
 		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok", CompactionIntervalSeconds: interval,
 	})
 	assert.Contains(t, single, "[compaction]")
-	var singleCfg s3.Config
+	var singleCfg compactionConfig
 	require.NoError(t, toml.Unmarshal([]byte(single), &singleCfg))
 	assert.Equal(t, interval, singleCfg.Compaction.IntervalSeconds)

@@ -511,11 +668,20 @@ func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
 		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", interval,
 	)
 	require.NoError(t, err)
-	var multiCfg s3.Config
+	var multiCfg compactionConfig
 	require.NoError(t, toml.Unmarshal([]byte(multi), &multiCfg))
 	assert.Equal(t, interval, multiCfg.Compaction.IntervalSeconds)
 }

+// compactionConfig parses just the rendered [compaction] block. Predastore's
+// s3.Config dropped its Compaction field, so this asserts the spinifex template
+// still emits a valid, parseable block carrying the interval.
+type compactionConfig struct {
+	Compaction struct {
+		IntervalSeconds int `toml:"interval_seconds"`
+	} `toml:"compaction"`
+}
+
 // The init/join flag must register with an unset (0) default and flow into the
 // rendered config via ConfigSettings.
 func TestAdminInitFlag_CompactionIntervalReachesRenderedConfig(t *testing.T) {
@@ -123,7 +123,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 		phaseStart := time.Now()
 		fmt.Printf("[%s] Sending to %d node(s)...\n", strings.ToUpper(phase), nodeCount)

-		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, timeout)
+		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, "", timeout)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "[%s] Error: %v\n", strings.ToUpper(phase), err)
 			if !force {
@@ -161,6 +161,71 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 	fmt.Printf("Cluster shutdown complete (%s)\n", time.Since(start).Round(time.Millisecond))
 }

+// runNodeDrainLocal runs the GATE and DRAIN shutdown phases against the local
+// node only, leaving STORAGE/PERSIST/INFRA to systemd's ordered unit teardown.
+// It is the ExecStop of spinifex-shutdown.service: guests are powered down and
+// their volumes unmounted while every service is still up, so the subsequent
+// service stops never tear storage out from under a running guest.
+func runNodeDrainLocal(cmd *cobra.Command, args []string) {
+	local, _ := cmd.Flags().GetBool("local")
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	if !local {
+		fmt.Fprintln(os.Stderr, "Error: node drain currently supports only --local")
+		os.Exit(1)
+	}
+
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	node := cfg.Node
+	fmt.Printf("Draining local node %q (gate -> drain), timeout %s/phase\n", node, timeout)
+
+	for _, phase := range []string{"gate", "drain"} {
+		topic := "spinifex.cluster.shutdown." + phase
+		req := daemon.ShutdownRequest{Phase: phase, Timeout: int(timeout.Seconds())}
+		reqData, err := json.Marshal(req)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshaling request: %v\n", err)
+			os.Exit(1)
+		}
+
+		ack, err := collectLocalShutdownACK(nc, topic, reqData, node, timeout)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[%s] %v\n", strings.ToUpper(phase), err)
+			os.Exit(1)
+		}
+		if ack.Error != "" {
+			fmt.Fprintf(os.Stderr, "[%s] %s: ERROR - %s\n", strings.ToUpper(phase), ack.Node, ack.Error)
+			os.Exit(1)
+		}
+		if len(ack.Stopped) > 0 {
+			fmt.Printf("[%s] %s: stopped %s\n", strings.ToUpper(phase), ack.Node, strings.Join(ack.Stopped, ", "))
+		} else {
+			fmt.Printf("[%s] %s: OK\n", strings.ToUpper(phase), ack.Node)
+		}
+	}
+
+	fmt.Printf("Local node %q drained; systemd may now stop storage services\n", node)
+}
+
+// collectLocalShutdownACK publishes a shutdown-phase request and returns the ACK
+// from the local node, ignoring ACKs from any other node. Returns an error if
+// the local node does not respond within timeout.
+func collectLocalShutdownACK(nc *nats.Conn, topic string, reqData []byte, node string, timeout time.Duration) (daemon.ShutdownACK, error) {
+	acks, err := collectShutdownACKs(nc, topic, reqData, 1, node, timeout)
+	if err != nil {
+		return daemon.ShutdownACK{}, err
+	}
+	if len(acks) == 0 {
+		return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
+	}
+	return acks[0], nil
+}
+
 // runClusterDrainDHCP asks every vpcd to DHCPRELEASE all external-pool leases
 // it currently holds, returning them to the upstream DHCP server. Intended for
 // the teardown path: an env reset otherwise strands held leases until TTL
@@ -235,7 +300,9 @@ func drainDHCPLeases(nc *nats.Conn, timeout time.Duration) (released, responders
 }

 // collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
-func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
+// When nodeFilter is non-empty, only ACKs from that node count toward nodeCount
+// and all others are ignored (used by the single-node local-drain path).
+func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, nodeFilter string, timeout time.Duration) ([]daemon.ShutdownACK, error) {
 	inbox := nats.NewInbox()
 	sub, err := nc.SubscribeSync(inbox)
 	if err != nil {
@@ -263,6 +330,9 @@ func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount
 		if err := json.Unmarshal(msg.Data, &ack); err != nil {
 			continue
 		}
+		if nodeFilter != "" && ack.Node != nodeFilter {
+			continue
+		}
 		acks = append(acks, ack)
 	}
 	return acks, nil
@@ -1,26 +1,12 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -152,6 +138,12 @@ func formatRoles(resp types.NodeStatusResponse) string {
 	if resp.NATSRole != "" {
 		roles = append(roles, "nats:"+resp.NATSRole)
 	}
+	if resp.OVNNBRole != "" {
+		roles = append(roles, "ovn-nb:"+resp.OVNNBRole)
+	}
+	if resp.OVNSBRole != "" {
+		roles = append(roles, "ovn-sb:"+resp.OVNSBRole)
+	}
 	if resp.PredastoreRole != "" {
 		roles = append(roles, "predastore:"+resp.PredastoreRole)
 	}
@@ -206,11 +198,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 	for name := range respondedNodes {
 		nodeSet[name] = struct{}{}
 	}
-	nodeNames := make([]string, 0, len(nodeSet))
-	for name := range nodeSet {
-		nodeNames = append(nodeNames, name)
-	}
-	sort.Strings(nodeNames)
+	nodeNames := slices.Sorted(maps.Keys(nodeSet))

 	for _, name := range nodeNames {
 		nodeCfg := cfg.Nodes[name]
@@ -291,7 +279,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 	})

 	tableData := pterm.TableData{
-		{"INSTANCE", "STATUS", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
+		{"INSTANCE", "STATUS", "HEALTH", "CRASHES", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
 	}

 	for _, v := range allVMs {
@@ -299,9 +287,15 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 		if v.LaunchTime > 0 {
 			age = formatUptime(time.Now().Unix() - v.LaunchTime)
 		}
+		health := v.Health
+		if health == "" {
+			health = "-"
+		}
 		tableData = append(tableData, []string{
 			v.InstanceID,
 			v.Status,
+			health,
+			strconv.Itoa(v.CrashCount),
 			v.InstanceType,
 			strconv.Itoa(v.VCPU),
 			formatMemGB(v.MemoryGB),
@@ -38,6 +38,26 @@ func TestFormatRoles(t *testing.T) {
 			resp: types.NodeStatusResponse{NATSRole: "leader", PredastoreRole: "leader"},
 			want: "nats:leader,predastore:leader",
 		},
+		{
+			name: "ovn roles only",
+			resp: types.NodeStatusResponse{OVNNBRole: "leader", OVNSBRole: "follower"},
+			want: "ovn-nb:leader,ovn-sb:follower",
+		},
+		{
+			name: "all four roles ordered",
+			resp: types.NodeStatusResponse{
+				NATSRole:       "leader",
+				OVNNBRole:      "leader",
+				OVNSBRole:      "follower",
+				PredastoreRole: "follower",
+			},
+			want: "nats:leader,ovn-nb:leader,ovn-sb:follower,predastore:follower",
+		},
+		{
+			name: "ovn-nb leader with nats, sb absent",
+			resp: types.NodeStatusResponse{NATSRole: "follower", OVNNBRole: "leader"},
+			want: "nats:follower,ovn-nb:leader",
+		},
 	}

 	for _, tt := range tests {
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -1,32 +1,20 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"os"
 	"strings"

 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/network/external"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/service"
 	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
+	"github.com/mulgadc/spinifex/spinifex/services/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
 	"github.com/mulgadc/spinifex/spinifex/services/viperblockd"
 	"github.com/mulgadc/spinifex/spinifex/vpcd"
@@ -39,6 +27,28 @@ var serviceCmd = &cobra.Command{
 	Short: "Manage Spinifex services",
 }

+// initTelemetry installs the JSON slog default (trace-stamping) and the OTel
+// providers for a service process. The returned func flushes exporters and
+// must be deferred; with no OTLP endpoint configured both are no-ops.
+func initTelemetry(serviceName string, debug bool) func() {
+	level := slog.LevelInfo
+	if debug {
+		level = slog.LevelDebug
+	}
+	otelsetup.SetDefaultJSONLogger(level)
+
+	shutdown, err := otelsetup.Init(context.Background(), serviceName)
+	if err != nil {
+		slog.Warn("otel init", "service", serviceName, "error", err)
+		return func() {}
+	}
+	return func() {
+		if err := shutdown(context.Background()); err != nil {
+			slog.Warn("otel shutdown", "service", serviceName, "error", err)
+		}
+	}
+}
+
 var predastoreCmd = &cobra.Command{
 	Use:   "predastore",
 	Short: "Manage the predastore service",
@@ -126,6 +136,8 @@ var predastoreStartCmd = &cobra.Command{
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")

+		defer initTelemetry("predastore", debug)()
+
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
@@ -288,6 +300,8 @@ var viperblockStartCmd = &cobra.Command{
 			encryptionKeyFile = envKey
 		}

+		defer initTelemetry("viperblockd", false)()
+
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -366,6 +380,8 @@ var natsStartCmd = &cobra.Command{

 		cfgFile := viper.GetString("config")

+		defer initTelemetry("nats", debug)()
+
 		service, err := service.New("nats", &nats.Config{
 			ConfigFile: cfgFile,
 			Port:       port,
@@ -459,6 +475,8 @@ var spinifexStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("spinifex-daemon", false)()
+
 		svc, err := service.New("spinifex", clusterConfig)

 		if err != nil {
@@ -564,6 +582,8 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("awsgw", viper.GetBool("debug"))()
+
 		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)

@@ -621,6 +641,8 @@ var spinifexUIStartCmd = &cobra.Command{
 		tlsCert := viper.GetString("spinifex-ui-tls-cert")
 		tlsKey := viper.GetString("spinifex-ui-tls-key")

+		defer initTelemetry("spinifex-ui", false)()
+
 		svc, err := service.New("spinifex-ui", &spinifexui.Config{
 			Port:    port,
 			Host:    host,
@@ -730,12 +752,13 @@ var vpcdStartCmd = &cobra.Command{
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]

 		// Map cluster-wide external pools to vpcd config
-		var extPools []vpcd.ExternalPoolConfig
+		var extPools []external.ExternalPoolConfig
 		for _, p := range clusterConfig.Network.ExternalPools {
-			extPools = append(extPools, vpcd.ExternalPoolConfig{
+			extPools = append(extPools, external.ExternalPoolConfig{
 				Name:            p.Name,
 				Source:          p.Source,
 				BindBridge:      p.BindBridge,
+				DHCPMAC:         p.DHCPMAC,
 				RangeStart:      p.RangeStart,
 				RangeEnd:        p.RangeEnd,
 				Gateway:         p.Gateway,
@@ -767,6 +790,8 @@ var vpcdStartCmd = &cobra.Command{
 			nodeConfig.BaseDir = baseDir
 		}

+		defer initTelemetry("vpcd", false)()
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -781,6 +806,7 @@ var vpcdStartCmd = &cobra.Command{
 			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
 			BridgeMode:        nodeConfig.VPCD.BridgeMode,
 			AZ:                nodeConfig.AZ,
+			NATExemptCIDRs:    clusterConfig.Network.NATExemptCIDRs,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)
@@ -823,6 +849,77 @@ var vpcdStatusCmd = &cobra.Command{
 	},
 }

+var qmpCollectorCmd = &cobra.Command{
+	Use:   "qmp-collector",
+	Short: "Manage the qmp-collector (guest metrics) service",
+}
+
+var qmpCollectorStartCmd = &cobra.Command{
+	Use:   "start",
+	Short: "Start the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Starting qmp-collector service...")
+
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			fmt.Println("Config file is not set")
+			return
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			fmt.Println("Error loading config file:", err)
+			return
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+
+		defer initTelemetry("qmp-collector", false)()
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{
+			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
+			NatsCACert: nodeConfig.NATS.CACert,
+			BaseDir:    nodeConfig.BaseDir,
+			NodeName:   clusterConfig.Node,
+		})
+		if err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			return
+		}
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service started")
+	},
+}
+
+var qmpCollectorStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Stopping qmp-collector service...")
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
+		if err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			return
+		}
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service stopped")
+	},
+}
+
+var qmpCollectorStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Get status of the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("qmp-collector service status: ...")
+	},
+}
+
 func init() {
 	viper.SetEnvPrefix("SPINIFEX") // Prefix for environment variables
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -873,11 +970,6 @@ func init() {
 	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
 	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))

-	// Predastore Backend
-	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
-	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
-	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
-
 	// Predastore Node ID. Default -1 is dev mode (launch every configured
 	// QUIC node in-process). Production deployments set this to the node's
 	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
@@ -1022,4 +1114,11 @@ func init() {
 	vpcdCmd.AddCommand(vpcdStartCmd)
 	vpcdCmd.AddCommand(vpcdStopCmd)
 	vpcdCmd.AddCommand(vpcdStatusCmd)
+
+	// qmp-collector
+	serviceCmd.AddCommand(qmpCollectorCmd)
+
+	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }
@@ -0,0 +1,24 @@
+package cmd
+
+import "testing"
+
+func TestInitTelemetryNoEndpoint(t *testing.T) {
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+
+	tests := []struct {
+		name  string
+		debug bool
+	}{
+		{"info level", false},
+		{"debug level", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			flush := initTelemetry("test-svc", tt.debug)
+			if flush == nil {
+				t.Fatal("initTelemetry returned nil flush func")
+			}
+			flush()
+		})
+	}
+}
@@ -1,5 +1,5 @@
 # AWS Gateway configuration file in TOML format.
-version = "2"
+version = "3"
 region = "{{.Region}}"

 # Enable verbose debugging
@@ -8,8 +8,8 @@ debug = false
 # API request throttling (per-account, per-action token bucket)
 [ratelimit]
 enabled = false
-rate = 20           # Default sustained requests per second per account per action
-burst = 100         # Default maximum burst capacity
+rate = 20 # Default sustained requests per second per account per action
+burst = 100 # Default maximum burst capacity

 [ratelimit.action.RunInstances]
 rate = 2
@@ -26,3 +26,12 @@ burst = 40
 [ratelimit.action.StopInstances]
 rate = 2
 burst = 40
+
+# Per-account service quotas (default off)
+[quota]
+enabled = false
+vcpus = 8
+vpcs = 8
+subnets = 16
+eips = 4
+volumes_gib = 100
@@ -16,28 +16,37 @@ internal_suffix = "spinifex.internal"
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
+{{- range .Pools}}

 [[network.external_pools]]
-name        = "{{.PoolName}}"
-{{- if .PoolSource}}
-source      = "{{.PoolSource}}"
+name        = "{{.Name}}"
+{{- if .Source}}
+source      = "{{.Source}}"
 {{- end}}
-{{- if .PoolBindBridge}}
-bind_bridge = "{{.PoolBindBridge}}"
+{{- if .BindBridge}}
+bind_bridge = "{{.BindBridge}}"
 {{- end}}
-{{- if .PoolStart}}
-range_start = "{{.PoolStart}}"
-range_end   = "{{.PoolEnd}}"
+{{- if .DHCPMAC}}
+dhcp_mac    = "{{.DHCPMAC}}"
 {{- end}}
-{{- if .PoolGateway}}
-gateway     = "{{.PoolGateway}}"
+{{- if .Start}}
+range_start = "{{.Start}}"
+range_end   = "{{.End}}"
 {{- end}}
-{{- if .PoolGatewayIP}}
-gateway_ip  = "{{.PoolGatewayIP}}"
+{{- if .Gateway}}
+gateway     = "{{.Gateway}}"
+{{- end}}
+{{- if .GatewayIP}}
+gateway_ip  = "{{.GatewayIP}}"
+{{- end}}
+{{- if .GwLrpRangeStart}}
+gw_lrp_range_start = "{{.GwLrpRangeStart}}"
+gw_lrp_range_end   = "{{.GwLrpRangeEnd}}"
+{{- end}}
+prefix_len  = {{if .PrefixLen}}{{.PrefixLen}}{{else}}24{{end}}
+{{- if .DNSServers}}
+dns_servers = [{{range $i, $s := .DNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
-prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
-{{- if .PoolDNSServers}}
-dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
 {{- end}}
 {{- if .BootstrapVpcId}}
@@ -107,6 +116,9 @@ ovn_sb_addr = "{{.OVNSBAddr}}"
 {{- if .ExternalIface}}
 external_interface = "{{.ExternalIface}}"
 {{- end}}
+{{- if .BridgeMode}}
+bridge_mode = "{{.BridgeMode}}"
+{{- end}}

 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"
@@ -1,26 +1,11 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"os"
-	"sort"
+	"slices"
 	"strconv"
 	"time"

@@ -81,11 +66,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 	for name := range respondedNodes {
 		nodeSet[name] = struct{}{}
 	}
-	nodeNames := make([]string, 0, len(nodeSet))
-	for name := range nodeSet {
-		nodeNames = append(nodeNames, name)
-	}
-	sort.Strings(nodeNames)
+	nodeNames := slices.Sorted(maps.Keys(nodeSet))

 	nodeTable := pterm.TableData{
 		{"NAME", "CPU (used/total)", "MEM (used/total)", "GPU (used/total)", "VMs"},
@@ -141,11 +122,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {

 	fmt.Println()

-	capNames := make([]string, 0, len(capacityMap))
-	for name := range capacityMap {
-		capNames = append(capNames, name)
-	}
-	sort.Strings(capNames)
+	capNames := slices.Sorted(maps.Keys(capacityMap))

 	capTable := pterm.TableData{
 		{"INSTANCE TYPE", "AVAILABLE", "VCPU", "MEMORY"},
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main

 import (
@@ -1,183 +0,0 @@
-# Spinifex Systemd Service
-
-This directory contains the systemd service configuration for running Spinifex services on Ubuntu and Debian systems.
-
-## Files
-
- `spinifex.service` - The systemd service file that manages all Spinifex services
- `install-spinifex-service.sh` - Installation script to set up the service
- `README.md` - This documentation file
-
-## Installation
-
-### Prerequisites
-
-1. Ensure Spinifex is installed in `/opt/spinifex/`
-2. Ensure the `spx` binary is available at `/opt/spinifex/bin/spx`
-3. Run the installation script as root
-
-### Setup Configuration
-
-```
-/opt/spinifex/bin/spx admin init
-```
-
-This will init the Spinifex environment and create the default configuration files in the home directory, e.g `~/spinifex/`.
-
-```
-/home/username/spinifex:
-amis  config  images  logs  nats  predastore  state  viperblock  volumes
-
-/home/username/spinifex/amis:
-
-/home/username/spinifex/config:
-awsgw  spinifex.toml  nats  predastore  server.key  server.pem
-
-/home/username/spinifex/config/awsgw:
-awsgw.toml
-
-/home/username/spinifex/config/nats:
-nats.conf
-
-/home/username/spinifex/config/predastore:
-predastore.toml
-
-/home/username/spinifex/images:
-
-/home/username/spinifex/logs:
-awsgw.log  awsgw.pid  spinifex.log  spinifex.pid  nats.log  nats.pid  predastore.log  predastore.pid  viperblock.log  viperblock.pid
-
-/home/username/spinifex/nats:
-
-/home/username/spinifex/predastore:
-data
-
-/home/username/spinifex/predastore/data:
-predastore
-
-/home/ben/spinifex/state:
-
-/home/ben/spinifex/viperblock:
-
-/home/ben/spinifex/volumes:
-```
-
-### Quick Installation
-
-```bash
-sudo ./config/install-spinifex-service.sh
-```
-
-### Manual Installation
-
-If you prefer to install manually:
-
-1. Copy the service file:
-   ```bash
-   sudo cp config/spinifex.service /etc/systemd/system/
-   ```
-
-2. Reload systemd:
-   ```bash
-   sudo systemctl daemon-reload
-   ```
-
-3. Enable the service:
-   ```bash
-   sudo systemctl enable spinifex.service
-   ```
-
-## Service Management
-
-### Start the service
-```bash
-sudo systemctl start spinifex
-```
-
-### Stop the service
-```bash
-sudo systemctl stop spinifex
-```
-
-### Check service status
-```bash
-sudo systemctl status spinifex
-```
-
-### Restart the service
-```bash
-sudo systemctl restart spinifex
-```
-
-### View service logs
-```bash
-sudo journalctl -u spinifex -f
-```
-
-### Disable auto-start
-```bash
-sudo systemctl disable spinifex
-```
-
-## Service Details
-
-The `spinifex.service` file:
-
- Starts services in the correct order: NATS → Predastore → Viperblock
- Includes proper delays between service starts to ensure dependencies are ready
- Stops services in reverse order during shutdown
- Runs as root (required for system services)
- Includes security hardening settings
- Provides proper logging to systemd journal
- Automatically restarts on failure
-
-## Service Order
-
-The service starts the following components in order:
-
-1. **NATS** - Message broker service
-2. **Predastore** - Data storage service (waits 3 seconds after NATS)
-3. **Viperblock** - Block storage service (waits 2 seconds after Predastore)
-
-## Troubleshooting
-
-### Check if the service is running
-```bash
-sudo systemctl is-active spinifex
-```
-
-### View recent logs
-```bash
-sudo journalctl -u spinifex --since "1 hour ago"
-```
-
-### Check service configuration
-```bash
-sudo systemctl cat spinifex
-```
-
-### Verify binary exists
-```bash
-ls -la /opt/spinifex/bin/spx
-```
-
-### Test service commands manually
-```bash
-/opt/spinifex/bin/spx service nats start
-/opt/spinifex/bin/spx service predastore start
-/opt/spinifex/bin/spx service viperblock start
-```
-
-## Customization
-
-If you need to modify the service configuration:
-
-1. Edit the service file: `sudo nano /etc/systemd/system/spinifex.service`
-2. Reload systemd: `sudo systemctl daemon-reload`
-3. Restart the service: `sudo systemctl restart spinifex`
-
-## Security Notes
-
- The service runs as root, which is necessary for system-level services
- Security hardening is enabled with `ProtectSystem=strict` and `NoNewPrivileges=true`
- Only specific paths are writable: `/opt/spinifex`, `/var/log`, `/var/run` 
@@ -1,74 +0,0 @@
-#!/bin/bash
-
-# Spinifex Service Installation Script
-# This script installs the spinifex systemd service on Ubuntu/Debian systems
-
-set -e
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Function to print colored output
-print_status() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-print_warning() {
-    echo -e "${YELLOW}[WARNING]${NC} $1"
-}
-
-print_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-# Check if running as root
-if [[ $EUID -ne 0 ]]; then
-   print_error "This script must be run as root (use sudo)"
-   exit 1
-fi
-
-# Check if systemd is available
-if ! command -v systemctl &> /dev/null; then
-    print_error "systemd is not available on this system"
-    exit 1
-fi
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SERVICE_FILE="$SCRIPT_DIR/spinifex.service"
-
-# Check if service file exists
-if [[ ! -f "$SERVICE_FILE" ]]; then
-    print_error "Service file not found at $SERVICE_FILE"
-    exit 1
-fi
-
-print_status "Installing Spinifex systemd service..."
-
-# Copy service file to systemd directory
-cp "$SERVICE_FILE" /etc/systemd/system/
-
-# Reload systemd to recognize the new service
-systemctl daemon-reload
-
-# Enable the service to start on boot
-systemctl enable spinifex.service
-
-print_status "Service installed successfully!"
-print_status "You can now use the following commands:"
-echo "  systemctl start spinifex    # Start the service"
-echo "  systemctl stop spinifex     # Stop the service"
-echo "  systemctl status spinifex   # Check service status"
-echo "  systemctl restart spinifex  # Restart the service"
-echo "  journalctl -u spinifex -f   # View service logs"
-
-# Check if spinifex binary exists
-if [[ ! -f "/opt/spinifex/bin/spinifex" ]]; then
-    print_warning "Spinifex binary not found at /opt/spinifex/bin/spinifex"
-    print_warning "Please ensure spinifex is installed in /opt/spinifex before starting the service"
-fi
-
-print_status "Installation complete!" 
--- a/Show More
+++ b/Show More