release: update go.mod for v1.11.0

chore: go 1.26.5
chore: rebuild ui with new deps
2026-10-23 20:57:02 +00:00 · 2026-07-09 10:02:22 +10:00 · 2026-07-09 09:43:32 +10:00 · 2026-07-09 09:43:32 +10:00 · 2026-07-09 09:43:32 +10:00 · 2026-07-09 09:43:32 +10:00
1414 changed files with 134143 additions and 99828 deletions
@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -463,8 +462,7 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }

-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report (Stage 1).
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.

 import (
@@ -0,0 +1,102 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">ECS · EC2 LAUNCH TYPE ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">container instances from spinifex-ecs-node AMI · awsvpc + ALB</text>
+
+  <!-- Internet -->
+  <rect x="120" y="52" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="220" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to ALB -->
+  <line x1="220" y1="92" x2="220" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="216,108 224,108 220,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- ALB -->
+  <g transform="translate(90, 116)">
+    <rect width="260" height="60" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">Application Load Balancer</text>
+    <line x1="16" y1="32" x2="244" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787">target group · target_type = ip</text>
+  </g>
+
+  <!-- Spinifex Gateway (control plane) -->
+  <g transform="translate(560, 110)">
+    <rect width="316" height="72" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">Spinifex Gateway :9999</text>
+    <text x="296" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">control plane</text>
+    <line x1="16" y1="32" x2="300" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">ECS agent registers · TLS + SigV4 (IMDS creds)</text>
+    <text x="20" y="63" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">task role creds → container at 169.254.170.2</text>
+  </g>
+
+  <!-- ALB fan-out to subnets (traffic) -->
+  <line x1="220" y1="176" x2="220" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="650" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="220" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="650" y1="200" x2="650" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="435" y="195" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">balances to each task's ENI</text>
+
+  <!-- agent path: subnets up to gateway -->
+  <line x1="718" y1="228" x2="718" y2="182" stroke="#F87171" stroke-width="1.1" stroke-dasharray="3,3" opacity="0.6"/>
+  <polygon points="714,186 722,186 718,178" fill="#F87171" opacity="0.8"/>
+
+  <!-- subnet_a -->
+  <g transform="translate(60, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- container instance -->
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="225" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(520, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="103" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">image pull</text>
+    <text x="225" y="117" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">registry / ECR</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="410" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">No ECS launch API — you add capacity by booting EC2 instances from the spinifex-ecs-node AMI (the console's "Provision capacity" wraps RunInstances)</text>
+  <text x="450" y="426" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">The service keeps the desired task count running and registers each task's ENI IP with the ALB target group</text>
+</svg>
@@ -0,0 +1,96 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="cp-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">EKS · MANAGED KUBERNETES ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">control plane VM + NLB · workers from eks-node AMI</text>
+
+  <!-- Internet / kubectl -->
+  <rect x="330" y="52" width="240" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet · kubectl + aws eks</text>
+
+  <!-- connector to control plane -->
+  <line x1="450" y1="92" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,108 454,108 450,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- Control plane (Spinifex-managed) -->
+  <g transform="translate(290, 116)">
+    <rect width="320" height="76" rx="8" fill="url(#cp-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">EKS Control Plane</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">Spinifex-managed</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="87" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">API NLB (public)</text>
+    <rect x="165" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="232" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">apiserver VM</text>
+  </g>
+
+  <!-- authn note -->
+  <rect x="624" y="138" width="172" height="32" rx="6" fill="#0a0a0a" stroke="#71717a" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="710" y="151" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">authn: API · access entries</text>
+  <text x="710" y="163" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">aws eks get-token (IAM → RBAC)</text>
+
+  <!-- fan-out to node-group subnets -->
+  <line x1="450" y1="192" x2="450" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="670" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="230" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="670" y1="210" x2="670" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="450" y="206" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">managed node group · workers join the control plane</text>
+
+  <!-- subnet_a -->
+  <g transform="translate(108, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- worker node -->
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(472, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">ECR</text>
+    <text x="248" y="107" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">pull</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="404" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">You provide the VPC, subnets, and IAM roles · Spinifex provisions the control plane, node-group VMs, and security groups</text>
+  <text x="450" y="420" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Node groups always boot the registered eks-node image · workers need egress (IGW / NAT) to pull images</text>
+</svg>
@@ -0,0 +1,7 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Changes
+
+<!-- The notable changes, as bullets. -->
@@ -35,7 +35,7 @@ jobs:
          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          ref: ${{ inputs.ref || github.ref }}

@@ -46,7 +46,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -16,6 +16,10 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  ddil:
@@ -35,12 +39,12 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Spinifex
        with:
          path: spinifex

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Mulga (tofu-cluster)
        with:
          repository: mulgadc/mulga
@@ -200,6 +204,7 @@ jobs:
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          ok "teardown complete"
@@ -17,6 +17,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  matrix:
@@ -35,6 +39,7 @@ jobs:
          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
+          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }

    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
    timeout-minutes: 35
@@ -74,7 +79,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          repository: mulgadc/mulga
@@ -82,7 +87,7 @@ jobs:
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          path: spinifex
@@ -121,7 +126,12 @@ jobs:
        env:
          MATRIX_NET: ${{ matrix.net }}
        run: |
-          if [ "$MATRIX_NET" = "dhcp" ]; then
+          if [ "$MATRIX_NET" = "nat" ]; then
+            # Routed-NAT mode with a static public pool: exercises both the
+            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
+            export SPX_EXTERNAL_MODE=nat
+            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
+          elif [ "$MATRIX_NET" = "dhcp" ]; then
            unset SPX_EXTERNAL_POOL
            export SPX_EXTERNAL_SOURCE=dhcp
          else
@@ -154,6 +164,28 @@ jobs:
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

+      - name: Run NAT Uplink E2E
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        # Skips cleanly (exit 0) when the tested branch predates the natuplink
+        # suite so scheduled runs against dev stay green until the merge lands.
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
        working-directory: mulga/scripts/tofu-cluster
@@ -312,6 +344,7 @@ jobs:
            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
          fi
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

@@ -3,7 +3,7 @@
 # Restructures the looped e2e.yml into env-grouped job blocks so each suite
 # surfaces as its own node in the Actions graph (mulga-siv-283):
 #
-#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks)
+#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
 #                     └─ single_teardown (if: always)
 #   multi_provision  ─┬─ multi_suite  (matrix: multinode)
 #                     └─ multi_teardown (if: always)
@@ -27,7 +27,7 @@ on:
      single_suites:
        description: "Single-node suites to run as matrix legs"
        required: false
-        default: "single cert lb eks"
+        default: "single cert lb eks iam"
      run_multinode:
        description: "Also run the multinode env block"
        type: boolean
@@ -44,6 +44,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.ref_name }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # ---------------------------------------------------------------------------
@@ -51,7 +55,7 @@ jobs:
  # ---------------------------------------------------------------------------
  single_provision:
    name: Single env · provision
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 40
    outputs:
      wan_ip: ${{ steps.vm.outputs.wan_ip }}
@@ -92,14 +96,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -114,7 +118,7 @@ jobs:
      - name: Compute suite matrix
        id: suites
        run: |
-          LIST="${{ inputs.single_suites || 'single cert lb eks' }}"
+          LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
          JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
@@ -199,7 +203,7 @@ jobs:
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      - name: Install kubectl on VM
@@ -266,7 +270,7 @@ jobs:
  single_suite:
    name: Single env · suite
    needs: single_provision
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 45
    strategy:
      fail-fast: false
@@ -277,7 +281,7 @@ jobs:
      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
      ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -373,9 +377,9 @@ jobs:

  single_teardown:
    name: Single env · teardown
-    needs: [ single_provision, single_suite ]
+    needs: [single_provision, single_suite]
    if: always()
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
    timeout-minutes: 20
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
@@ -386,7 +390,7 @@ jobs:
      # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
      # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
      # again would wipe the local tofu state and orphan the VM.
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -458,6 +462,7 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"
@@ -468,7 +473,7 @@ jobs:
  multi_provision:
    name: Multi env · provision
    if: ${{ inputs.run_multinode }}
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 45
    outputs:
      node1_ip: ${{ steps.cluster.outputs.node1_ip }}
@@ -505,14 +510,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -615,19 +620,19 @@ jobs:
  multi_suite:
    name: Multi env · suite
    needs: multi_provision
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
-        suite: [ multinode ]
+        suite: [multinode]
    env:
      NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -723,16 +728,16 @@ jobs:

  multi_teardown:
    name: Multi env · teardown
-    needs: [ multi_provision, multi_suite ]
+    needs: [multi_provision, multi_suite]
    if: always() && inputs.run_multinode
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 20
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -779,6 +784,7 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"
@@ -786,8 +792,8 @@ jobs:
  e2e_suites_status:
    name: E2E suite blocks · summary
    if: always()
-    needs: [ single_suite, multi_suite ]
-    runs-on: ubuntu-latest
+    needs: [single_suite, multi_suite]
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Summarize
@@ -14,13 +14,10 @@ on:
      - .github/workflows/e2e.yml
  workflow_dispatch:
    inputs:
-      sha:
-        description: "PR head SHA to report status on"
-        required: false
      suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
        required: false
-        default: "e2e-single e2e-multinode e2e-cert e2e-lb"
+        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
      test_run:
        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
        required: false
@@ -32,8 +29,12 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
@@ -41,10 +42,10 @@ jobs:
  # can compare predicted vs actual before flipping it on.
  select_suites:
    name: Select Suites (shadow)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v6
@@ -78,9 +79,9 @@ jobs:
    # suites='e2e-multinode' only). Push triggers and the default SUITES list
    # both include single/cert/lb so the inputs.suites null-check keeps the
    # job running on plain PR pushes.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-iam')
    runs-on: [self-hosted, ci-single]
-    timeout-minutes: 45
+    timeout-minutes: 50
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -110,6 +111,10 @@ jobs:
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -118,14 +123,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -223,13 +228,10 @@ jobs:
          scp "${SSH_OPTS[@]}" \
            /tmp/eks-node-image-build/eks-node-alpine.raw \
            "$VM":/tmp/eks-node-alpine.raw
-          # Drop the import progress bar (pterm redraws to stdout — thousands of
-          # lines over the SSH pipe). Errors go to stderr, so they survive; the
-          # exit code still gates the step.
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -459,6 +461,7 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"
@@ -466,14 +469,14 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

-  e2e_real_multi:
-    name: Real Multi-Node E2E
+  e2e_multi:
+    name: Multi-Node E2E
    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
@@ -506,6 +509,10 @@ jobs:
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -514,14 +521,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -593,12 +600,15 @@ jobs:
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # single + iam are single-node-only suites — their fixtures skip
+          # unless SPINIFEX_MODE=single, so building/running them on multinode
+          # is wasted work.
          # eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
          # stream and create-cluster needs an elected leader, but multinode
          # JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
          # validated single-node only for now; re-enable here once clustered
          # JetStream + multinode EKS are a deliberate target.
-          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
+          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
@@ -660,9 +670,6 @@ jobs:
            echo "=== systemctl is-active spinifex-* ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
            echo
-            echo "=== which spx + version ==="
-            ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true'
-            echo
            echo "=== spx get nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
            echo
@@ -681,7 +688,7 @@ jobs:
        id: e2e
        run: |
          set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -869,6 +876,7 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"
@@ -876,7 +884,7 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
@@ -885,28 +893,24 @@ jobs:
  e2e_status:
    name: Report E2E Status
    if: always()
-    needs: [e2e_single, e2e_real_multi]
-    runs-on: ubuntu-latest
+    needs: [e2e_single, e2e_multi]
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
        env:
          GH_TOKEN: ${{ github.token }}
-          REPORT_SHA: ${{ inputs.sha || github.sha }}
+          REPORT_SHA: ${{ github.sha }}
        run: |
-          if [[ -z "${REPORT_SHA}" ]]; then
-            echo "::error::no SHA available (inputs.sha and github.sha both empty)"
-            exit 1
-          fi
          single_ok=true; multi_ok=true
          case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
-          case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
+          case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
          if $single_ok && $multi_ok; then
            STATE="success"
            DESC="E2E suites passed"
          else
            STATE="failure"
-            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
            -f state="$STATE" \
@@ -1,9 +1,11 @@
-name: Publish EKS node AMI
+name: Publish EKS + ECS node AMIs

-# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
-# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
+# Builds the eks-node and ecs-agent system AMIs (libguestfs, on the runner) and
+# publishes each compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket
+# that backs iso.mulgadc.com/system-ami/. Operators then install them on any
+# cluster with:
 #   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger.
@@ -25,24 +27,24 @@ concurrency:

 jobs:
  publish:
-    name: Build + publish eks-node AMI
+    name: Build + publish eks-node + ecs-agent AMIs
    runs-on: [ self-hosted, ci-single ]

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -73,7 +75,7 @@ jobs:
            rm -rf /tmp/awscliv2.zip /tmp/aws-install
          fi

-      - name: Build + publish to R2
+      - name: Build + publish eks-node to R2
        working-directory: spinifex
        env:
          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -94,3 +96,22 @@ jobs:
          fi
          ./scripts/publish-system-image.sh \
            scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
+
+      - name: Build + publish ecs-agent to R2
+        working-directory: spinifex
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          FLAGS=(--build)
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/publish-system-image.sh \
+            scripts/images/ecs-agent/manifest.conf "${FLAGS[@]}"
@@ -9,21 +9,21 @@ permissions:

 jobs:
  distro-amd64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -34,7 +34,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v6
        with:
-          go-version: "1.26.4"
+          go-version: "1.26.5"

      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
        run: cd spinifex && make build-microvm-image
@@ -54,21 +54,21 @@ jobs:
            spinifex/scripts/setup.sh

  distro-arm64:
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-26.04-arm
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -91,7 +91,7 @@ jobs:

  release:
    needs: [ distro-amd64, distro-arm64 ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Download amd64 Artifacts
        uses: actions/download-artifact@v8
@@ -143,7 +143,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -190,7 +190,7 @@ jobs:

  promote:
    needs: [ verify, baremetal ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -211,7 +211,7 @@ jobs:
  mark-failed:
    needs: [ verify, baremetal ]
    if: failure()
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -251,25 +251,25 @@ jobs:
          repositories: mulga

      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          ref: ${{ github.ref }}

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock

      - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -23,24 +23,24 @@ permissions:
 jobs:
  build_and_test:
    name: Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          fetch-depth: 2

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -72,23 +72,23 @@ jobs:

  race_detection:
    name: Race Detection
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -108,23 +108,23 @@ jobs:

  lint_and_security:
    name: Lint and Security
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -23,14 +23,14 @@ permissions:
 jobs:
  ui_build_and_test:
    name: UI Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    defaults:
      run:
        working-directory: spinifex/services/spinifexui/frontend

    steps:
      - name: Checkout Code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          sparse-checkout: spinifex/services/spinifexui/frontend

@@ -37,13 +37,3 @@ Thumbs.db

 # Test reports
 *-report.txt
-
-# Internal docs (moved to private mulga repo)
-docs/INTERNAL.md
-docs/PRIVATE.md
-docs/DEVNOTES.md
-docs/PLAN.md
-docs/TODO.md
-docs/development/
-docs/internal/
-scripts/iac/aws/spinifex-test.pem
@@ -57,7 +57,6 @@ linters:
      check-type-assertions: true
      exclude-functions:
        - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
-        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable

    govet:
      enable-all: true
@@ -68,6 +68,10 @@ build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
 	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go

+build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
+	@echo -e "\n....Building ecs-agent (static)"
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
@@ -90,6 +94,12 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build

+build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=ecs-agent
+
+import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -208,9 +218,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base

 install-go:
-	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -287,7 +297,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/

-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
@@ -0,0 +1,50 @@
+#!/bin/sh
+# JetStream liveness watchdog for spinifex-nats.
+#
+# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
+# flushing stream state) and keep the process alive serving KV as read-only. That
+# is not a process failure, so Restart=on-failure never fires and KV-backed
+# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+#
+# This probes the NATS monitoring endpoint's JS health and restarts the service
+# only when JetStream is sustainedly unhealthy while the service is running. The
+# multi-sample window avoids restarting during transient startup/quorum 503s.
+
+set -eu
+
+HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
+SERVICE="spinifex-nats.service"
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
+    exit 0
+fi
+
+# curl is the probe transport; if it is unavailable, do nothing rather than
+# false-restart a healthy server.
+if ! command -v curl >/dev/null 2>&1; then
+    echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2
+    exit 0
+fi
+
+# A single 200 means JetStream is healthy — exit fast on the common case.
+js_healthy() {
+    curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
+}
+
+i=1
+while [ "$i" -le "$SAMPLES" ]; do
+    if js_healthy; then
+        exit 0
+    fi
+    if [ "$i" -lt "$SAMPLES" ]; then
+        sleep "$SAMPLE_GAP"
+    fi
+    i=$((i + 1))
+done
+
+echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+systemctl try-restart "$SERVICE"
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-gw
@@ -6,6 +6,7 @@ After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.servi
 Wants=openvswitch-ipsec.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -34,7 +35,9 @@ DeviceAllow=/dev/net/tun rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf

 # RG-9: hardening baseline
 ProtectHome=yes
@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog
+PartOf=spinifex.target
+After=spinifex-nats.service
+
+[Service]
+Type=oneshot
+# Runs as root: the probe is a local curl to the NATS monitoring port, and the
+# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
+# the process stays alive (which Restart=on-failure cannot catch).
+ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+
+# Hardening (kept compatible with talking to PID1 over /run/systemd).
+NoNewPrivileges=yes
+ProtectHome=yes
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+LockPersonality=yes
+RestrictSUIDSGID=yes
@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog timer
+PartOf=spinifex.target
+
+[Timer]
+# Delay the first probe so NATS startup + JetStream cluster quorum settle (a
+# forming cluster reports JS not-current transiently, which is not a fault).
+OnBootSec=60s
+OnUnitActiveSec=30s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-nats
@@ -5,6 +5,10 @@ After=spinifex-nats.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -0,0 +1,56 @@
+[Unit]
+Description=Spinifex QMP Telemetry Collector (guest metrics)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-daemon.service
+Wants=spinifex-nats.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# spinifex-daemon user: the collector reads the daemon's telemetry QMP
+# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.
+User=spinifex-daemon
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service qmp-collector start
+Restart=on-failure
+RestartSec=5
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
+# Same runtime dir as the daemon: telemetry sockets + metadata live there
+Environment=XDG_RUNTIME_DIR=/run/spinifex
+
+# RG-9: read-mostly telemetry tier — no capabilities, no device access
+ProtectSystem=full
+PrivateDevices=yes
+NoNewPrivileges=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads
+# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=512
+
+[Install]
+WantedBy=spinifex.target
@@ -0,0 +1,18 @@
+[Unit]
+Description=Spinifex graceful guest drain
+PartOf=spinifex.target
+# After= all services so it stops FIRST on target/host shutdown: the drain runs
+# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).
+After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/true
+# ExecStop only runs when the target/host goes down, not on a single-service
+# restart — so a daemon restart still leaves guests running (DDIL).
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+TimeoutStopSec=180
+
+[Install]
+WantedBy=spinifex.target
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 After=spinifex-awsgw.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-ui
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-viperblock
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ope
 Wants=spinifex-nats.service ovn-controller.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-vpcd
@@ -17,26 +18,34 @@ OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/

-# RG-9: network tier. vpcd MUST share the host mount namespace — IMDS enters
-# per-VPC netns via setns(), and any filesystem-sandbox directive (ProtectSystem/
-# Home, PrivateTmp, ProtectKernel*, ProtectControlGroups/Proc, Read*Paths) forks a
-# private mount ns that traps the /run/netns bind-mount and breaks setns(2). Those,
-# plus MemoryDenyWriteExecute, are omitted by design — not regressions.
-PrivateDevices=no
-RestrictSUIDSGID=no
+# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
+# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
+# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+NoNewPrivileges=no
+ProtectSystem=full
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
 LockPersonality=yes
 RestrictRealtime=yes
+RestrictNamespaces=yes
 ProtectClock=yes
-# mnt alongside net: in-netns `ip` unshares a mount ns for its own /sys
-RestrictNamespaces=net mnt
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
 SystemCallArchitectures=native
-MemoryDenyWriteExecute=no
+MemoryDenyWriteExecute=yes
 # AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
-# Caps: NET_ADMIN (ovs/ip), NET_RAW (AF_PACKET), NET_BIND_SERVICE (:68), SETUID/SETGID, SYS_ADMIN (setns into VPC netns)
-AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-NoNewPrivileges=no
 ProtectHostname=yes
 KeyringMode=private
 RemoveIPC=yes
@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service

 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,205 @@
+// ecr-credential-provider is a kubelet exec credential provider that lets a
+// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
+//
+// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
+// reading a CredentialProviderResponse on STDOUT. The provider fetches the
+// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
+// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
+// for the requested registry host.
+//
+// Static node config (gateway URL, CA, region) is read from the cloud-init env
+// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+const (
+	credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
+	defaultCacheDuration   = "10h"
+	// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
+	// before the JWT actually expires mid-run.
+	expiryMargin = 30 * time.Minute
+
+	defaultEnvFile   = "/etc/spinifex-eks/agent.env"
+	defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
+	defaultIMDSBase  = "http://169.254.169.254/latest"
+)
+
+// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
+type CredentialProviderRequest struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Image      string `json:"image"`
+}
+
+// AuthConfig is a single registry credential entry.
+type AuthConfig struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
+type CredentialProviderResponse struct {
+	APIVersion    string                `json:"apiVersion"`
+	Kind          string                `json:"kind"`
+	CacheKeyType  string                `json:"cacheKeyType"`
+	CacheDuration string                `json:"cacheDuration"`
+	Auth          map[string]AuthConfig `json:"auth"`
+}
+
+// config holds the static node settings the provider needs.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	IMDSBase   string
+}
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
+	if err != nil {
+		slog.Error("ecr-credential-provider: build gateway client", "err", err)
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+
+	if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
+		slog.Error("ecr-credential-provider: run failed", "err", err)
+		// Still emit a valid empty-auth response so kubelet has a parseable reply.
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+}
+
+// run is the testable core: it reads a CredentialProviderRequest from in,
+// resolves credentials via IMDS + the gateway, and writes a
+// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
+func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
+	var req CredentialProviderRequest
+	if err := json.NewDecoder(in).Decode(&req); err != nil {
+		return fmt.Errorf("decode request: %w", err)
+	}
+	if req.Image == "" {
+		return fmt.Errorf("request image is empty")
+	}
+	imageHost := ecrauth.HostFromImage(req.Image)
+
+	creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
+	if err != nil {
+		return fmt.Errorf("fetch IMDS credentials: %w", err)
+	}
+
+	tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
+		creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
+	if err != nil {
+		return fmt.Errorf("get authorization token: %w", err)
+	}
+
+	auth := AuthConfig{Username: tok.Username, Password: tok.Password}
+	authMap := map[string]AuthConfig{imageHost: auth}
+	if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
+		authMap[tok.ProxyHost] = auth
+	}
+
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: cacheDurationFrom(tok.ExpiresAt),
+		Auth:          authMap,
+	}
+	return json.NewEncoder(out).Encode(&resp)
+}
+
+// cacheDurationFrom derives a Go-duration string from the token expiry minus a
+// safety margin, falling back to the default when expiry is unknown or too near.
+func cacheDurationFrom(expiresAt time.Time) string {
+	if expiresAt.IsZero() {
+		return defaultCacheDuration
+	}
+	d := time.Until(expiresAt) - expiryMargin
+	if d <= 0 {
+		return defaultCacheDuration
+	}
+	return d.Round(time.Second).String()
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override. The
+// IMDS base is fixed (not in the env file) but overridable for tests.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL: get("EKS_GATEWAY_URL"),
+		GatewayCA:  get("EKS_GATEWAY_CA"),
+		Region:     get("EKS_REGION"),
+		IMDSBase:   get("EKS_IMDS_BASE"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
+
+// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
+func emitEmpty(out io.Writer) {
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: defaultCacheDuration,
+		Auth:          map[string]AuthConfig{},
+	}
+	if err := json.NewEncoder(out).Encode(&resp); err != nil {
+		slog.Error("ecr-credential-provider: encode empty response", "err", err)
+	}
+}
@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
+// ECR GetAuthorizationToken on one TLS server, routed by request path.
+func fakeBackend(t *testing.T) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+
+	// IMDSv2 token.
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("imds-v2-token"))
+	})
+	// IMDS role name.
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		// Role credentials.
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"Type":            "AWS-HMAC",
+			"AccessKeyId":     "AKIATEST",
+			"SecretAccessKey": "secrettest",
+			"Token":           "sessiontoken",
+			"AccountId":       "123456789012",
+		})
+	})
+	// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": authToken,
+				"proxyEndpoint":      "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
+			}},
+		})
+	})
+
+	srv := httptest.NewTLSServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func clientTrusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestRun_ReturnsBasicAuth(t *testing.T) {
+	srv := fakeBackend(t)
+	client := clientTrusting(srv)
+
+	cfg := config{
+		GatewayURL: srv.URL,
+		Region:     "us-east-1",
+		IMDSBase:   srv.URL + "/latest",
+	}
+
+	const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
+	reqBody, err := json.Marshal(CredentialProviderRequest{
+		APIVersion: credProviderAPIVersion,
+		Kind:       "CredentialProviderRequest",
+		Image:      image,
+	})
+	if err != nil {
+		t.Fatalf("marshal request: %v", err)
+	}
+
+	var out bytes.Buffer
+	if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	var resp CredentialProviderResponse
+	if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
+		t.Fatalf("unmarshal response: %v\n%s", err, out.String())
+	}
+
+	if resp.Kind != "CredentialProviderResponse" {
+		t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
+	}
+	if resp.CacheKeyType != "Registry" {
+		t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
+	}
+
+	host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
+	auth, ok := resp.Auth[host]
+	if !ok {
+		t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
+	}
+	if auth.Username != "AWS" {
+		t.Errorf("username = %q, want AWS", auth.Username)
+	}
+	if auth.Password != "testjwt" {
+		t.Errorf("password = %q, want testjwt", auth.Password)
+	}
+}
@@ -0,0 +1,188 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	goruntime "runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// version is the agent build version, reported in the register message.
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
+// container runtime, and an ECR resolver. It registers the host as a container
+// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
+// task assignments, and runs them through containerd, reporting state back over
+// the gateway. It never connects to NATS — the bus stays host-internal.
+type Agent struct {
+	cfg      config
+	id       identity
+	cp       controlPlane
+	puller   ctrruntime.ImagePuller
+	runner   ctrruntime.Runner
+	resolver ctrruntime.Resolver
+
+	reg *registrar
+	hb  *heartbeater
+
+	// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
+	netns *taskNetns
+
+	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
+	cred *credEndpoint
+
+	closers []func() error
+}
+
+// newAgent assembles an Agent from already-built seams. Tests use this directly
+// with fakes; New builds the production seams and delegates here. runner may be
+// nil when containerd is unavailable; the assign path then reports the task
+// STOPPED rather than crashing.
+func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
+	return &Agent{
+		cfg:      cfg,
+		id:       id,
+		cp:       cp,
+		puller:   puller,
+		runner:   runner,
+		resolver: resolver,
+		reg:      newRegistrar(cp, id),
+		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		netns:    newTaskNetns(execNetRunner{}),
+	}
+}
+
+// New builds an Agent from config: it resolves the host identity from IMDS,
+// builds the gateway control-plane client, the ECR resolver and (best-effort)
+// the containerd runtime. A containerd connect failure is logged, not fatal —
+// registration and heartbeat still run so the instance is visible while the
+// runtime recovers. The ECR gateway client is built lazily on first image pull
+// (not here), so a missing or malformed gateway CA does not stop registration.
+func New(cfg config) (*Agent, error) {
+	imdsClient := &http.Client{Timeout: 5 * time.Second}
+
+	meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
+	if err != nil {
+		return nil, fmt.Errorf("instance metadata: %w", err)
+	}
+	host, _ := os.Hostname()
+	id := identity{
+		AccountID:    meta.AccountID,
+		ClusterName:  cfg.ClusterName,
+		InstanceID:   meta.InstanceID,
+		AZ:           meta.AZ,
+		Hostname:     host,
+		Capacity:     detectCapacity(),
+		AgentVersion: version,
+	}
+
+	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
+
+	var puller ctrruntime.ImagePuller
+	var runner ctrruntime.Runner
+	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
+	} else {
+		puller = rt
+		runner = rt
+	}
+
+	cp, err := newGatewayControlPlane(cfg, creds)
+	if err != nil {
+		return nil, fmt.Errorf("build gateway control-plane: %w", err)
+	}
+
+	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
+		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
+	if puller != nil {
+		a.closers = append(a.closers, puller.Close)
+	}
+	return a, nil
+}
+
+// Run registers the instance, starts the heartbeat and assignment-poll loops,
+// and blocks until ctx is cancelled, then tears down.
+func (a *Agent) Run(ctx context.Context) error {
+	if err := a.reg.Register(); err != nil {
+		slog.Error("ecs-agent: initial registration failed", "err", err)
+	} else {
+		slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
+	}
+
+	if a.cred != nil {
+		if err := a.cred.Start(); err != nil {
+			slog.Error("ecs-agent: credential endpoint start failed", "err", err)
+		}
+	}
+
+	go a.hb.Run(ctx)
+
+	// Re-adopt containers still running from before this restart, then poll with
+	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
+	go a.pollAssignments(ctx, a.reconcile(ctx))
+
+	<-ctx.Done()
+	return a.Stop()
+}
+
+// Stop closes the runtime. Safe to call once.
+func (a *Agent) Stop() error {
+	var firstErr error
+	if a.cred != nil {
+		_ = a.cred.Stop()
+	}
+	for _, c := range a.closers {
+		if err := c(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
+func detectCapacity() bus.InstanceCapacity {
+	return bus.InstanceCapacity{
+		CPU:       goruntime.NumCPU() * 1024,
+		MemoryMiB: memTotalMiB(),
+	}
+}
+
+// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
+func memTotalMiB() int {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			return 0
+		}
+		kb, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return 0
+		}
+		return kb / 1024
+	}
+	return 0
+}
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
+	puller := &ctrruntime.FakePuller{}
+	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a.closers = append(a.closers, puller.Close)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	// Give it time to register + re-register (heartbeat) a few times.
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("Run returned error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+
+	// One boot register + at least one heartbeat re-register.
+	if cp.registerCount() < 2 {
+		t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
+	}
+	if !puller.Closed {
+		t.Error("Stop did not close the puller")
+	}
+}
+
+func TestDetectCapacity_Positive(t *testing.T) {
+	c := detectCapacity()
+	if c.CPU <= 0 {
+		t.Errorf("CPU = %d, want > 0", c.CPU)
+	}
+	// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
+	if c.MemoryMiB < 0 {
+		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
+	}
+}
@@ -0,0 +1,119 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	defaultEnvFile          = "/etc/spinifex-ecs/agent.env"
+	defaultGatewayCA        = "/etc/spinifex-ecs/gateway-ca.pem"
+	defaultIMDSBase         = "http://169.254.169.254/latest"
+	defaultContainerdSocket = "/run/containerd/containerd.sock"
+	defaultHeartbeat        = 30 * time.Second
+	defaultPollInterval     = 5 * time.Second
+)
+
+// config holds the static settings the agent reads at boot. Cluster identity
+// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
+// only the cluster *name* the instance was launched into is static. AccessKey /
+// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
+// gateway calls with instance-role credentials from IMDS; the agent never holds
+// a NATS token.
+type config struct {
+	GatewayURL       string
+	GatewayCA        string
+	Region           string
+	ClusterName      string
+	AccessKey        string
+	SecretKey        string
+	IMDSBase         string
+	ContainerdSocket string
+	Heartbeat        time.Duration
+	PollInterval     time.Duration
+	CredEndpointIP   string
+	CredEndpointPort int
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL:       get("ECS_GATEWAY_URL"),
+		GatewayCA:        get("ECS_GATEWAY_CA"),
+		Region:           get("ECS_REGION"),
+		ClusterName:      get("ECS_CLUSTER"),
+		AccessKey:        get("ECS_ACCESS_KEY"),
+		SecretKey:        get("ECS_SECRET_KEY"),
+		IMDSBase:         get("ECS_IMDS_BASE"),
+		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	if cfg.ContainerdSocket == "" {
+		cfg.ContainerdSocket = defaultContainerdSocket
+	}
+	if cfg.ClusterName == "" {
+		cfg.ClusterName = "default"
+	}
+	cfg.Heartbeat = defaultHeartbeat
+	if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.Heartbeat = d
+		}
+	}
+	cfg.PollInterval = defaultPollInterval
+	if v := get("ECS_POLL_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollInterval = d
+		}
+	}
+	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
+	if cfg.CredEndpointIP == "" {
+		cfg.CredEndpointIP = defaultCredEndpointIP
+	}
+	cfg.CredEndpointPort = defaultCredEndpointPort
+	if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.CredEndpointPort = p
+		}
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
@@ -0,0 +1,88 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	// Ensure env vars don't leak in from the host.
+	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
+		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
+		t.Setenv(k, "")
+	}
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.GatewayCA != defaultGatewayCA {
+		t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
+	}
+	if cfg.IMDSBase != defaultIMDSBase {
+		t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
+	}
+	if cfg.ContainerdSocket != defaultContainerdSocket {
+		t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
+	}
+	if cfg.ClusterName != "default" {
+		t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
+	}
+	if cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
+	}
+	if cfg.PollInterval != defaultPollInterval {
+		t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
+	}
+}
+
+func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
+	dir := t.TempDir()
+	envFile := filepath.Join(dir, "agent.env")
+	body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
+	if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
+	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
+
+	cfg := loadConfig(envFile)
+	if cfg.GatewayURL != "https://gw.env" {
+		t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
+	}
+	if cfg.Region != "us-west-2" {
+		t.Errorf("Region = %q, want from file", cfg.Region)
+	}
+	if cfg.ClusterName != "prod" {
+		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
+	}
+	if cfg.AccessKey != "AKIAENV" {
+		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
+	}
+}
+
+func TestLoadConfig_HeartbeatOverride(t *testing.T) {
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
+		t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
+	}
+	// Garbage value falls back to the default, not zero.
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
+	}
+}
+
+func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "f.env")
+	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	m := parseEnvFile(p)
+	if m["A"] != "1" || m["B"] != "2" {
+		t.Errorf("parse mismatch: %#v", m)
+	}
+	if _, ok := m["nokeyval"]; ok {
+		t.Errorf("line without = should be skipped")
+	}
+}
@@ -0,0 +1,135 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecsgw"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
+// NATS publisher: every method is a SigV4-signed gateway request, so the NATS bus
+// stays host-internal. Tests inject a fake; production uses gatewayControlPlane.
+type controlPlane interface {
+	// Register (re-)registers the container instance. Called at boot and on every
+	// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
+	// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
+	Register(id identity) error
+	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
+	SubmitTaskState(st bus.TaskState) error
+	// PollAssignments drains this instance's assignment + stop inboxes, acking the
+	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
+	// still-pending assignments and stop directives.
+	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+}
+
+// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
+type gatewayControlPlane struct {
+	client *ecsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// pollTimeout bounds a single PollAssignments request; larger than the register
+// timeout to leave room for a future server-side long-poll.
+const pollTimeout = 30 * time.Second
+
+// newGatewayControlPlane builds the SigV4 client signing with the instance-role
+// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
+func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
+// per-call credentials hook.
+func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
+	return func(ctx context.Context) (string, string, string, error) {
+		c, err := p.Retrieve(ctx)
+		if err != nil {
+			return "", "", "", err
+		}
+		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
+	}
+}
+
+func (g *gatewayControlPlane) Register(id identity) error {
+	in := &ecs.RegisterContainerInstanceInput{
+		Cluster:                  aws.String(id.ClusterName),
+		InstanceIdentityDocument: aws.String(id.InstanceID),
+		TotalResources: []*ecs.Resource{
+			{Name: aws.String("CPU"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.CPU))},
+			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
+		},
+		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal register: %w", err)
+	}
+	if _, err := g.client.Call("RegisterContainerInstance", body); err != nil {
+		return fmt.Errorf("register: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
+	in := &ecs.SubmitTaskStateChangeInput{
+		Cluster: aws.String(st.ClusterName),
+		Task:    aws.String(st.TaskID),
+		Status:  aws.String(st.LastStatus),
+		Reason:  aws.String(st.Reason),
+	}
+	for _, c := range st.Containers {
+		csc := &ecs.ContainerStateChange{
+			ContainerName: aws.String(c.Name),
+			Status:        aws.String(c.Status),
+			RuntimeId:     aws.String(c.ContainerID),
+		}
+		if c.ExitCode != nil {
+			csc.ExitCode = aws.Int64(int64(*c.ExitCode))
+		}
+		in.Containers = append(in.Containers, csc)
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal task-state: %w", err)
+	}
+	if _, err := g.client.Call("SubmitTaskStateChange", body); err != nil {
+		return fmt.Errorf("submit task-state: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	in := &handlers_ecs.PollAssignmentsInput{
+		Cluster:           cluster,
+		ContainerInstance: instance,
+		AckTaskIDs:        ackAssigns,
+		AckStopIDs:        ackStops,
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal poll: %w", err)
+	}
+	resp, err := g.client.Call("PollAssignments", body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("poll: %w", err)
+	}
+	var out handlers_ecs.PollAssignmentsOutput
+	if err := json.Unmarshal(resp, &out); err != nil {
+		return nil, nil, fmt.Errorf("decode poll: %w", err)
+	}
+	return out.Assignments, out.Stops, nil
+}
@@ -0,0 +1,400 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+const (
+	// defaultCredEndpointIP is the AWS-reserved link-local address tasks reach for
+	// container credentials; defaultCredEndpointPort is the standard HTTP port.
+	defaultCredEndpointIP   = "169.254.170.2"
+	defaultCredEndpointPort = 80
+	// defaultCredProxyPort is the high port the listener actually binds; a nat
+	// DNAT rule rewrites 169.254.170.2:80 to it so nothing holds a socket on :80
+	// and host/bridge tasks sharing the netns can bind :80 themselves.
+	defaultCredProxyPort = 51679
+	// credRefreshMargin refreshes a cached credential set this long before expiry
+	// so a container never starts a request with a token about to lapse.
+	credRefreshMargin = 5 * time.Minute
+	// credDummyIface is the dummy interface the endpoint IP is added to in the host
+	// netns so the agent can bind it.
+	credDummyIface = "ecs-cred"
+)
+
+// credResponse is the ECS task-credentials JSON the container's AWS SDK expects
+// at AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.
+type credResponse struct {
+	RoleArn         string `json:"RoleArn"`
+	AccessKeyID     string `json:"AccessKeyId"`
+	SecretAccessKey string `json:"SecretAccessKey"`
+	Token           string `json:"Token"`
+	Expiration      string `json:"Expiration"`
+}
+
+// credEntry caches one task's assumed-role credentials alongside the role it was
+// assumed from.
+type credEntry struct {
+	roleARN string
+	creds   stsauth.Credentials
+}
+
+// credEndpoint is the host-netns HTTP server that hands a task its IAM role
+// credentials. It maps credID -> taskRoleARN (registered as tasks start) and
+// mints/caches credentials on demand by assuming the role over the gateway with
+// the agent's instance credentials.
+type credEndpoint struct {
+	provider   credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+	ip         string
+	port       int
+	proxyPort  int
+	run        netCmdRunner
+
+	// assume mints fresh credentials for a role; defaults to AssumeRole over the
+	// gateway and is overridable in tests.
+	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+
+	mu         sync.Mutex
+	roles      map[string]string    // credID -> taskRoleARN
+	cache      map[string]credEntry // credID -> cached creds
+	httpClient *http.Client
+
+	srv *http.Server
+	ln  net.Listener
+}
+
+// newCredEndpoint builds the endpoint. An empty ip/port falls back to the
+// AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
+// adds the link-local address to the dummy interface (real or fake).
+func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+	if ip == "" {
+		ip = defaultCredEndpointIP
+	}
+	if port == 0 {
+		port = defaultCredEndpointPort
+	}
+	c := &credEndpoint{
+		provider:   provider,
+		region:     region,
+		gatewayURL: gatewayURL,
+		caPath:     caPath,
+		ip:         ip,
+		port:       port,
+		proxyPort:  defaultCredProxyPort,
+		run:        run,
+		roles:      map[string]string{},
+		cache:      map[string]credEntry{},
+	}
+	c.assume = c.assumeOverGateway
+	return c
+}
+
+// Register associates a credID with the task role it serves; Deregister removes
+// it and drops any cached credentials. Both are no-ops for an empty credID.
+func (c *credEndpoint) Register(credID, roleARN string) {
+	if credID == "" || roleARN == "" {
+		return
+	}
+	c.mu.Lock()
+	c.roles[credID] = roleARN
+	c.mu.Unlock()
+}
+
+func (c *credEndpoint) Deregister(credID string) {
+	if credID == "" {
+		return
+	}
+	c.mu.Lock()
+	delete(c.roles, credID)
+	delete(c.cache, credID)
+	c.mu.Unlock()
+}
+
+// relativeURI is the value injected into a container's
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI for the given credID.
+func credRelativeURI(credID string) string { return "/v2/credentials/" + credID }
+
+// ServeHTTP serves GET /v2/credentials/{credID}. Unknown paths/credIDs return
+// 404; non-GET returns 405; an AssumeRole failure returns 502.
+func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	credID, ok := strings.CutPrefix(r.URL.Path, "/v2/credentials/")
+	if !ok || credID == "" || strings.Contains(credID, "/") {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	creds, roleARN, err := c.fetch(r.Context(), credID)
+	if err != nil {
+		if err == errUnknownCredID {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		http.Error(w, "credentials unavailable", http.StatusBadGateway)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(credResponse{
+		RoleArn:         roleARN,
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		Token:           creds.SessionToken,
+		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+	}); err != nil {
+		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+	}
+}
+
+// errUnknownCredID signals a credID with no registered task role.
+var errUnknownCredID = fmt.Errorf("unknown credId")
+
+// fetch returns valid credentials for credID, reusing the cache until within the
+// refresh margin and otherwise assuming the task role over the gateway.
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+	c.mu.Lock()
+	roleARN, known := c.roles[credID]
+	if !known {
+		c.mu.Unlock()
+		return stsauth.Credentials{}, "", errUnknownCredID
+	}
+	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+		c.mu.Unlock()
+		return e.creds, roleARN, nil
+	}
+	c.mu.Unlock()
+
+	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	if err != nil {
+		return stsauth.Credentials{}, "", err
+	}
+
+	c.mu.Lock()
+	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.mu.Unlock()
+	return creds, roleARN, nil
+}
+
+// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
+// roleARN over the gateway — the execution-role path for ECR image pulls. It
+// caches the assumed set and re-assumes within the refresh margin. session scopes
+// the STS session name.
+func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
+}
+
+// assumeProvider adapts a credEndpoint's AssumeRole path to the agent's
+// CredentialsProvider interface, caching the assumed credentials.
+type assumeProvider struct {
+	ep      *credEndpoint
+	roleARN string
+	session string
+
+	mu     sync.Mutex
+	cached stsauth.Credentials
+}
+
+var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+
+func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !credValid(p.cached, credRefreshMargin) {
+		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
+		if err != nil {
+			return credentials.Credentials{}, err
+		}
+		p.cached = creds
+	}
+	return credentials.Credentials{
+		AccessKeyID:     p.cached.AccessKeyID,
+		SecretAccessKey: p.cached.SecretAccessKey,
+		SessionToken:    p.cached.SessionToken,
+		Expiration:      p.cached.Expiration,
+	}, nil
+}
+
+// assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
+// against the gateway with the agent's instance credentials.
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	instCreds, err := c.provider.Retrieve(ctx)
+	if err != nil {
+		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+	}
+	client, err := c.client()
+	if err != nil {
+		return stsauth.Credentials{}, err
+	}
+	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
+		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
+		roleARN, sessionName)
+}
+
+// client builds the pinned-CA gateway HTTP client on first use and caches it.
+func (c *credEndpoint) client() (*http.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.httpClient != nil {
+		return c.httpClient, nil
+	}
+	hc, err := ecrauth.GatewayHTTPClient(c.caPath)
+	if err != nil {
+		return nil, err
+	}
+	c.httpClient = hc
+	return hc, nil
+}
+
+// Start adds the endpoint IP to a dummy interface in the host netns, serves HTTP
+// on the high proxy port, and DNATs the advertised :80 to it. A bind, address,
+// or nat failure is returned so the caller can log it without aborting
+// register/heartbeat.
+func (c *credEndpoint) Start() error {
+	if err := c.addEndpointAddr(); err != nil {
+		return err
+	}
+	ln, err := net.Listen("tcp", net.JoinHostPort(c.ip, strconv.Itoa(c.listenPort())))
+	if err != nil {
+		return fmt.Errorf("listen %s:%d: %w", c.ip, c.listenPort(), err)
+	}
+	if err := c.addRedirect(); err != nil {
+		_ = ln.Close()
+		return err
+	}
+	c.ln = ln
+	c.srv = &http.Server{Handler: c, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		if err := c.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("ecs-agent: credential endpoint serve", "err", err)
+		}
+	}()
+	slog.Info("ecs-agent: credential endpoint listening", "addr", ln.Addr().String())
+	return nil
+}
+
+// Stop shuts the server down and removes the dummy interface. Safe on a nil/
+// never-started endpoint.
+func (c *credEndpoint) Stop() error {
+	if c == nil {
+		return nil
+	}
+	if c.srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = c.srv.Shutdown(ctx)
+	}
+	c.delRedirect()
+	c.delEndpointAddr()
+	return nil
+}
+
+// listenPort is the port the socket binds. Loopback test mode binds the
+// caller-supplied (ephemeral) port directly; the real endpoint binds the high
+// proxy port and reaches :80 via the DNAT rules.
+func (c *credEndpoint) listenPort() int {
+	if c.ip == "127.0.0.1" {
+		return c.port
+	}
+	return c.proxyPort
+}
+
+// addRedirect installs nat DNAT rules rewriting the advertised :80 to the proxy
+// port, for both locally generated traffic (host/bridge task containers, OUTPUT)
+// and traffic arriving over the awsvpc credential veth (PREROUTING). The dst IP
+// is left on the endpoint IP so no route_localnet handling is needed. Skipped in
+// loopback test mode, which binds the port directly.
+func (c *credEndpoint) addRedirect() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		if _, err := c.run.Run("iptables", c.natArgs("-C", chain)...); err == nil {
+			continue // rule already present
+		}
+		if _, err := c.run.Run("iptables", c.natArgs("-A", chain)...); err != nil {
+			return fmt.Errorf("add nat redirect %s: %w", chain, err)
+		}
+	}
+	return nil
+}
+
+func (c *credEndpoint) delRedirect() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		_, _ = c.run.Run("iptables", c.natArgs("-D", chain)...)
+	}
+}
+
+// natArgs builds the iptables nat rule for op (-A/-C/-D) on chain: DNAT the
+// advertised endpoint :80 to the proxy port on the same IP.
+func (c *credEndpoint) natArgs(op, chain string) []string {
+	return []string{
+		"-t", "nat", op, chain,
+		"-d", c.ip, "-p", "tcp", "--dport", strconv.Itoa(c.port),
+		"-j", "DNAT", "--to-destination", net.JoinHostPort(c.ip, strconv.Itoa(c.proxyPort)),
+	}
+}
+
+// addEndpointAddr brings up the dummy interface holding the endpoint IP. Skipped
+// for loopback (tests bind 127.0.0.1 directly, no interface plumbing needed).
+func (c *credEndpoint) addEndpointAddr() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	_, _ = c.run.Run("ip", "link", "add", credDummyIface, "type", "dummy")
+	if _, err := c.run.Run("ip", "addr", "add", c.ip+"/32", "dev", credDummyIface); err != nil &&
+		!strings.Contains(err.Error(), "exists") {
+		return fmt.Errorf("add endpoint addr %s: %w", c.ip, err)
+	}
+	if _, err := c.run.Run("ip", "link", "set", credDummyIface, "up"); err != nil {
+		return fmt.Errorf("bring up %s: %w", credDummyIface, err)
+	}
+	return nil
+}
+
+func (c *credEndpoint) delEndpointAddr() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	_, _ = c.run.Run("ip", "link", "del", credDummyIface)
+}
+
+// credValid reports whether creds are usable now with margin to spare.
+func credValid(c stsauth.Credentials, margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// STS [\w+=,.@-]{2,64} constraint.
+func sessionName(credID string) string {
+	name := "ecs-" + credID
+	if len(name) > 64 {
+		name = name[:64]
+	}
+	return name
+}
@@ -0,0 +1,197 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+// newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
+// counting calls so cache behaviour is observable. run is nil (no iface plumbing).
+func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+	var calls int32
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		atomic.AddInt32(&calls, 1)
+		return creds, nil
+	}
+	return c, &calls
+}
+
+func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
+	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	})
+	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String())
+	}
+	var got credResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || got.SecretAccessKey != "secret" || got.Token != "tok" {
+		t.Errorf("bad creds in response: %+v", got)
+	}
+	if got.RoleArn != "arn:aws:iam::111122223333:role/task" {
+		t.Errorf("RoleArn = %q", got.RoleArn)
+	}
+	if got.Expiration != exp.Format(time.RFC3339) {
+		t.Errorf("Expiration = %q, want %q", got.Expiration, exp.Format(time.RFC3339))
+	}
+}
+
+func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404", rr.Code)
+	}
+	if *calls != 0 {
+		t.Errorf("assume called %d times for unknown credID, want 0", *calls)
+	}
+}
+
+func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("status = %d, want 405", rr.Code)
+	}
+}
+
+func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	for hit := range 3 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+		if rr.Code != http.StatusOK {
+			t.Fatalf("hit %d: status %d", hit, rr.Code)
+		}
+	}
+	if *calls != 1 {
+		t.Errorf("assume called %d times, want 1 (cached)", *calls)
+	}
+}
+
+func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
+	// Expiry inside the refresh margin -> every hit re-assumes.
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	})
+	c.Register("cred-1", "arn:role")
+	for range 2 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	}
+	if *calls != 2 {
+		t.Errorf("assume called %d times, want 2 (expiry within margin)", *calls)
+	}
+}
+
+func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	c.Deregister("cred-1")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404 after deregister", rr.Code)
+	}
+}
+
+func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		return stsauth.Credentials{}, context.DeadlineExceeded
+	}
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502", rr.Code)
+	}
+}
+
+func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	c.port = 0 // bind an ephemeral port instead of privileged 80
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	url := "http://" + c.ln.Addr().String() + credRelativeURI("cred-1")
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+}
+
+func TestCredEndpoint_RedirectInstallsAndRemovesDNAT(t *testing.T) {
+	// failOn the -C existence check so addRedirect falls through to -A.
+	f := &fakeNetRunner{failOn: "-t nat -C"}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "169.254.170.2", 80, f)
+	if c.listenPort() != defaultCredProxyPort {
+		t.Fatalf("listenPort = %d, want %d (proxy)", c.listenPort(), defaultCredProxyPort)
+	}
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	target := "-d 169.254.170.2 -p tcp --dport 80 -j DNAT --to-destination 169.254.170.2:51679"
+	for _, chain := range []string{"-A OUTPUT", "-A PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing add rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+	c.delRedirect()
+	for _, chain := range []string{"-D OUTPUT", "-D PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing delete rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+}
+
+func TestCredEndpoint_RedirectSkippedOnLoopback(t *testing.T) {
+	f := &fakeNetRunner{}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, f)
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	c.delRedirect()
+	if f.sawAny("iptables") {
+		t.Errorf("loopback mode must not touch iptables; calls=%v", f.joined())
+	}
+	if c.listenPort() != c.port {
+		t.Errorf("loopback listenPort = %d, want caller port %d", c.listenPort(), c.port)
+	}
+}
+
+func TestCredRelativeURI(t *testing.T) {
+	if got := credRelativeURI("abc"); got != "/v2/credentials/abc" {
+		t.Errorf("credRelativeURI = %q", got)
+	}
+}
@@ -0,0 +1,59 @@
+package credentials
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+)
+
+// refreshMargin is how long before expiry the cache is considered stale.
+const refreshMargin = 5 * time.Minute
+
+// IMDSProvider fetches instance-role credentials from IMDS and caches them until
+// they near expiry. Safe for concurrent use.
+type IMDSProvider struct {
+	client *http.Client
+	base   string
+
+	mu     sync.Mutex
+	cached Credentials
+}
+
+var _ CredentialsProvider = (*IMDSProvider)(nil)
+
+// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
+// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	return &IMDSProvider{client: client, base: base}
+}
+
+// Retrieve returns cached credentials when still valid, else fetches fresh ones.
+func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.cached.Valid(refreshMargin) {
+		return p.cached, nil
+	}
+	if err := ctx.Err(); err != nil {
+		return Credentials{}, err
+	}
+
+	raw, err := imdscreds.Fetch(p.client, p.base)
+	if err != nil {
+		return Credentials{}, err
+	}
+	p.cached = Credentials{
+		AccessKeyID:     raw.AccessKeyID,
+		SecretAccessKey: raw.SecretAccessKey,
+		SessionToken:    raw.SessionToken,
+		Expiration:      raw.Expiration,
+	}
+	return p.cached, nil
+}
@@ -0,0 +1,104 @@
+package credentials
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits != nil {
+			atomic.AddInt32(hits, 1)
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestRetrieve_FetchesAndCaches(t *testing.T) {
+	var hits int32
+	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		"Expiration":      exp.Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	got, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("Retrieve: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
+		t.Errorf("creds mismatch: %+v", got)
+	}
+
+	// Second call within validity window must not re-hit IMDS.
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatalf("Retrieve 2: %v", err)
+	}
+	if hits != 1 {
+		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
+	}
+}
+
+func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+	var hits int32
+	// Expiry inside the refresh margin → never cached, always refetched.
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+	})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if hits != 2 {
+		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+	}
+}
+
+func TestRetrieve_CancelledContext(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := p.Retrieve(ctx); err == nil {
+		t.Fatal("expected error from cancelled context")
+	}
+}
+
+func TestCredentials_Valid(t *testing.T) {
+	if (Credentials{}).Valid(0) {
+		t.Error("empty creds should be invalid")
+	}
+	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
+	if !noExp.Valid(time.Hour) {
+		t.Error("zero expiry should be treated valid")
+	}
+	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
+	if soon.Valid(5 * time.Minute) {
+		t.Error("expiry inside margin should be invalid")
+	}
+}
@@ -0,0 +1,36 @@
+// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
+// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
+// with the host's instance-role credentials, fetched from IMDS and cached until
+// near expiry.
+package credentials
+
+import (
+	"context"
+	"time"
+)
+
+// Credentials is a SigV4 credential set with an expiry.
+type Credentials struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	SessionToken    string
+	Expiration      time.Time
+}
+
+// Valid reports whether the credentials are usable now, leaving a margin so a
+// caller does not start a request with a token about to expire mid-flight.
+func (c Credentials) Valid(margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
+// must be safe for concurrent use.
+type CredentialsProvider interface {
+	Retrieve(ctx context.Context) (Credentials, error)
+}
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func TestTaskCredID(t *testing.T) {
+	cases := []struct {
+		name string
+		as   bus.Assign
+		want string
+	}{
+		{"no role", bus.Assign{TaskID: "t1"}, ""},
+		{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
+	}
+	for _, tc := range cases {
+		if got := taskCredID(&tc.as); got != tc.want {
+			t.Errorf("%s: taskCredID = %q, want %q", tc.name, got, tc.want)
+		}
+	}
+}
+
+func TestWithCredEnv(t *testing.T) {
+	base := map[string]string{"FOO": "bar"}
+
+	// No credID -> env returned unchanged (same map).
+	if got := withCredEnv(base, ""); got["FOO"] != "bar" || got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "" {
+		t.Errorf("blank credID mutated env: %v", got)
+	}
+
+	got := withCredEnv(base, "cred-1")
+	if got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "/v2/credentials/cred-1" {
+		t.Errorf("missing relative-uri: %v", got)
+	}
+	if got["FOO"] != "bar" {
+		t.Errorf("dropped existing env: %v", got)
+	}
+	// Original map must be untouched (copy semantics).
+	if _, ok := base["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]; ok {
+		t.Errorf("source env mutated: %v", base)
+	}
+}
+
+func TestSessionName(t *testing.T) {
+	if got := sessionName("abc"); got != "ecs-abc" {
+		t.Errorf("sessionName = %q", got)
+	}
+	long := sessionName(string(make([]byte, 100)))
+	if len(long) != 64 {
+		t.Errorf("sessionName not truncated to 64: len=%d", len(long))
+	}
+}
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+)
+
+// ecrResolver implements runtime.Resolver by minting an ECR authorization token
+// per pull: it fetches the host's IMDS credentials, SigV4-signs
+// GetAuthorizationToken against the gateway, and returns the AWS:<jwt> as a
+// user/password pair for containerd.
+//
+// The gateway HTTP client is built lazily on first Authorize, not at agent boot:
+// a missing or malformed gateway CA only matters when an image is actually
+// pulled, so the agent can still register + heartbeat on a host with absent or
+// wrong ECR config.
+type ecrResolver struct {
+	creds      credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+
+	mu         sync.Mutex
+	httpClient *http.Client
+}
+
+var _ runtime.Resolver = (*ecrResolver)(nil)
+
+// newECRResolver builds a resolver with an injected HTTP client (tests). The
+// client is used as-is and never rebuilt from caPath.
+func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
+}
+
+// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
+// from caPath on first use (production path).
+func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
+}
+
+// client returns the gateway HTTP client, building it from caPath on first call
+// and caching the result. An injected client (newECRResolver) is returned as-is.
+func (e *ecrResolver) client() (*http.Client, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.httpClient != nil {
+		return e.httpClient, nil
+	}
+	c, err := ecrauth.GatewayHTTPClient(e.caPath)
+	if err != nil {
+		return nil, err
+	}
+	e.httpClient = c
+	return c, nil
+}
+
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
+	host, _, _ := strings.Cut(ref, "/")
+	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
+		return false // no registry host -> docker.io default
+	}
+	return strings.Contains(host, ".dkr.ecr.")
+}
+
+// Authorize returns the ECR token's user/password and proxy endpoint for ref.
+// Non-ECR refs resolve to anonymous pull (empty credentials).
+func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
+	if !isECRRef(ref) {
+		return "", "", "", nil
+	}
+	c, err := e.creds.Retrieve(ctx)
+	if err != nil {
+		return "", "", "", err
+	}
+	httpClient, err := e.client()
+	if err != nil {
+		return "", "", "", err
+	}
+	tok, err := ecrauth.GetAuthorizationToken(e.region, e.gatewayURL, httpClient,
+		c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
+	if err != nil {
+		return "", "", "", err
+	}
+	return tok.Username, tok.Password, tok.ProxyHost, nil
+}
@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+)
+
+// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+type stubCreds struct {
+	c   credentials.Credentials
+	err error
+}
+
+func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+	return s.c, s.err
+}
+
+func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if status != http.StatusOK {
+			w.WriteHeader(status)
+			return
+		}
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
+				"proxyEndpoint":      proxy,
+			}},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func trusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestECRResolver_Authorize(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	user, pass, endpoint, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+	if endpoint != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
+		t.Errorf("endpoint = %q", endpoint)
+	}
+}
+
+func TestECRResolver_CredsError(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("no imds")}, "us-east-1", "https://x", http.DefaultClient)
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected creds error to propagate")
+	}
+}
+
+func TestECRResolver_GatewayError(t *testing.T) {
+	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected gateway error")
+	}
+}
+
+func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	// Construction with a bogus CA path must not fail — only Authorize does.
+	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected lazy gateway-client build to fail on missing CA")
+	}
+}
+
+// TestECRResolver_NonECRAnonymous asserts that a public-registry ref pulls
+// anonymously: Authorize returns empty creds without ever touching IMDS, even
+// when the credentials provider would error.
+func TestECRResolver_NonECRAnonymous(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("imds must not be called")}, "us-east-1", "https://x", http.DefaultClient)
+	for _, ref := range []string{"nginx:alpine", "docker.io/library/nginx:alpine", "ghcr.io/org/app:1"} {
+		user, pass, endpoint, err := r.Authorize(context.Background(), ref)
+		if err != nil || user != "" || pass != "" || endpoint != "" {
+			t.Errorf("ref %q: got (%q,%q,%q,%v), want anonymous", ref, user, pass, endpoint, err)
+		}
+	}
+}
+
+func TestIsECRRef(t *testing.T) {
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	}
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+		}
+	}
+}
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// heartbeater keeps the container instance alive by periodically re-registering
+// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
+// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
+// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
+type heartbeater struct {
+	cp       controlPlane
+	id       identity
+	interval time.Duration
+}
+
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	return &heartbeater{cp: cp, id: id, interval: interval}
+}
+
+// beat re-registers the instance once, refreshing its LastSeen.
+func (h *heartbeater) beat() error {
+	return h.cp.Register(h.id)
+}
+
+// Run beats every interval until ctx is cancelled. A failed beat is logged and
+// retried on the next tick rather than killing the loop.
+func (h *heartbeater) Run(ctx context.Context) {
+	ticker := time.NewTicker(h.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := h.beat(); err != nil {
+				slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
+			}
+		}
+	}
+}
@@ -0,0 +1,39 @@
+// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
+// the ECS-AMI, with containerd baked in). It registers the host with the ECS
+// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
+// re-registering, polls the gateway for task assignments, runs them through
+// containerd, and reports state back over the gateway.
+//
+// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
+// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
+// comes from IMDS at boot.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	agent, err := New(cfg)
+	if err != nil {
+		slog.Error("ecs-agent: startup failed", "err", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if err := agent.Run(ctx); err != nil {
+		slog.Error("ecs-agent: run failed", "err", err)
+		os.Exit(1)
+	}
+}
@@ -0,0 +1,97 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
+type instanceMetadata struct {
+	AccountID  string
+	InstanceID string
+	AZ         string
+}
+
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
+// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
+	token, err := imdsToken(client, base)
+	if err != nil {
+		return instanceMetadata{}, err
+	}
+
+	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
+	}
+	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
+	}
+
+	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
+	}
+	var ident struct {
+		AccountID string `json:"accountId"`
+	}
+	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
+		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
+	}
+
+	return instanceMetadata{
+		AccountID:  ident.AccountID,
+		InstanceID: strings.TrimSpace(instanceID),
+		AZ:         strings.TrimSpace(az),
+	}, nil
+}
+
+// imdsToken fetches an IMDSv2 session token.
+func imdsToken(client *http.Client, base string) (string, error) {
+	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("imds token: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
+
+// imdsGet performs a token-authenticated IMDS GET.
+func imdsGet(client *http.Client, url, token string) (string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	guard := func(w http.ResponseWriter, r *http.Request) bool {
+		if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("i-0abc123"))
+		}
+	})
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("us-east-1a"))
+		}
+	})
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
+		}
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestFetchInstanceMetadata_OK(t *testing.T) {
+	srv := metadataStub(t, true)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
+		t.Errorf("metadata mismatch: %+v", got)
+	}
+}
+
+func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
+	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
+	srv := metadataStub(t, true)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+		t.Fatalf("v2 path should succeed: %v", err)
+	}
+}
+
+func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+		t.Fatal("expected parse error on bad identity document")
+	}
+}
@@ -0,0 +1,167 @@
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// netnsRunDir is the iproute2 netns mount directory; a name here is the netns
+// path containerd joins via the OCI network namespace path.
+const netnsRunDir = "/var/run/netns"
+
+// Credentials path constants: each task netns gets a /30 veth into the host netns
+// from 169.254.172.0/22 and a host-routed /32 to the credential endpoint, keeping
+// the endpoint off the task ENI's VPC route. credTaskVeth is the task-side name
+// (unique per netns); the host side is per-task (credVethName).
+const (
+	credTaskVeth     = "ecsc0"
+	credEndpointCIDR = defaultCredEndpointIP + "/32"
+)
+
+// netCmdRunner executes a host networking command. Abstracted so unit tests can
+// assert the ip/netns sequence without touching the kernel.
+type netCmdRunner interface {
+	Run(name string, args ...string) (string, error)
+}
+
+// execNetRunner runs real commands.
+type execNetRunner struct{}
+
+func (execNetRunner) Run(name string, args ...string) (string, error) {
+	out, err := exec.Command(name, args...).CombinedOutput()
+	return string(out), err
+}
+
+// taskNetns builds and tears down per-task awsvpc network namespaces. The ENI is
+// already hot-plugged into the VM (scheduler did that); the agent finds the NIC
+// by MAC, moves it into a task netns, and DHCPs the OVN-assigned ENI IP — the
+// same way the primary NIC obtains its address.
+type taskNetns struct {
+	run     netCmdRunner
+	nicWait time.Duration
+	poll    time.Duration
+}
+
+func newTaskNetns(run netCmdRunner) *taskNetns {
+	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+}
+
+func netnsName(taskID string) string { return "ecs-" + taskID }
+func netnsPathFor(taskID string) string {
+	return netnsRunDir + "/" + netnsName(taskID)
+}
+
+// Setup creates the task netns, moves the ENI NIC (matched by MAC) into it, and
+// brings it up with a DHCP lease. It returns the netns path for containerd. On
+// any failure it tears the partial netns down so a retry starts clean.
+func (n *taskNetns) Setup(taskID, mac string) (string, error) {
+	iface, err := n.waitForNIC(mac)
+	if err != nil {
+		return "", err
+	}
+	name := netnsName(taskID)
+	if _, err := n.run.Run("ip", "netns", "add", name); err != nil {
+		return "", fmt.Errorf("netns add %s: %w", name, err)
+	}
+	hostVeth := credVethName(taskID)
+	hostIP, taskIP := credSubnet(taskID)
+	steps := [][]string{
+		{"ip", "link", "set", iface, "netns", name},
+		{"ip", "-n", name, "link", "set", "lo", "up"},
+		{"ip", "-n", name, "link", "set", iface, "up"},
+		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		// Credentials path: a veth into the host netns plus a host-routed /32 to the
+		// credential endpoint, so the container reaches 169.254.170.2 without it
+		// leaking onto the task ENI's VPC route.
+		{"ip", "link", "add", hostVeth, "type", "veth", "peer", "name", credTaskVeth, "netns", name},
+		{"ip", "addr", "add", hostIP + "/30", "dev", hostVeth},
+		{"ip", "link", "set", hostVeth, "up"},
+		{"ip", "-n", name, "addr", "add", taskIP + "/30", "dev", credTaskVeth},
+		{"ip", "-n", name, "link", "set", credTaskVeth, "up"},
+		{"ip", "-n", name, "route", "add", credEndpointCIDR, "via", hostIP},
+	}
+	for _, s := range steps {
+		if _, err := n.run.Run(s[0], s[1:]...); err != nil {
+			_ = n.Teardown(taskID)
+			return "", fmt.Errorf("netns setup %v: %w", s, err)
+		}
+	}
+	return netnsPathFor(taskID), nil
+}
+
+// credVethName is the host-side veth name for a task's credential path. Derived
+// from the taskID and kept within the 15-char interface-name limit.
+func credVethName(taskID string) string {
+	return fmt.Sprintf("ecsv%08x", crc32.ChecksumIEEE([]byte(taskID)))
+}
+
+// credSubnet derives a unique /30 in 169.254.172.0/22 for a task's credential
+// veth, returning the host-side and task-side addresses. The host side is the
+// container's gateway to the credential endpoint.
+func credSubnet(taskID string) (hostIP, taskIP string) {
+	slot := crc32.ChecksumIEEE([]byte(taskID)) % 256 // 256 /30s in the /22
+	third := 172 + (slot*4)/256
+	base := (slot * 4) % 256
+	hostIP = fmt.Sprintf("169.254.%d.%d", third, base+1)
+	taskIP = fmt.Sprintf("169.254.%d.%d", third, base+2)
+	return hostIP, taskIP
+}
+
+// Teardown deletes the task netns (releasing the moved NIC). Idempotent enough
+// for the stop path: a missing netns is reported but not fatal to the caller.
+func (n *taskNetns) Teardown(taskID string) error {
+	name := netnsName(taskID)
+	// Deleting the netns destroys the task-side veth and its host peer; remove the
+	// host side explicitly too so a leaked half from a partial setup is reaped.
+	_, _ = n.run.Run("ip", "link", "del", credVethName(taskID))
+	if _, err := n.run.Run("ip", "netns", "del", name); err != nil {
+		return fmt.Errorf("netns del %s: %w", name, err)
+	}
+	return nil
+}
+
+// waitForNIC polls until a NIC with the given MAC appears in the host netns,
+// returning its interface name. The hot-plug pipeline is async, so the device
+// may lag the assign by a few seconds.
+func (n *taskNetns) waitForNIC(mac string) (string, error) {
+	want := strings.ToLower(strings.TrimSpace(mac))
+	deadline := time.Now().Add(n.nicWait)
+	for {
+		iface, err := n.findNICByMAC(want)
+		if err == nil {
+			return iface, nil
+		}
+		if time.Now().After(deadline) {
+			return "", fmt.Errorf("nic with mac %s not found within %s", want, n.nicWait)
+		}
+		time.Sleep(n.poll)
+	}
+}
+
+// findNICByMAC parses `ip -o link` and returns the interface whose link/ether
+// matches mac.
+func (n *taskNetns) findNICByMAC(mac string) (string, error) {
+	out, err := n.run.Run("ip", "-o", "link")
+	if err != nil {
+		return "", fmt.Errorf("ip -o link: %w", err)
+	}
+	for line := range strings.SplitSeq(out, "\n") {
+		if !strings.Contains(strings.ToLower(line), mac) {
+			continue
+		}
+		// "<idx>: <ifname>: <flags> ... link/ether <mac> ..."
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		name := strings.TrimSuffix(strings.TrimSuffix(fields[1], ":"), "@")
+		if i := strings.IndexByte(name, '@'); i >= 0 {
+			name = name[:i]
+		}
+		return name, nil
+	}
+	return "", fmt.Errorf("no interface with mac %s", mac)
+}
@@ -0,0 +1,154 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeNetRunner records every command and serves canned `ip -o link` output. A
+// failOn substring forces the matching command to error so teardown paths run.
+type fakeNetRunner struct {
+	mu      sync.Mutex
+	calls   [][]string
+	linkOut string
+	failOn  string
+}
+
+func (f *fakeNetRunner) Run(name string, args ...string) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	cmd := append([]string{name}, args...)
+	f.calls = append(f.calls, cmd)
+	joined := strings.Join(cmd, " ")
+	if f.failOn != "" && strings.Contains(joined, f.failOn) {
+		return "", errors.New("boom: " + joined)
+	}
+	if name == "ip" && len(args) >= 2 && args[0] == "-o" && args[1] == "link" {
+		return f.linkOut, nil
+	}
+	return "", nil
+}
+
+func (f *fakeNetRunner) joined() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]string, len(f.calls))
+	for i, c := range f.calls {
+		out[i] = strings.Join(c, " ")
+	}
+	return out
+}
+
+func (f *fakeNetRunner) sawAny(sub string) bool {
+	for _, j := range f.joined() {
+		if strings.Contains(j, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:00:00:00 ...
+2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
+3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
+
+func newTestNetns(f *fakeNetRunner) *taskNetns {
+	n := newTaskNetns(f)
+	n.nicWait = 50 * time.Millisecond
+	n.poll = 5 * time.Millisecond
+	return n
+}
+
+func TestNetns_FindNICByMAC(t *testing.T) {
+	n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+	iface, err := n.findNICByMAC("52:54:00:de:ad:01")
+	if err != nil || iface != "eth1" {
+		t.Fatalf("want eth1, got %q err=%v", iface, err)
+	}
+	if _, err := n.findNICByMAC("52:54:00:ff:ff:ff"); err == nil {
+		t.Fatal("want error for absent mac")
+	}
+}
+
+func TestNetns_SetupSequence(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	path, err := n.Setup("t-001", "52:54:00:DE:AD:01")
+	if err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if path != netnsPathFor("t-001") {
+		t.Fatalf("want %q, got %q", netnsPathFor("t-001"), path)
+	}
+	hostIP, taskIP := credSubnet("t-001")
+	hostVeth := credVethName("t-001")
+	want := []string{
+		"ip netns add ecs-t-001",
+		"ip link set eth1 netns ecs-t-001",
+		"ip -n ecs-t-001 link set lo up",
+		"ip -n ecs-t-001 link set eth1 up",
+		"ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		"ip link add " + hostVeth + " type veth peer name ecsc0 netns ecs-t-001",
+		"ip addr add " + hostIP + "/30 dev " + hostVeth,
+		"ip -n ecs-t-001 addr add " + taskIP + "/30 dev ecsc0",
+		"ip -n ecs-t-001 route add 169.254.170.2/32 via " + hostIP,
+	}
+	for _, w := range want {
+		if !f.sawAny(w) {
+			t.Errorf("missing command %q in %v", w, f.joined())
+		}
+	}
+}
+
+func TestCredSubnet_UniquePer30(t *testing.T) {
+	hostA, taskA := credSubnet("task-a")
+	hostB, taskB := credSubnet("task-b")
+	if hostA == hostB {
+		t.Errorf("distinct tasks share a host IP: %s", hostA)
+	}
+	for _, ip := range []string{hostA, taskA, hostB, taskB} {
+		if !strings.HasPrefix(ip, "169.254.17") {
+			t.Errorf("cred IP %s outside 169.254.172.0/22", ip)
+		}
+	}
+	// host and task ends of one task sit in the same /30 (differ in last octet).
+	if hostA[:strings.LastIndexByte(hostA, '.')] != taskA[:strings.LastIndexByte(taskA, '.')] {
+		t.Errorf("host %s and task %s not in same /30", hostA, taskA)
+	}
+}
+
+func TestNetns_SetupNICNotFound(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:00:00:99"); err == nil {
+		t.Fatal("want timeout error for absent NIC")
+	}
+	if f.sawAny("netns add") {
+		t.Error("should not create netns when NIC never appears")
+	}
+}
+
+func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI, failOn: "udhcpc"}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when udhcpc fails")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown after step failure, got %v", f.joined())
+	}
+}
+
+func TestNetns_Teardown(t *testing.T) {
+	f := &fakeNetRunner{}
+	n := newTestNetns(f)
+	if err := n.Teardown("t-001"); err != nil {
+		t.Fatalf("teardown: %v", err)
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected netns del, got %v", f.joined())
+	}
+}
@@ -0,0 +1,72 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// reconcile re-adopts containers that survived an agent restart. It lists the
+// runtime's containers and, for each task still running in this cluster,
+// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
+// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// gateway's re-delivered assignments for these tasks are acked but not re-run.
+//
+// It is best-effort: a nil runner or a List error returns an empty set and the
+// agent falls back to its pre-reconcile behaviour (poll + run).
+func (a *Agent) reconcile(ctx context.Context) map[string]bool {
+	adopted := map[string]bool{}
+	if a.runner == nil {
+		return adopted
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: reconcile list failed", "err", err)
+		return adopted
+	}
+
+	// Group this cluster's running containers back into their tasks. The Assign is
+	// reconstructed from labels — enough to re-register creds and tear down netns.
+	type task struct {
+		as         *bus.Assign
+		containers []ctrruntime.Container
+	}
+	tasks := map[string]*task{}
+	for _, c := range containers {
+		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+			continue
+		}
+		taskID := c.Labels[labelTaskID]
+		if taskID == "" {
+			continue
+		}
+		t := tasks[taskID]
+		if t == nil {
+			t = &task{as: &bus.Assign{
+				TaskID:        taskID,
+				ClusterName:   a.id.ClusterName,
+				CredID:        c.Labels[labelCredID],
+				TaskRoleARN:   c.Labels[labelTaskRoleARN],
+				ENIMacAddress: c.Labels[labelENIMac],
+			}}
+			tasks[taskID] = t
+		}
+		t.containers = append(t.containers, c)
+	}
+
+	for taskID, t := range tasks {
+		a.registerTaskCreds(t.as)
+		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
+		for _, c := range t.containers {
+			name := c.Labels[labelContainerName]
+			go a.waitContainer(ctx, t.as, name, c.ID)
+			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
+		}
+		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
+		adopted[taskID] = true
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+	}
+	return adopted
+}
@@ -0,0 +1,181 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+const testTaskRole = "arn:aws:iam::111122223333:role/task"
+
+// adoptedContainer builds a runtime.Container as the reconciler would discover it,
+// stamping only the mulga.ecs.* labels that are non-empty.
+func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
+	labels := map[string]string{
+		labelTaskID:        taskID,
+		labelContainerName: name,
+		labelClusterName:   cluster,
+	}
+	if credID != "" {
+		labels[labelCredID] = credID
+	}
+	if role != "" {
+		labels[labelTaskRoleARN] = role
+	}
+	if mac != "" {
+		labels[labelENIMac] = mac
+	}
+	return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
+}
+
+func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
+	t.Helper()
+	deadline := time.After(time.Second)
+	for !slices.Contains(rt.Waits(), id) {
+		select {
+		case <-deadline:
+			t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// A running, correctly-labeled container is re-adopted: its credentials are
+// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
+func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if !adopted["t-001"] {
+		t.Fatalf("t-001 not adopted; got %v", adopted)
+	}
+	if got := a.cred.roles["t-001"]; got != testTaskRole {
+		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
+	}
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
+	}
+	waitedFor(t, rt, containerID("t-001", "web"))
+}
+
+// pollAssignments seeded with an adopted task acks its re-delivered assignment
+// but does not re-run it, while a genuinely new task still dispatches.
+func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
+	mk := func(id string) bus.Assign {
+		return bus.Assign{
+			AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
+			Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
+		}
+	}
+	adopted, fresh := mk("t-001"), mk("t-002")
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{adopted, fresh},
+		{adopted, fresh},
+		nil,
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
+
+	deadline := time.After(time.Second)
+	for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
+		select {
+		case <-deadline:
+			t.Fatal("fresh task t-002 was never dispatched")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
+	}
+	var t1Acked bool
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == "t-001" {
+				t1Acked = true
+			}
+		}
+	}
+	if !t1Acked {
+		t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
+	}
+}
+
+// Containers for another cluster or with a stopped task are not adopted.
+func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-other", "web", "other", "", "", "", true),
+			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-ok", "web", "default", "", "", "", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if len(adopted) != 1 || !adopted["t-ok"] {
+		t.Fatalf("adopted = %v, want only t-ok", adopted)
+	}
+}
+
+// A nil runner or a List error degrades to an empty set without panicking.
+func TestReconcile_DegradesGracefully(t *testing.T) {
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("nil runner: adopted = %v, want empty", got)
+	}
+
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	if got := listErr.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("list error: adopted = %v, want empty", got)
+	}
+}
+
+// taskLabels emits the optional cred/role/MAC labels only when set.
+func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
+	full := taskLabels(&bus.Assign{
+		TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
+	}, "web")
+	for k, want := range map[string]string{
+		labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
+	} {
+		if full[k] != want {
+			t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
+		}
+	}
+
+	bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
+	if len(bare) != 3 {
+		t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
+	}
+	for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
+		if _, ok := bare[k]; ok {
+			t.Errorf("bare labels unexpectedly carry %q", k)
+		}
+	}
+}
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// identity is the container instance's stable identity, assembled at boot from
+// IMDS (account, instance, AZ) and config (cluster). It is the join key the
+// scheduler uses to track this instance.
+type identity struct {
+	AccountID    string
+	ClusterName  string
+	InstanceID   string
+	AZ           string
+	Hostname     string
+	Capacity     bus.InstanceCapacity
+	AgentVersion string
+}
+
+// registrar registers the host as a container instance through the gateway.
+type registrar struct {
+	cp controlPlane
+	id identity
+}
+
+func newRegistrar(cp controlPlane, id identity) *registrar {
+	return &registrar{cp: cp, id: id}
+}
+
+// Register calls the gateway's RegisterContainerInstance. The scheduler records
+// (or refreshes) the container instance on receipt.
+func (r *registrar) Register() error {
+	if err := r.cp.Register(r.id); err != nil {
+		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
+	}
+	return nil
+}
@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// fakeCP is a test double for the gateway control-plane. It records registers and
+// task-state reports, and replays a scripted queue of poll responses.
+type fakeCP struct {
+	mu sync.Mutex
+
+	registers    int
+	states       []bus.TaskState
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int
+
+	registerErr bool
+	submitErr   bool
+}
+
+var _ controlPlane = (*fakeCP)(nil)
+
+func (f *fakeCP) Register(_ identity) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.registerErr {
+		return errors.New("register boom")
+	}
+	f.registers++
+	return nil
+}
+
+func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.submitErr {
+		return errors.New("submit boom")
+	}
+	f.states = append(f.states, st)
+	return nil
+}
+
+func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
+	f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
+	var out []bus.Assign
+	if f.pollCalls < len(f.pollReplies) {
+		out = f.pollReplies[f.pollCalls]
+	}
+	var stops []bus.StopDirective
+	if f.pollCalls < len(f.stopReplies) {
+		stops = f.stopReplies[f.pollCalls]
+	}
+	f.pollCalls++
+	return out, stops, nil
+}
+
+func (f *fakeCP) registerCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.registers
+}
+
+func (f *fakeCP) taskStates() []bus.TaskState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]bus.TaskState(nil), f.states...)
+}
+
+func (f *fakeCP) acks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollAcks...)
+}
+
+func (f *fakeCP) stopAcks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollStopAcks...)
+}
+
+func testIdentity() identity {
+	return identity{
+		AccountID:    "123456789012",
+		ClusterName:  "default",
+		InstanceID:   "i-abc123",
+		AZ:           "us-east-1a",
+		Hostname:     "ecs-host",
+		Capacity:     bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
+		AgentVersion: "test",
+	}
+}
+
+func TestRegistrar_Register(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("registers = %d, want 1", cp.registerCount())
+	}
+}
+
+func TestRegistrar_RegisterError(t *testing.T) {
+	cp := &fakeCP{registerErr: true}
+	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+		t.Fatal("expected register error")
+	}
+}
+
+func TestHeartbeater_Beat(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("beat should re-register once, got %d", cp.registerCount())
+	}
+}
+
+func TestHeartbeater_DefaultInterval(t *testing.T) {
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	if h.interval != defaultHeartbeat {
+		t.Errorf("interval = %v, want default", h.interval)
+	}
+}
+
+func TestHeartbeater_RunStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	done := make(chan struct{})
+	go func() { h.Run(ctx); close(done) }()
+
+	time.Sleep(25 * time.Millisecond)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("Run did not stop on context cancel")
+	}
+	if cp.registerCount() == 0 {
+		t.Error("expected at least one re-register before cancel")
+	}
+}
@@ -0,0 +1,118 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/remotes"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/distribution/reference"
+)
+
+// ecsNamespace isolates ecs-agent images/containers in containerd, mirroring the
+// "moby"/"k8s.io" convention other runtimes use.
+const ecsNamespace = "ecs"
+
+// servingProbeTimeout bounds the boot-time daemon liveness check.
+const servingProbeTimeout = 3 * time.Second
+
+// containerdPuller drives a real containerd daemon over its unix socket.
+type containerdPuller struct {
+	client *containerd.Client
+}
+
+var _ ImagePuller = (*containerdPuller)(nil)
+
+// New connects to the containerd daemon at socket (e.g.
+// "/run/containerd/containerd.sock") and returns an ImagePuller. The containerd
+// client dials lazily, so New actively probes the daemon with IsServing and
+// returns an error if it is unreachable — otherwise a dead socket would only
+// surface on the first pull, and the agent's boot-time "containerd unavailable"
+// log would never fire.
+func New(socket string) (Runtime, error) {
+	client, err := containerd.New(socket, containerd.WithDefaultNamespace(ecsNamespace))
+	if err != nil {
+		return nil, fmt.Errorf("connect containerd %s: %w", socket, err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), servingProbeTimeout)
+	defer cancel()
+	serving, err := client.IsServing(ctx)
+	if err != nil || !serving {
+		_ = client.Close()
+		return nil, fmt.Errorf("containerd %s not serving: %w", socket, err)
+	}
+	return &containerdPuller{client: client}, nil
+}
+
+// Pull resolves credentials via r, then pulls and unpacks the image.
+func (p *containerdPuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Ref)
+	if err != nil {
+		return Image{}, err
+	}
+
+	resolver, err := newDockerResolver(ctx, ref, r)
+	if err != nil {
+		return Image{}, err
+	}
+
+	img, err := p.client.Pull(ctx, ref,
+		containerd.WithPullUnpack,
+		containerd.WithResolver(resolver),
+	)
+	if err != nil {
+		return Image{}, fmt.Errorf("pull %s: %w", ref, err)
+	}
+
+	target := img.Target()
+	return Image{
+		Ref:    ref,
+		Digest: target.Digest.String(),
+		Size:   target.Size,
+	}, nil
+}
+
+// normalizeRef expands a task-definition image string into the fully-qualified
+// reference containerd's resolver requires: a bare "nginx:alpine" becomes
+// "docker.io/library/nginx:alpine" and an untagged ref gets ":latest". Without
+// this, containerd parses the bare ref as "dummy://nginx:alpine" and rejects it.
+func normalizeRef(ref string) (string, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", fmt.Errorf("parse image ref %q: %w", ref, err)
+	}
+	return reference.TagNameOnly(named).String(), nil
+}
+
+// Close releases the containerd client connection.
+func (p *containerdPuller) Close() error {
+	if p.client == nil {
+		return nil
+	}
+	return p.client.Close()
+}
+
+// newDockerResolver builds a containerd remote resolver whose per-host
+// credentials come from r (the ECR token resolver). Credentials are fetched once
+// for ref and reused for every host containerd asks about during the pull.
+func newDockerResolver(ctx context.Context, ref string, r Resolver) (remotes.Resolver, error) {
+	creds := func(string) (string, string, error) { return "", "", nil }
+	if r != nil {
+		user, pass, _, err := r.Authorize(ctx, ref)
+		if err != nil {
+			return nil, fmt.Errorf("authorize %s: %w", ref, err)
+		}
+		creds = func(string) (string, string, error) { return user, pass, nil }
+	}
+
+	authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+	return docker.NewResolver(docker.ResolverOptions{
+		Hosts: docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer)),
+	}), nil
+}
@@ -0,0 +1,142 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cio"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+var _ Runner = (*containerdPuller)(nil)
+
+// Run creates a container from an already-pulled image and starts its task. The
+// container joins the host network namespace (bridge/CNI task networking is a
+// later sprint) and carries spec.Labels for the reboot reconciler. id must be
+// unique within the "ecs" namespace.
+func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (string, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Image)
+	if err != nil {
+		return "", err
+	}
+	image, err := p.client.GetImage(ctx, ref)
+	if err != nil {
+		return "", fmt.Errorf("get image %s: %w", ref, err)
+	}
+
+	specOpts := []oci.SpecOpts{
+		oci.WithImageConfig(image),
+		oci.WithHostHostsFile,
+		oci.WithHostResolvconf,
+	}
+	if spec.NetnsPath != "" {
+		// awsvpc: join the task ENI netns built by the agent.
+		specOpts = append(specOpts, oci.WithLinuxNamespace(specs.LinuxNamespace{
+			Type: specs.NetworkNamespace,
+			Path: spec.NetnsPath,
+		}))
+	} else {
+		// bridge/host: share the VM (host) netns.
+		specOpts = append(specOpts, oci.WithHostNamespace(specs.NetworkNamespace))
+	}
+	if len(spec.Command) > 0 {
+		specOpts = append(specOpts, oci.WithProcessArgs(spec.Command...))
+	}
+	if len(spec.Env) > 0 {
+		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
+	}
+
+	container, err := p.client.NewContainer(ctx, id,
+		containerd.WithNewSnapshot(id+"-snapshot", image),
+		containerd.WithNewSpec(specOpts...),
+		containerd.WithContainerLabels(spec.Labels),
+	)
+	if err != nil {
+		return "", fmt.Errorf("create container %s: %w", id, err)
+	}
+
+	task, err := container.NewTask(ctx, cio.NullIO)
+	if err != nil {
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("create task %s: %w", id, err)
+	}
+	if err := task.Start(ctx); err != nil {
+		_, _ = task.Delete(ctx)
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("start task %s: %w", id, err)
+	}
+	return id, nil
+}
+
+// Wait blocks until the container's task exits and returns its exit code.
+func (p *containerdPuller) Wait(ctx context.Context, containerID string) (RunStatus, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("load container %s: %w", containerID, err)
+	}
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("attach task %s: %w", containerID, err)
+	}
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("wait task %s: %w", containerID, err)
+	}
+	status := <-statusC
+	code, _, err := status.Result()
+	return RunStatus{ExitCode: int(code)}, err
+}
+
+// List enumerates every container in the ecs namespace with its labels and
+// whether its task is running, so the agent can re-adopt live containers after a
+// restart. Per-container label/task errors are treated as "not running" rather
+// than failing the whole list.
+func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	containers, err := p.client.Containers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list containers: %w", err)
+	}
+	out := make([]Container, 0, len(containers))
+	for _, c := range containers {
+		labels, _ := c.Labels(ctx)
+		running := false
+		if task, terr := c.Task(ctx, nil); terr == nil {
+			if status, serr := task.Status(ctx); serr == nil {
+				running = status.Status == containerd.Running
+			}
+		}
+		out = append(out, Container{ID: c.ID(), Labels: labels, Running: running})
+	}
+	return out, nil
+}
+
+// Remove kills and deletes the container's task, then the container + snapshot.
+func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return nil //nolint:nilerr // load failure means the container is already gone
+	}
+	if task, terr := container.Task(ctx, nil); terr == nil {
+		_, _ = task.Delete(ctx, containerd.WithProcessKill)
+	}
+	return container.Delete(ctx, containerd.WithSnapshotCleanup)
+}
+
+// envSlice renders an env map as sorted "K=V" entries for deterministic specs.
+func envSlice(env map[string]string) []string {
+	out := make([]string, 0, len(env))
+	for k, v := range env {
+		out = append(out, k+"="+v)
+	}
+	sort.Strings(out)
+	return out
+}
@@ -0,0 +1,82 @@
+// Package runtime is the ecs-agent's container-runtime seam. The agent pulls
+// task images and (from Sprint 4e) runs containers through an ImagePuller; the
+// real implementation drives containerd, but the interface lets tests and the
+// scheduler-only build run against a fake.
+package runtime
+
+import "context"
+
+// PullSpec describes one image to pull.
+type PullSpec struct {
+	// Ref is the full image reference, e.g.
+	// "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:latest".
+	Ref string
+}
+
+// Image identifies a pulled image in the runtime's content store.
+type Image struct {
+	Ref    string
+	Digest string
+	Size   int64
+}
+
+// Resolver supplies registry credentials for a given image reference. The
+// ecs-agent backs this with ECR GetAuthorizationToken (see ecrresolver.go).
+type Resolver interface {
+	// Authorize returns the username, password and registry endpoint to use for
+	// ref. An empty user/pass means anonymous pull.
+	Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error)
+}
+
+// ImagePuller pulls images into the local runtime. Close releases the runtime
+// client connection.
+type ImagePuller interface {
+	Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error)
+	Close() error
+}
+
+// RunSpec describes a single container the agent must create and start. v1 runs
+// the container in the host network namespace (bridge/CNI task networking lands
+// in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
+// reconciler.
+type RunSpec struct {
+	Image   string
+	Command []string
+	Env     map[string]string
+	Labels  map[string]string
+	// NetnsPath, when set, joins the container to a pre-built network namespace
+	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
+	// bridge/host mode behaviour.
+	NetnsPath string
+}
+
+// RunStatus is a finished container's outcome.
+type RunStatus struct {
+	ExitCode int
+}
+
+// Container is a container the runtime already knows about, discovered by List.
+// Labels carry the mulga.ecs.* task identity so the agent can re-adopt running
+// containers after a restart; Running reports whether its task is live.
+type Container struct {
+	ID      string
+	Labels  map[string]string
+	Running bool
+}
+
+// Runner creates and starts containers from already-pulled images. id is a
+// caller-unique container ID; Wait blocks until the container exits; Remove
+// tears down the container + its task; List enumerates known containers so the
+// reboot reconciler can re-adopt the ones still running.
+type Runner interface {
+	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
+	Wait(ctx context.Context, containerID string) (RunStatus, error)
+	Remove(ctx context.Context, containerID string) error
+	List(ctx context.Context) ([]Container, error)
+}
+
+// Runtime is the full container runtime the ecs-agent drives: pull + run.
+type Runtime interface {
+	ImagePuller
+	Runner
+}
@@ -0,0 +1,125 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+// FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
+// It records pull requests and replays a programmed result or error.
+type FakePuller struct {
+	mu     sync.Mutex
+	Pulls  []PullSpec
+	Authzd []string
+	Result Image
+	Err    error
+	Closed bool
+	OnPull func(PullSpec) (Image, error)
+
+	// Run bookkeeping.
+	Runs     []RunSpec
+	RunErr   error
+	WaitCode int
+	WaitErr  error
+	Waited   []string
+	Removed  []string
+	OnRun    func(string, RunSpec) (string, error)
+
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
+	// Listed records that List was called.
+	Containers []Container
+	ListErr    error
+	Listed     bool
+}
+
+var (
+	_ ImagePuller = (*FakePuller)(nil)
+	_ Runner      = (*FakePuller)(nil)
+)
+
+// Pull records the spec, exercises the resolver, and returns the programmed
+// result. If OnPull is set it takes precedence over Result/Err.
+func (f *FakePuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	f.mu.Lock()
+	f.Pulls = append(f.Pulls, spec)
+	f.mu.Unlock()
+
+	if r != nil {
+		user, _, _, err := r.Authorize(ctx, spec.Ref)
+		if err != nil {
+			return Image{}, fmt.Errorf("authorize %s: %w", spec.Ref, err)
+		}
+		f.mu.Lock()
+		f.Authzd = append(f.Authzd, user)
+		f.mu.Unlock()
+	}
+
+	if f.OnPull != nil {
+		return f.OnPull(spec)
+	}
+	if f.Err != nil {
+		return Image{}, f.Err
+	}
+	res := f.Result
+	if res.Ref == "" {
+		res.Ref = spec.Ref
+	}
+	return res, nil
+}
+
+// Close marks the puller closed.
+func (f *FakePuller) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Closed = true
+	return nil
+}
+
+// Run records the spec and returns id (or the programmed result/error).
+func (f *FakePuller) Run(_ context.Context, id string, spec RunSpec) (string, error) {
+	f.mu.Lock()
+	f.Runs = append(f.Runs, spec)
+	f.mu.Unlock()
+	if f.OnRun != nil {
+		return f.OnRun(id, spec)
+	}
+	if f.RunErr != nil {
+		return "", f.RunErr
+	}
+	return id, nil
+}
+
+// Wait records the container ID and returns the programmed exit code/error.
+func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, error) {
+	f.mu.Lock()
+	f.Waited = append(f.Waited, containerID)
+	f.mu.Unlock()
+	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
+}
+
+// Remove records the container ID.
+func (f *FakePuller) Remove(_ context.Context, containerID string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Removed = append(f.Removed, containerID)
+	return nil
+}
+
+// Waits returns a copy of the container IDs passed to Wait, for assertions.
+func (f *FakePuller) Waits() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.Waited...)
+}
+
+// List replays the programmed container set (or error).
+func (f *FakePuller) List(_ context.Context) ([]Container, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Listed = true
+	if f.ListErr != nil {
+		return nil, f.ListErr
+	}
+	return f.Containers, nil
+}
@@ -0,0 +1,89 @@
+package runtime
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// staticResolver is a test Resolver returning fixed creds or an error.
+type staticResolver struct {
+	user, pass, endpoint string
+	err                  error
+}
+
+func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
+	return s.user, s.pass, s.endpoint, s.err
+}
+
+func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
+	f := &FakePuller{}
+	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
+	if err != nil {
+		t.Fatalf("Pull: %v", err)
+	}
+	if img.Ref != "repo/app:1" {
+		t.Errorf("Ref defaulted wrong: %q", img.Ref)
+	}
+	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
+		t.Errorf("Pulls = %+v", f.Pulls)
+	}
+	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
+		t.Errorf("Authzd = %+v", f.Authzd)
+	}
+}
+
+func TestFakePuller_ResolverError(t *testing.T) {
+	f := &FakePuller{}
+	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
+	if err == nil {
+		t.Fatal("expected resolver error to propagate")
+	}
+}
+
+func TestFakePuller_ProgrammedError(t *testing.T) {
+	f := &FakePuller{Err: errors.New("pull failed")}
+	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
+		t.Fatal("expected programmed error")
+	}
+}
+
+func TestFakePuller_OnPullOverrides(t *testing.T) {
+	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
+	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
+		return want, nil
+	}}
+	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
+	if err != nil || got != want {
+		t.Fatalf("got %+v, %v; want %+v", got, err, want)
+	}
+}
+
+func TestFakePuller_Close(t *testing.T) {
+	f := &FakePuller{}
+	if err := f.Close(); err != nil || !f.Closed {
+		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
+	}
+}
+
+func TestNormalizeRef(t *testing.T) {
+	cases := map[string]string{
+		"nginx:alpine":      "docker.io/library/nginx:alpine",
+		"nginx":             "docker.io/library/nginx:latest",
+		"ghcr.io/org/app:1": "ghcr.io/org/app:1",
+		"123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1": "123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1",
+	}
+	for in, want := range cases {
+		got, err := normalizeRef(in)
+		if err != nil {
+			t.Errorf("normalizeRef(%q): %v", in, err)
+			continue
+		}
+		if got != want {
+			t.Errorf("normalizeRef(%q) = %q, want %q", in, got, want)
+		}
+	}
+	if _, err := normalizeRef("Bad::Ref"); err == nil {
+		t.Error("normalizeRef accepted invalid ref")
+	}
+}
@@ -0,0 +1,294 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"maps"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// pollAssignments drains the instance's assignment inbox on a ticker until ctx is
+// cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
+// on a poll are acked on the next poll so the gateway can drop them — a crash
+// before ack re-delivers (at-least-once), matching ACS. A failed poll is logged
+// and retried on the next tick rather than killing the loop. dispatched is seeded
+// by the startup reconcile with tasks already adopted from running containers, so
+// their re-delivered assignments are acked but not re-run.
+func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool) {
+	ticker := time.NewTicker(a.cfg.PollInterval)
+	defer ticker.Stop()
+
+	stopping := map[string]bool{}
+	var ackAssigns, ackStops []string
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			assigns, stops, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackAssigns, ackStops)
+			if err != nil {
+				slog.Warn("ecs-agent: poll assignments failed", "err", err)
+				continue
+			}
+			// Dispatch stops first so a task stopped in the same poll it was assigned
+			// is never started.
+			ackStops = ackStops[:0]
+			for i := range stops {
+				sd := stops[i]
+				if !stopping[sd.TaskID] {
+					stopping[sd.TaskID] = true
+					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
+					go a.stopTask(ctx, sd)
+				}
+				ackStops = append(ackStops, sd.TaskID)
+			}
+			ackAssigns = ackAssigns[:0]
+			for i := range assigns {
+				as := assigns[i]
+				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
+					dispatched[as.TaskID] = true
+					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
+					go a.runTask(ctx, &as)
+				}
+				ackAssigns = append(ackAssigns, as.TaskID)
+			}
+		}
+	}
+}
+
+// stopTask reaps a task's containers on a control-plane stop directive: it lists
+// the runtime's containers, removes the ones labeled with this task (kill +
+// delete, releasing host ports/netns), then reports the task STOPPED with the
+// directive's reason. Idempotent — a task with no live containers still reports
+// STOPPED so the scheduler releases its capacity.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
+	if a.runner == nil {
+		return
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
+		return
+	}
+	statuses := make([]bus.ContainerStatus, 0)
+	for _, c := range containers {
+		if c.Labels[labelTaskID] != sd.TaskID {
+			continue
+		}
+		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
+			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	reason := sd.Reason
+	if reason == "" {
+		reason = "Task stopped"
+	}
+	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
+	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+}
+
+// runTask pulls each container image, starts the containers, and reports task
+// state on the bus. A missing runtime or any per-container failure reports the
+// task STOPPED with a reason; success reports RUNNING and waits for exit.
+func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
+	if a.puller == nil || a.runner == nil {
+		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
+		return
+	}
+
+	// awsvpc: build the task netns from the hot-plugged ENI before any container.
+	netnsPath, err := a.setupTaskNetns(as)
+	if err != nil {
+		slog.Error("ecs-agent: task netns setup failed", "task", as.TaskID, "err", err)
+		a.reportTaskState(as, bus.TaskStatusStopped, "network setup failed: "+err.Error(), nil)
+		return
+	}
+
+	credID := a.registerTaskCreds(as)
+	resolver := a.pullResolver(as)
+
+	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
+	for _, c := range as.Containers {
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
+			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
+			return
+		}
+
+		cid := containerID(as.TaskID, c.Name)
+		spec := ctrruntime.RunSpec{
+			Image:     c.Image,
+			Command:   c.Command,
+			Env:       withCredEnv(c.Environment, credID),
+			Labels:    taskLabels(as, c.Name),
+			NetnsPath: netnsPath,
+		}
+		id, err := a.runner.Run(ctx, cid, spec)
+		if err != nil {
+			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
+			return
+		}
+		statuses = append(statuses, bus.ContainerStatus{Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id})
+		go a.waitContainer(ctx, as, c.Name, id)
+	}
+
+	a.reportTaskState(as, bus.TaskStatusRunning, "", statuses)
+	slog.Info("ecs-agent: task running", "task", as.TaskID, "containers", len(statuses))
+}
+
+// waitContainer blocks until a container exits, then reports the task STOPPED
+// with the exit code. v1 stops the whole task when its first container exits.
+func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
+	status, err := a.runner.Wait(ctx, containerID)
+	if err != nil {
+		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
+		return
+	}
+	exit := status.ExitCode
+	a.teardownTaskNetns(as)
+	statuses := []bus.ContainerStatus{{Name: name, Status: bus.TaskStatusStopped, ContainerID: containerID, ExitCode: &exit}}
+	a.reportTaskState(as, bus.TaskStatusStopped, fmt.Sprintf("container %s exited (%d)", name, exit), statuses)
+	slog.Info("ecs-agent: task stopped", "task", as.TaskID, "container", name, "exitCode", exit)
+}
+
+// setupTaskNetns builds the awsvpc task netns from the hot-plugged ENI. Bridge/
+// host tasks (no ENI MAC) and a nil controller are no-ops returning an empty path.
+func (a *Agent) setupTaskNetns(as *bus.Assign) (string, error) {
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return "", nil
+	}
+	return a.netns.Setup(as.TaskID, as.ENIMacAddress)
+}
+
+// teardownTaskNetns releases the task's IAM credential registration, then removes
+// the awsvpc task netns (the latter is a no-op for bridge/host tasks). It runs at
+// every task-stop path so credentials never outlive the task that owns them.
+func (a *Agent) teardownTaskNetns(as *bus.Assign) {
+	if a.cred != nil {
+		a.cred.Deregister(taskCredID(as))
+	}
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return
+	}
+	if err := a.netns.Teardown(as.TaskID); err != nil {
+		slog.Warn("ecs-agent: task netns teardown", "task", as.TaskID, "err", err)
+	}
+}
+
+// pullResolver returns the ECR resolver used for a task's image pulls. When the
+// assign carries an execution role, it authorizes pulls as that role (assumed
+// over the gateway); an empty role or a nil credential endpoint (unit tests)
+// falls back to the instance-role resolver, so a pull is never worse off.
+func (a *Agent) pullResolver(as *bus.Assign) ctrruntime.Resolver {
+	if as.ExecutionRoleARN == "" || a.cred == nil {
+		return a.resolver
+	}
+	prov := a.cred.AssumeProvider(as.ExecutionRoleARN, sessionName(as.TaskID))
+	return newLazyECRResolver(prov, a.cfg.Region, a.cfg.GatewayURL, a.cfg.GatewayCA)
+}
+
+// registerTaskCreds registers the task's credID -> role mapping with the
+// credential endpoint and returns the credID (empty when the task has no role).
+func (a *Agent) registerTaskCreds(as *bus.Assign) string {
+	credID := taskCredID(as)
+	if credID == "" || a.cred == nil {
+		return credID
+	}
+	a.cred.Register(credID, as.TaskRoleARN)
+	return credID
+}
+
+// taskCredID is the credential ID a task's containers fetch credentials under:
+// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
+// carries no IAM role.
+func taskCredID(as *bus.Assign) string {
+	if as.TaskRoleARN == "" {
+		return ""
+	}
+	if as.CredID != "" {
+		return as.CredID
+	}
+	return as.TaskID
+}
+
+// withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for
+// credID, copying the map so the assign's container env is left untouched. A
+// blank credID (no task role) returns env unchanged.
+func withCredEnv(env map[string]string, credID string) map[string]string {
+	if credID == "" {
+		return env
+	}
+	out := make(map[string]string, len(env)+1)
+	maps.Copy(out, env)
+	out["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = credRelativeURI(credID)
+	return out
+}
+
+// reportTaskState reports a task transition through the gateway's
+// SubmitTaskStateChange action.
+func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
+	if a.cp == nil {
+		return
+	}
+	msg := bus.TaskState{
+		AccountID:   a.id.AccountID,
+		ClusterName: a.id.ClusterName,
+		InstanceID:  a.id.InstanceID,
+		TaskID:      as.TaskID,
+		LastStatus:  status,
+		Containers:  containers,
+		Reason:      reason,
+		ReportedAt:  time.Now().UTC(),
+	}
+	if err := a.cp.SubmitTaskState(msg); err != nil {
+		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
+	}
+}
+
+// containerID composes a containerd-valid container ID from a task + container.
+func containerID(taskID, name string) string {
+	return fmt.Sprintf("%s-%s", taskID, name)
+}
+
+// mulga.ecs.* label keys stamped on every container. The reboot reconciler reads
+// them back to re-associate running containers with their task on restart; the
+// cred/role/MAC labels make a container self-describing enough to re-register its
+// credentials and tear down its netns without the original assignment.
+const (
+	labelTaskID        = "mulga.ecs.taskID"
+	labelContainerName = "mulga.ecs.containerName"
+	labelClusterName   = "mulga.ecs.clusterName"
+	labelCredID        = "mulga.ecs.credID"
+	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
+	labelENIMac        = "mulga.ecs.eniMac"
+)
+
+// taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC
+// labels are omitted when empty so a container carries only what it needs.
+func taskLabels(as *bus.Assign, name string) map[string]string {
+	labels := map[string]string{
+		labelTaskID:        as.TaskID,
+		labelContainerName: name,
+		labelClusterName:   as.ClusterName,
+	}
+	if credID := taskCredID(as); credID != "" {
+		labels[labelCredID] = credID
+	}
+	if as.TaskRoleARN != "" {
+		labels[labelTaskRoleARN] = as.TaskRoleARN
+	}
+	if as.ENIMacAddress != "" {
+		labels[labelENIMac] = as.ENIMacAddress
+	}
+	return labels
+}
@@ -0,0 +1,403 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func testAssign() *bus.Assign {
+	return &bus.Assign{
+		AccountID:   "123456789012",
+		ClusterName: "default",
+		InstanceID:  "i-abc123",
+		TaskID:      "t-001",
+		Containers: []bus.AssignContainer{
+			{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"},
+				Environment: map[string]string{"FOO": "bar"}},
+		},
+	}
+}
+
+func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
+	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+}
+
+// runTask with no runtime reports the task STOPPED instead of crashing.
+func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED state, got %+v", st)
+	}
+	if st[0].Reason == "" {
+		t.Error("STOPPED state missing reason")
+	}
+}
+
+// Happy path: pull + run reports RUNNING with the container ID; Wait blocking
+// (forced error) means no STOPPED overwrite, so we can assert the RUNNING frame.
+func TestRunTask_PullRunReportsRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if len(rt.Pulls) != 1 || rt.Pulls[0].Ref != "registry/web:1" {
+		t.Fatalf("expected one pull of registry/web:1, got %+v", rt.Pulls)
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].Image != "registry/web:1" {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].Labels["mulga.ecs.taskID"] != "t-001" {
+		t.Errorf("missing task label: %+v", rt.Runs[0].Labels)
+	}
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	if len(st[len(st)-1].Containers) != 1 || st[len(st)-1].Containers[0].ContainerID != "t-001-web" {
+		t.Errorf("RUNNING state container wrong: %+v", st[len(st)-1].Containers)
+	}
+}
+
+// A pull failure reports STOPPED and never starts the container.
+func TestRunTask_PullFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Err: errors.New("pull denied")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 0 {
+		t.Errorf("container should not run after pull failure, got %+v", rt.Runs)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// A run failure reports STOPPED.
+func TestRunTask_RunFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{RunErr: errors.New("start boom")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// On container exit, waitContainer reports STOPPED with the exit code.
+func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitCode: 7}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	deadline := time.After(time.Second)
+	for {
+		st := cp.taskStates()
+		if len(st) > 0 && st[len(st)-1].LastStatus == bus.TaskStatusStopped {
+			last := st[len(st)-1]
+			if len(last.Containers) != 1 || last.Containers[0].ExitCode == nil || *last.Containers[0].ExitCode != 7 {
+				t.Fatalf("want exit code 7, got %+v", last.Containers)
+			}
+			return
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("no STOPPED state after exit; got %+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// pollAssignments dispatches each assign once and acks it on the next poll.
+func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{*as}, // poll 1: one pending assign
+		{*as}, // poll 2: still present (not yet acked when poll 2 was issued)
+		nil,   // poll 3: gateway dropped it after the ack
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	// runTask reports RUNNING through the (mutex-guarded) control plane. With the
+	// assign returned on two consecutive polls, dedup means exactly one dispatch →
+	// exactly one RUNNING report.
+	deadline := time.After(time.Second)
+	for {
+		if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("assign was never dispatched (no RUNNING report)")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 1 {
+		t.Errorf("task dispatched %d times, want exactly 1 (dedup on taskID)", running)
+	}
+	// The taskID must appear in a later poll's ack list.
+	acked := false
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("task %s was never acked; acks=%v", as.TaskID, cp.acks())
+	}
+}
+
+// stopTask reaps the task's labeled containers and reports STOPPED with the
+// directive reason and container status.
+func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		}},
+		{ID: "t-999-other", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-999", "mulga.ecs.containerName": "web",
+		}},
+	}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "bye" {
+		t.Errorf("want reason %q, got %q", "bye", st[0].Reason)
+	}
+	if len(st[0].Containers) != 1 || st[0].Containers[0].Status != bus.TaskStatusStopped {
+		t.Errorf("want one STOPPED container, got %+v", st[0].Containers)
+	}
+}
+
+// A stop for a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity.
+func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "gone"})
+
+	if len(rt.Removed) != 0 {
+		t.Errorf("nothing to remove, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+}
+
+// A stop delivered by poll is dispatched once, acked in the stop-ack list, and
+// suppresses a same-poll assign for the same task.
+func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
+		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for {
+		if stopped := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusStopped); stopped >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("stop never reaped; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	// The assign for the stopping task must never have started a container.
+	if len(rt.Runs) != 0 {
+		t.Errorf("assign for a stopping task should not run, got %+v", rt.Runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopping task should never report RUNNING, got %d", running)
+	}
+	acked := false
+	for _, ack := range cp.stopAcks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("stop %s was never acked; stopAcks=%v", as.TaskID, cp.stopAcks())
+	}
+}
+
+// stopTask with no runtime is a no-op: nothing to reap, no state reported.
+func TestStopTask_NilRunnerNoOp(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A List failure aborts the reap without reporting a (false) STOPPED state.
+func TestStopTask_ListErrorNoReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("list boom")}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("list error should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A directive with no reason reaps the matching container and reports STOPPED with
+// the default reason.
+func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "Task stopped" {
+		t.Errorf("want default reason %q, got %q", "Task stopped", st[0].Reason)
+	}
+}
+
+func countStatus(states []bus.TaskState, taskID, status string) int {
+	n := 0
+	for _, st := range states {
+		if st.TaskID == taskID && st.LastStatus == status {
+			n++
+		}
+	}
+	return n
+}
+
+// awsvpc: an assign carrying an ENI MAC builds the task netns and passes its
+// path to the container RunSpec.
+func TestRunTask_AwsvpcBuildsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	as := testAssign()
+	as.ENIMacAddress = "52:54:00:de:ad:01"
+	a.runTask(context.Background(), as)
+
+	if !f.sawAny("ip netns add ecs-t-001") {
+		t.Fatalf("expected task netns build, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != netnsPathFor("t-001") {
+		t.Fatalf("want NetnsPath %q on RunSpec, got %+v", netnsPathFor("t-001"), rt.Runs)
+	}
+}
+
+// bridge/host: no ENI MAC means no netns is built and the container runs in the
+// host (VM) netns (empty NetnsPath).
+func TestRunTask_NoMacSkipsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	a.runTask(context.Background(), testAssign())
+
+	if len(f.joined()) != 0 {
+		t.Fatalf("expected no netns commands, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != "" {
+		t.Fatalf("want empty NetnsPath, got %+v", rt.Runs)
+	}
+}
+
+func TestContainerID(t *testing.T) {
+	if got := containerID("t-1", "web"); got != "t-1-web" {
+		t.Errorf("containerID = %q, want t-1-web", got)
+	}
+}
+
+// sentinelResolver is an identity-comparable Resolver for pullResolver tests.
+type sentinelResolver struct{}
+
+func (sentinelResolver) Authorize(context.Context, string) (string, string, string, error) {
+	return "", "", "", nil
+}
+
+// TestPullResolver covers siv-459: an assign with no execution role (or no
+// credential endpoint) uses the instance resolver; with both, a distinct
+// exec-role resolver is built.
+func TestPullResolver(t *testing.T) {
+	fake := sentinelResolver{}
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+
+	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
+		t.Fatalf("no execution role: want instance resolver fallback")
+	}
+	execAssign := &bus.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
+	if got := a.pullResolver(execAssign); got != fake {
+		t.Fatalf("nil cred endpoint: want instance resolver fallback")
+	}
+
+	a.cred = &credEndpoint{}
+	got := a.pullResolver(execAssign)
+	if got == nil || got == fake {
+		t.Fatalf("execution role + cred endpoint: want distinct exec-role resolver, got %v", got)
+	}
+}
@@ -17,10 +17,14 @@
 // Usage:
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
+//	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//
+// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
+// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
-// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
+// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
 package main

 import (
@@ -68,6 +72,7 @@ func main() {
 		accountID   string
 		clusterName string
 		resource    string
+		instanceID  string
 	)

 	flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
@@ -77,7 +82,8 @@ func main() {
 	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
 	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
-	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons")
+	flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
+	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()

 	switch {
@@ -85,15 +91,29 @@ func main() {
 		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case clusterName == "":
 		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case resource != "addons":
-		fatal("--resource must be addons")
+	case resource != "addons" && resource != "recovery":
+		fatal("--resource must be addons or recovery")
+	case resource == "recovery" && instanceID == "":
+		fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}

 	client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
 	if err != nil {
 		fatal(fmt.Sprintf("build gateway client: %v", err))
 	}
-	path := "/clusters/" + clusterName + "/internal-addons/" + accountID
+
+	var (
+		path string
+		emit func(io.Writer, []byte) error
+	)
+	switch resource {
+	case "recovery":
+		path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
+		emit = emitRecoveryTSV
+	default:
+		path = "/clusters/" + clusterName + "/internal-addons/" + accountID
+		emit = emitAddonsTSV
+	}

 	var lastErr error
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
@@ -106,7 +126,7 @@ func main() {
 			}
 			continue
 		}
-		if werr := emitAddonsTSV(os.Stdout, body); werr != nil {
+		if werr := emit(os.Stdout, body); werr != nil {
 			fatal(fmt.Sprintf("render response: %v", werr))
 		}
 		return
@@ -133,6 +153,36 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 	return w.Flush()
 }

+// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
+// here to keep this tiny VM binary free of the handler package's dependency graph.
+type recoveryDirective struct {
+	Epoch    int64  `json:"epoch"`
+	Action   string `json:"action"`
+	Snapshot string `json:"snapshot"`
+}
+
+type internalRecoveryResponse struct {
+	Directive recoveryDirective `json:"directive"`
+}
+
+// emitRecoveryTSV decodes the recovery directive and writes a single
+// epoch\taction\tsnapshot line the on-VM k3s-recovery agent consumes without a
+// JSON parser. An unset action defaults to "none" (steady state).
+func emitRecoveryTSV(out io.Writer, body []byte) error {
+	var resp internalRecoveryResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-recovery response: %w", err)
+	}
+	d := resp.Directive
+	if d.Action == "" {
+		d.Action = "none"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot); err != nil {
+		return err
+	}
+	return nil
+}
+
 func fatal(msg string) {
 	slog.Error("eks-gateway-fetch: " + msg)
 	os.Exit(1)
@@ -0,0 +1,245 @@
+// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
+// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
+//
+// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
+// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
+// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
+// serving cert with that CA is therefore what lets the agent validate the
+// reverse-tunnel endpoint with no separate CA distribution.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
+// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
+// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
+// and cache it under -dir so the OpenRC service is byte-stable across restarts.
+//
+// Usage:
+//
+//	eks-konnectivity-cert -dir /var/lib/spinifex-eks/konnectivity \
+//	    -ca-cert /var/lib/rancher/k3s/server/tls/server-ca.crt \
+//	    -ca-key  /var/lib/rancher/k3s/server/tls/server-ca.key \
+//	    -cn konnectivity-server -sans 10.32.100.4,203.0.113.9,eks-toc.example
+//
+// Writes <dir>/tls.crt and <dir>/tls.key; prints the two paths (tab-separated).
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
+	caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
+	cn := flag.String("cn", "konnectivity-server", "certificate common name")
+	sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
+	flag.Parse()
+
+	if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
+		fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
+	}
+	dns, ips := splitSANs(*sansCSV)
+	if len(dns) == 0 && len(ips) == 0 {
+		fatal("eks-konnectivity-cert: -sans lists no usable names")
+	}
+
+	certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
+	if err != nil {
+		fatal("eks-konnectivity-cert: " + err.Error())
+	}
+	fmt.Printf("%s\t%s\n", certPath, keyPath)
+}
+
+// ensureCert returns the cached cert/key paths under dir, minting and persisting
+// a fresh CA-signed pair when none is present, readable, and still SAN-current.
+func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if certCoversSANs(certPath, keyPath, dns, ips) {
+		return certPath, keyPath, nil
+	}
+
+	caCert, caKey, err := loadCA(caCertPath, caKeyPath)
+	if err != nil {
+		return "", "", err
+	}
+	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	if err != nil {
+		return "", "", err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPath, keyPath, nil
+}
+
+// certCoversSANs reports whether the cached cert exists, parses, has a key, and
+// already carries every requested SAN — so a cert minted before the endpoint
+// set changed is re-issued rather than served stale.
+func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
+	c, err := os.ReadFile(certPath)
+	if err != nil {
+		return false
+	}
+	if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
+		return false
+	}
+	der := decodeFirst(c)
+	if der == nil {
+		return false
+	}
+	crt, err := x509.ParseCertificate(der)
+	if err != nil {
+		return false
+	}
+	have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
+	for _, n := range crt.DNSNames {
+		have[n] = struct{}{}
+	}
+	for _, ip := range crt.IPAddresses {
+		have[ip.String()] = struct{}{}
+	}
+	for _, n := range dns {
+		if _, ok := have[n]; !ok {
+			return false
+		}
+	}
+	for _, ip := range ips {
+		if _, ok := have[ip.String()]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
+func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	certPEM, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
+	}
+	der := decodeFirst(certPEM)
+	if der == nil {
+		return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
+	}
+	caCert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parse ca cert: %w", err)
+	}
+
+	keyPEM, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
+	}
+	keyDER := decodeFirst(keyPEM)
+	if keyDER == nil {
+		return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
+	}
+	caKey, err := parseECKey(keyDER)
+	if err != nil {
+		return nil, nil, err
+	}
+	return caCert, caKey, nil
+}
+
+// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
+func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
+	if k, err := x509.ParseECPrivateKey(der); err == nil {
+		return k, nil
+	}
+	k8, err := x509.ParsePKCS8PrivateKey(der)
+	if err != nil {
+		return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
+	}
+	ec, ok := k8.(*ecdsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
+	}
+	return ec, nil
+}
+
+// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
+func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
+// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
+func splitSANs(s string) (dns []string, ips []net.IP) {
+	for p := range strings.SplitSeq(s, ",") {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		if ip := net.ParseIP(p); ip != nil {
+			ips = append(ips, ip)
+			continue
+		}
+		dns = append(dns, p)
+	}
+	return dns, ips
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,153 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
+// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
+func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
+	t.Helper()
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen ca key: %v", err)
+	}
+	tmpl := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "k3s-server-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatalf("create ca cert: %v", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		t.Fatalf("marshal ca key: %v", err)
+	}
+	certPath = filepath.Join(dir, "server-ca.crt")
+	keyPath = filepath.Join(dir, "server-ca.key")
+	if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		t.Fatalf("write ca cert: %v", err)
+	}
+	if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
+		t.Fatalf("write ca key: %v", err)
+	}
+	return certPath, keyPath
+}
+
+func leafFrom(t *testing.T, certPath string) *x509.Certificate {
+	t.Helper()
+	b, err := os.ReadFile(certPath)
+	if err != nil {
+		t.Fatalf("read leaf: %v", err)
+	}
+	crt, err := x509.ParseCertificate(decodeFirst(b))
+	if err != nil {
+		t.Fatalf("parse leaf: %v", err)
+	}
+	return crt
+}
+
+func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
+	certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if _, err := os.Stat(keyPath); err != nil {
+		t.Fatalf("key not written: %v", err)
+	}
+
+	leaf := leafFrom(t, certPath)
+	if leaf.IsCA {
+		t.Error("konnectivity serving cert must be a leaf, not a CA")
+	}
+	// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
+	// so the IP must be a SAN or the tunnel TLS handshake fails.
+	if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
+		t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
+	}
+
+	// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
+	// agent trusts, so no separate CA distribution is needed.
+	caPEM, _ := os.ReadFile(caCert)
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(caPEM) {
+		t.Fatal("append CA failed")
+	}
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   "eks-toc.example",
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("leaf does not chain to the K3s server CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,eks-toc.example")
+	c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	first, _ := os.ReadFile(c1)
+
+	// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
+	// the konnectivity-server on every restart).
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if second, _ := os.ReadFile(c1); string(first) != string(second) {
+		t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
+	}
+
+	// A new endpoint in the SAN set must force a re-issue.
+	dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
+		t.Fatalf("third ensureCert: %v", err)
+	}
+	if third, _ := os.ReadFile(c1); string(first) == string(third) {
+		t.Error("ensureCert served a stale cert that does not cover the new SAN")
+	}
+	if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
+		t.Errorf("re-issued cert missing new SAN: %v", err)
+	}
+}
+
+func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
+	dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
+	if len(dns) != 1 || dns[0] != "host.example" {
+		t.Errorf("dns = %v, want [host.example]", dns)
+	}
+	wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
+	if len(ips) != len(wantIPs) {
+		t.Fatalf("ips = %v, want %v", ips, wantIPs)
+	}
+	for i, ip := range ips {
+		if !ip.Equal(wantIPs[i]) {
+			t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
+		}
+	}
+}
@@ -18,14 +18,10 @@ import (
 	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )

-// tlsCertificate is the webhook's loopback serving certificate.
-type tlsCertificate struct {
-	cert tls.Certificate
-}
-
-func (c tlsCertificate) serverTLSConfig() *tls.Config {
+// serverTLSConfig is the webhook's loopback serving TLS config.
+func serverTLSConfig(cert tls.Certificate) *tls.Config {
 	return &tls.Config{
-		Certificates:     []tls.Certificate{c.cert},
+		Certificates:     []tls.Certificate{cert},
 		MinVersion:       tls.VersionTLS13,
 		CurvePreferences: tlsconfig.Curves,
 	}
@@ -36,30 +32,30 @@ func (c tlsCertificate) serverTLSConfig() *tls.Config {
 // across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
-func ensureServingCert(certPath, keyPath string) (tlsCertificate, []byte, error) {
+func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
 	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
 		cert, err := tls.X509KeyPair(certPEM, keyPEM)
 		if err == nil {
-			return tlsCertificate{cert: cert}, certPEM, nil
+			return cert, certPEM, nil
 		}
 		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}

 	certPEM, keyPEM, err := generateSelfSigned()
 	if err != nil {
-		return tlsCertificate{}, nil, err
+		return tls.Certificate{}, nil, err
 	}
 	if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
 	}
 	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
 	}
 	cert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
+		return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
 	}
-	return tlsCertificate{cert: cert}, certPEM, nil
+	return cert, certPEM, nil
 }

 func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
@@ -10,6 +10,9 @@
 // On startup it writes the apiserver webhook kubeconfig (with its self-signed
 // serving CA) so k3s can be pointed at it via
 // --authentication-token-webhook-config-file.
+//
+// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
 package main

 import (
@@ -56,11 +59,13 @@ func loadConfig() (config, error) {
 		keyPath:     envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
 		kubeconfig:  envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
 	}
+	// Static SigV4 creds are optional: when absent, eksgw.New signs with the
+	// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
+	// sibling helpers use. The CP VM launches on an instance profile, so
+	// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
 	switch {
 	case c.gatewayURL == "":
 		return c, errors.New("EKS_GATEWAY_URL not set")
-	case c.accessKey == "" || c.secretKey == "":
-		return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
 	case c.accountID == "":
 		return c, errors.New("EKS_ACCOUNT_ID not set")
 	case c.clusterName == "":
@@ -125,7 +130,12 @@ func run(cfg config) error {
 	})
 	mux.HandleFunc("/authenticate", authr.handle)

-	server := newServer(cfg.addr, mux, tlsCert)
+	server := &http.Server{
+		Addr:              cfg.addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		TLSConfig:         serverTLSConfig(tlsCert),
+	}
 	slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
 	if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("serve: %w", err)
@@ -214,14 +224,3 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
 		slog.Error("encode TokenReview response", "err", err)
 	}
 }
-
-// newServer builds the loopback TLS server. Split out so tests can construct it
-// without binding a port.
-func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
-	return &http.Server{
-		Addr:              addr,
-		Handler:           handler,
-		ReadHeaderTimeout: 5 * time.Second,
-		TLSConfig:         cert.serverTLSConfig(),
-	}
-}
@@ -99,7 +99,7 @@ func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
 	c1, pem1, err := ensureServingCert(certPath, keyPath)
 	require.NoError(t, err)
 	require.NotEmpty(t, pem1)
-	require.NotNil(t, c1.serverTLSConfig())
+	require.NotNil(t, serverTLSConfig(c1))

 	// Second call reuses the persisted pair (same cert bytes).
 	_, pem2, err := ensureServingCert(certPath, keyPath)
@@ -121,3 +121,41 @@ func TestWriteAPIServerKubeconfig(t *testing.T) {
 	assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
 	assert.Contains(t, body, "current-context: webhook")
 }
+
+// The CP VM launches on an instance profile, so cloud-init omits the static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
+// must accept absent static keys; gating on them aborts every IMDS-mode boot,
+// leaving the apiserver webhook kubeconfig unwritten.
+func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
+	setEnv := func(t *testing.T, kv map[string]string) {
+		t.Helper()
+		for _, k := range []string{
+			"EKS_GATEWAY_URL", "EKS_ACCESS_KEY", "EKS_SECRET_KEY",
+			"EKS_ACCOUNT_ID", "EKS_CLUSTER_NAME",
+		} {
+			t.Setenv(k, "")
+		}
+		for k, v := range kv {
+			t.Setenv(k, v)
+		}
+	}
+
+	// IMDS mode: required vars set, static creds absent → accepted.
+	setEnv(t, map[string]string{
+		"EKS_GATEWAY_URL":  "https://gw:9999",
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	cfg, err := loadConfig()
+	require.NoError(t, err)
+	assert.Empty(t, cfg.accessKey)
+	assert.Empty(t, cfg.secretKey)
+
+	// Genuinely-required vars are still enforced.
+	setEnv(t, map[string]string{
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	_, err = loadConfig()
+	require.Error(t, err)
+}
@@ -0,0 +1,147 @@
+// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving
+// certificate for an in-cluster admission webhook and prints its base64 PEM so
+// the addon renderer can embed it as the webhook Secret + each caBundle.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent of the
+// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster,
+// cached on disk under -dir, stable across addon-sync ticks. The single cert is
+// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf
+// and the caBundle the apiserver trusts.
+//
+// Usage:
+//
+//	eks-webhook-cert -dir /var/lib/spinifex-eks/webhook-certs/<addon> \
+//	    -cn <svc>.<ns>.svc -dns <svc>.<ns>.svc,<svc>.<ns>.svc.cluster.local
+//
+// Output (one line): <ca_b64>\t<tls_crt_b64>\t<tls_key_b64>
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	cn := flag.String("cn", "", "certificate common name")
+	dnsCSV := flag.String("dns", "", "comma-separated DNS SANs")
+	flag.Parse()
+
+	if *dir == "" || *cn == "" || *dnsCSV == "" {
+		fatal("eks-webhook-cert: -dir, -cn and -dns are required")
+	}
+	dnsNames := splitCSV(*dnsCSV)
+	if len(dnsNames) == 0 {
+		fatal("eks-webhook-cert: -dns lists no names")
+	}
+
+	certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames)
+	if err != nil {
+		fatal("eks-webhook-cert: " + err.Error())
+	}
+	b := base64.StdEncoding.EncodeToString
+	fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM))
+}
+
+// ensureCert returns the cached cert/key under dir, minting and persisting a
+// fresh self-signed pair (for cn + dnsNames) when none is present or readable.
+func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if c, cerr := os.ReadFile(certPath); cerr == nil {
+		if k, kerr := os.ReadFile(keyPath); kerr == nil {
+			if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
+				return c, k, nil
+			}
+		}
+	}
+
+	certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPEM, keyPEM, nil
+}
+
+// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
+// a server cert, with the given common name and DNS SANs.
+func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		DNSNames:              dnsNames,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func splitCSV(s string) []string {
+	var out []string
+	for p := range strings.SplitSeq(s, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/x509"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const (
+	testCN  = "aws-load-balancer-webhook-service.kube-system.svc"
+	testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local"
+)
+
+func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
+	t.Helper()
+	leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
+	if err != nil {
+		t.Fatalf("parse cert: %v", err)
+	}
+	return leaf
+}
+
+func TestEnsureCertMintsServingCert(t *testing.T) {
+	dir := t.TempDir()
+	certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS})
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if len(keyPEM) == 0 {
+		t.Fatal("empty key PEM")
+	}
+	leaf := parseLeaf(t, certPEM)
+
+	// Both webhook service DNS names must be present, or the apiserver's TLS
+	// dial to the webhook fails SAN verification.
+	for _, want := range []string{testCN, testDNS} {
+		if err := leaf.VerifyHostname(want); err != nil {
+			t.Errorf("SAN %q missing: %v", want, err)
+		}
+	}
+	if !leaf.IsCA {
+		t.Error("cert must be a CA so the same PEM serves as the webhook caBundle")
+	}
+
+	// The single self-signed cert must verify against itself as a root — this is
+	// exactly what the apiserver does with caBundle == the serving cert.
+	roots := x509.NewCertPool()
+	roots.AddCert(leaf)
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   testCN,
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("self-signed cert does not verify as its own CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesCachedPair(t *testing.T) {
+	dir := t.TempDir()
+	cert1, key1, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	cert2, key2, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if string(cert1) != string(cert2) || string(key1) != string(key2) {
+		t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick")
+	}
+
+	// The private key must not be world-readable on disk.
+	info, err := os.Stat(filepath.Join(dir, "tls.key"))
+	if err != nil {
+		t.Fatalf("stat key: %v", err)
+	}
+	if info.Mode().Perm()&0o077 != 0 {
+		t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm())
+	}
+}
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package autoinstall reads headless install parameters from environment
 // variables exported by spinifex-init.sh from the kernel cmdline.
 // Parameters are set in the GRUB "Headless" menu entry in grub.cfg.
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package branding centralises Mulga/Spinifex visual identity constants used
 // by the installer TUI.
 //
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package firstboot writes the oneshot systemd service and configuration that
 // completes Spinifex provisioning on the first real boot after installation.
 package firstboot
@@ -91,6 +74,31 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}

+	// Pre-start OVS and OVN central so their databases are initialised before
+	// setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
+	// longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
+	// here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
+	// the moment it starts — no races, no timeout failures.
+	ovnPrestart := `systemctl start openvswitch-switch
+systemctl start ovn-central
+echo "Waiting for OVN NB DB to initialise..."
+for _i in $(seq 1 120); do
+    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+        echo "OVN NB DB ready (${_i}s)"
+        break
+    fi
+    sleep 1
+done`
+
+	// When a provisioning controller owns formation (SkipFormation) it also owns
+	// the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must
+	// not start a standalone ovn-central here: the single-node .db it would leave
+	// blocks ovn-ctl's create/join (which require a clean DB). Defer both.
+	if cfg.SkipFormation {
+		ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"`
+		setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"`
+	}
+
 	callbackBlock := ""
 	if cfg.InstallCallback != "" {
 		callbackBlock = fmt.Sprintf(
@@ -123,21 +131,7 @@ fi
 # Set hostname
 hostnamectl set-hostname %s

-# Pre-start OVS and OVN central so their databases are initialised before
-# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
-# longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
-# here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
-# the moment it starts — no races, no timeout failures.
-systemctl start openvswitch-switch
-systemctl start ovn-central
-echo "Waiting for OVN NB DB to initialise..."
-for _i in $(seq 1 120); do
-    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
-        echo "OVN NB DB ready (${_i}s)"
-        break
-    fi
-    sleep 1
-done
+%s

 # Create default nameservers if DHCP server returns blank
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
@@ -229,7 +223,7 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
        echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
    fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)
+`, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock)

 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)
@@ -66,6 +66,54 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	}
 }

+func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script should run setup-ovn --management when firstboot owns formation")
+	}
+	if !strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script should pre-start ovn-central when firstboot owns formation")
+	}
+}
+
+func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script must not run setup-ovn --management when a controller owns OVN")
+	}
+	if strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script must not pre-start ovn-central when a controller owns OVN")
+	}
+	if !strings.Contains(content, "setup-ovn deferred") {
+		t.Error("script should note setup-ovn is deferred under SkipFormation")
+	}
+}
+
+func readScript(t *testing.T, root string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	return string(b)
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package install performs the disk installation steps after the UI has
 // collected configuration.
 package install
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 package install

 import (
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main

 import (
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package systemd writes systemd unit files and drop-ins into an installed
 // system root during the Spinifex ISO installation process.
 package systemd
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package ui presents the interactive installer TUI using bubbletea and lipgloss.
 package ui

@@ -49,12 +49,13 @@ func main() {
 		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
 		os.Exit(1)
 	}
-	if accessKey == "" || secretKey == "" {
-		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
-		os.Exit(1)
+	// Empty keys select IMDS instance-role credentials (the SDK default chain
+	// falls back to the in-VM IMDS endpoint); static keys sign as-is.
+	credSource := "imds"
+	if accessKey != "" && secretKey != "" {
+		credSource = "static"
 	}
-
-	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL, "credSource", credSource)

 	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
 	if err != nil {
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -35,6 +19,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -49,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/viperblock/viperblock"
@@ -115,6 +101,23 @@ Each phase waits for all nodes to ACK before proceeding to the next.`,
 	Run: runClusterShutdown,
 }

+var nodeCmd = &cobra.Command{
+	Use:   "node",
+	Short: "Node-local operations",
+	Long:  `Node-local administrative operations such as a graceful local guest drain.`,
+}
+
+var nodeDrainCmd = &cobra.Command{
+	Use:   "drain",
+	Short: "Gracefully drain guests on the local node",
+	Long: `Run the GATE and DRAIN shutdown phases against the local node only: power down
+its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
+every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
+ordered unit teardown. This is the ExecStop of spinifex-shutdown.service, so a
+systemctl stop or host reboot drains guests before any storage service stops.`,
+	Run: runNodeDrainLocal,
+}
+
 var clusterDrainDHCPCmd = &cobra.Command{
 	Use:   "drain-dhcp",
 	Short: "Release all upstream DHCP leases held by vpcd",
@@ -176,6 +179,18 @@ unless --force is passed. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesRemoveCmd,
 }

+var imagesPromoteCmd = &cobra.Command{
+	Use:   "promote",
+	Short: "Promote an account-owned AMI to a system image",
+	Long: `Rewrite an account-owned AMI's owner to the system alias so it becomes
+visible to all accounts via DescribeImages, matching the behaviour of AMIs
+imported via 'spx admin images import'.
+
+No data is copied — only the config.json owner field is updated. The change
+takes effect immediately. Prompts for confirmation unless --yes is passed.`,
+	Run: runimagesPromoteCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -258,10 +273,16 @@ func init() {
 	clusterCmd.AddCommand(clusterDrainDHCPCmd)
 	clusterDrainDHCPCmd.Flags().Duration("timeout", 30*time.Second, "Reply-collection window for vpcd drain responders")

+	adminCmd.AddCommand(nodeCmd)
+	nodeCmd.AddCommand(nodeDrainCmd)
+	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
+	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
+
 	adminCmd.AddCommand(imagesCmd)
 	imagesCmd.AddCommand(imagesImportCmd)
 	imagesCmd.AddCommand(imagesListCmd)
 	imagesCmd.AddCommand(imagesRemoveCmd)
+	imagesCmd.AddCommand(imagesPromoteCmd)

 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -298,13 +319,14 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("token-ttl", "30m", "Join token validity duration (e.g. 30m, 1h, 2h)")
+	adminInitCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
 	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")

 	// External networking flags
-	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
+	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected), 'nat' (routed; non-bridgeable uplinks; add --external-pool or --external-source=dhcp for public IPs), or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
 	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
 	adminInitCmd.Flags().String("external-bind-bridge", "", "Linux bridge for upstream DHCP DORA (default 'br-wan' when --external-source=dhcp)")
@@ -331,6 +353,7 @@ func init() {
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
+	adminJoinCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 	adminJoinCmd.MarkFlagRequired("token")
@@ -355,6 +378,12 @@ func init() {
 	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
 		panic(err)
 	}
+
+	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
+	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
+	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
+		panic(err)
+	}
 }

 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -641,6 +670,13 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		EncryptionEnabled: mkey != nil,
 	}

+	// Bake the deployment CA into the image trust store so a stock cloud image
+	// trusts the gateway from first boot. Best-effort: never fails the import.
+	// The CA path derives from the --spinifex-dir data root (its config/ symlink
+	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
+	// command, which would collapse the path to a relative config/ca.pem.
+	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
+
 	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath)

 	if err != nil {
@@ -653,6 +689,64 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }

+// caBakeRunCommand installs the uploaded CA into the guest trust store across
+// the distro families, then removes the staged copy. Each branch is gated on its
+// updater existing so only the matching distro's branch writes anything (install
+// -D creates the anchor dir on minimal images). Debian/Ubuntu use
+// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
+// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
+// appending the PEM to the static bundle. The install chain's exit code is
+// preserved past the cleanup rm so a failure surfaces instead of a false success.
+// The RHEL branch restorecon's only the trust-store paths it touched so the cert
+// is correctly labelled without the image-wide --no-selinux-relabel (below)
+// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
+const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
+	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
+	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
+	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
+	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
+
+// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
+// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
+const caBakeTimeout = 5 * time.Minute
+
+// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
+// into the disk image at imagePath and installs it into the guest trust store.
+// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
+// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
+// reboot, so the run-command relabels only the touched trust paths instead.
+func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
+	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
+		"--no-selinux-relabel",
+		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
+		"--run-command", caBakeRunCommand)
+}
+
+// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
+var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
+	if _, err := exec.LookPath("virt-customize"); err != nil {
+		return nil, fmt.Errorf("virt-customize not found: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
+	defer cancel()
+	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
+}
+
+// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
+// virt-customize so an imported stock image trusts the gateway from first boot.
+// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
+// cannot inspect logs and continues — image import never fails on the CA.
+func bakeCACertIntoImage(imagePath, caCertPath string) {
+	if _, err := os.Stat(caCertPath); err != nil {
+		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+		return
+	}
+	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
+		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
+		return
+	}
+}
+
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 	imageID, _ := cmd.Flags().GetString("image-id")
 	force, _ := cmd.Flags().GetBool("force")
@@ -783,6 +877,67 @@ func printDependents(w io.Writer, d admin.Dependents) {
 	}
 }

+func runimagesPromoteCmd(cmd *cobra.Command, args []string) {
+	imageID, _ := cmd.Flags().GetString("image-id")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	cfgFile, _ := cmd.Flags().GetString("config")
+	if cfgFile == "" {
+		cfgFile = DefaultConfigFile()
+	}
+
+	appConfig, err := config.LoadConfig(cfgFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error loading config file:", err)
+		os.Exit(1)
+	}
+
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+	bucket := node.Predastore.Bucket
+
+	// Read current metadata for the confirmation prompt.
+	meta, err := admin.GetAMIMetadata(store, bucket, imageID)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to inspect AMI:", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("About to promote AMI to system image:")
+	fmt.Println()
+	fmt.Printf("  Image ID:       %s\n", imageID)
+	fmt.Printf("  Name:           %s\n", meta.Name)
+	fmt.Printf("  Current owner:  %s\n", meta.ImageOwnerAlias)
+	fmt.Printf("  New owner:      %s\n", admin.SystemOwnerAlias)
+	if !meta.CreationDate.IsZero() {
+		fmt.Printf("  Created:        %s\n", meta.CreationDate.UTC().Format("2006-01-02T15:04:05Z"))
+	}
+	fmt.Println()
+	fmt.Println("After promotion this AMI will be visible to all accounts.")
+
+	if !yes {
+		fmt.Print("Type 'yes' to proceed: ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		if strings.TrimSpace(answer) != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	if _, err := admin.PromoteSystemImage(store, bucket, admin.PromoteImageOpts{ImageID: imageID}); err != nil {
+		fmt.Fprintln(os.Stderr, "Promote failed:", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("✅ Promoted %s to system image (owner: %s).\n", imageID, admin.SystemOwnerAlias)
+}
+
 // List remote images available
 func runimagesListCmd(cmd *cobra.Command, args []string) {
 	//fmt.Println(availableImages)
@@ -791,17 +946,8 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 		{"NAME", "DISTRO", "VERSION", "ARCH", "BOOT"},
 	}

-	// Sort A .. Z
-	// 1. Collect keys
-	keys := make([]string, 0, len(utils.AvailableImages))
-	for k := range utils.AvailableImages {
-		keys = append(keys, k)
-	}
-
-	// 2. Sort keys alphabetically (A→Z)
-	sort.Strings(keys)
-
-	// 3. Iterate in sorted order
+	// Sort A→Z then iterate.
+	keys := slices.Sorted(maps.Keys(utils.AvailableImages))
 	for _, k := range keys {
 		img := utils.AvailableImages[k]

@@ -846,6 +992,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
 	formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout")
 	tokenTTLStr, _ := cmd.Flags().GetString("token-ttl")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")

@@ -944,6 +1091,13 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				// --external-pool is omitted the validator below will error with
 				// a SuggestPoolRange hint.
 				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+					if isNonBridgeableUplink(detected.WAN.Name) {
+						fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
+						fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+						fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
+						fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
+						os.Exit(1)
+					}
 					externalMode = "pool"
 				}
 			}
@@ -951,60 +1105,60 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}

 	// Validate external networking flags
-	if externalMode != "" && externalMode != "pool" {
-		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool' or empty, got: %s\n", externalMode)
+	if externalMode != "" && externalMode != "pool" && externalMode != "nat" {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode)
 		os.Exit(1)
 	}
-	if externalMode == "pool" {
-		// Default source: dhcp when no --external-pool, else static.
-		if externalSource == "" {
-			if externalPool == "" {
-				externalSource = "dhcp"
-			} else {
-				externalSource = "static"
-			}
-		}
-		switch externalSource {
-		case "dhcp":
-			if externalPool != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)\n")
-				os.Exit(1)
-			}
-			if externalBindBridge == "" {
-				externalBindBridge = "br-wan"
-			}
-		case "static":
-			if externalBindBridge != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge only valid with --external-source=dhcp\n")
-				os.Exit(1)
-			}
-			if externalPool == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)\n")
-				if detectedNet != nil && detectedNet.WAN != nil {
-					sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
-					fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
-				}
-				os.Exit(1)
-			}
-			if externalGateway == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required with --external-source=static\n")
-				os.Exit(1)
-			}
-			parts := strings.SplitN(externalPool, "-", 2)
-			if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool)
-				os.Exit(1)
-			}
-			poolStart, poolEnd = parts[0], parts[1]
-		default:
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-source must be 'static' or 'dhcp', got: %s\n", externalSource)
+	// A public pool alongside nat's transit pool restores EIP / public-subnet
+	// parity where the operator has spare LAN IPs; without these flags nat
+	// stays Tier-1-only (host-jumpbox access, no public IPs).
+	natPublicPool := externalMode == "nat" && (externalPool != "" || externalSource != "")
+	// natPublicGateway keeps the upstream gateway for the public pool before
+	// the transit segment claims externalGateway below.
+	natPublicGateway := externalGateway
+	if externalMode == "nat" {
+		if nodes >= 2 {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-mode=nat is single-node only (v1); use --nodes=1\n")
 			os.Exit(1)
 		}
+		if !natPublicPool && (externalBindBridge != "" || gatewayIP != "") {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge/--gateway-ip require --external-pool or --external-source in --external-mode=nat\n")
+			os.Exit(1)
+		}
+		if natPublicPool {
+			// DHCP DORA in nat mode binds the uplink interface itself — there
+			// is no br-wan (nothing is bridged in routed mode).
+			src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, natPublicGateway, externalIface)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+				os.Exit(1)
+			}
+			externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
+		}
+		// The transit segment is fixed: the host veth owns the gateway IP and
+		// masquerades the /24.
+		externalGateway = host.NATTransitGatewayIP
+	}
+	if externalMode == "pool" {
+		src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, externalGateway, "br-wan")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			if strings.Contains(err.Error(), "--external-pool is required") && detectedNet != nil && detectedNet.WAN != nil {
+				sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
+				fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
+			}
+			os.Exit(1)
+		}
+		externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
 	}
 	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
 		os.Exit(1)
 	}
+	if natPublicGateway != "" && net.ParseIP(natPublicGateway) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", natPublicGateway)
+		os.Exit(1)
+	}
 	if gatewayIP != "" && net.ParseIP(gatewayIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --gateway-ip is not a valid IP: %s\n", gatewayIP)
 		os.Exit(1)
@@ -1019,6 +1173,35 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}

+	// Assemble the external pool blocks rendered into spinifex.toml.
+	var externalPools []admin.PoolData
+	switch externalMode {
+	case "nat":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP,
+			PrefixLen: 24, DNSServers: dnsServers,
+		})
+		if natPublicPool {
+			// WiFi/WWAN uplinks drop frames with foreign source MACs, so DHCP
+			// leases must go out with the interface's own MAC.
+			dhcpMAC := ""
+			if externalSource == "dhcp" && isNonBridgeableUplink(externalBindBridge) {
+				dhcpMAC = "interface"
+			}
+			externalPools = append(externalPools, admin.PoolData{
+				Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+				DHCPMAC: dhcpMAC, Start: poolStart, End: poolEnd, Gateway: natPublicGateway,
+				GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+			})
+		}
+	case "pool":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+			Start: poolStart, End: poolEnd, Gateway: externalGateway,
+			GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+		})
+	}
+
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1145,7 +1328,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   AWS Profile: spinifex\n")

 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)

 	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
 	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
@@ -1256,7 +1439,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}

 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1297,23 +1480,17 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,

 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",

-		ExternalMode:   externalMode,
-		ExternalIface:  externalIface,
-		PoolName:       "wan",
-		PoolSource:     externalSource,
-		PoolBindBridge: externalBindBridge,
-		PoolStart:      poolStart,
-		PoolEnd:        poolEnd,
-		PoolGateway:    externalGateway,
-		PoolGatewayIP:  gatewayIP,
-		PoolPrefixLen:  externalPrefixLen,
-		PoolDNSServers: dnsServers,
+		ExternalMode:  externalMode,
+		ExternalIface: externalIface,
+		BridgeMode:    bridgeModeFor(externalMode),
+		Pools:         externalPools,

 		OperatorEmail:       email,
 		BootstrapAccountId:  admin.DefaultAccountID(),
@@ -1330,7 +1507,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}

 	// Print external networking summary
-	if externalMode != "" {
+	if externalMode == "nat" {
+		if natPublicPool {
+			fmt.Printf("\n📡 External networking: nat (routed) with public pool — EIPs enabled\n")
+			if externalSource == "static" {
+				fmt.Printf("  Public pool:   %s - %s (source: static)\n", poolStart, poolEnd)
+			} else {
+				fmt.Printf("  Public pool:   dhcp via %s\n", externalBindBridge)
+			}
+		} else {
+			fmt.Printf("\n📡 External networking: nat (routed, outbound-only — no public IPs/EIPs)\n")
+		}
+		fmt.Printf("  Transit:       %s via %s (host masquerades out any uplink)\n", host.NATTransitCIDR, host.NATTransitHostEnd)
+		fmt.Printf("  Host setup:    ./scripts/setup-ovn.sh --nat-uplink (run before starting services)\n")
+	} else if externalMode != "" {
 		fmt.Printf("\n📡 External networking: %s\n", externalMode)
 		fmt.Printf("  WAN interface: %s\n", externalIface)
 		switch externalSource {
@@ -1403,6 +1593,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
+
 	joinToken, err := formation.GenerateJoinToken()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error generating join token: %v\n", err)
@@ -1531,6 +1723,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
 	predastoreNodes := formation.BuildPredastoreNodes(allNodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(allNodes)

 	fmt.Println("\n📝 Creating configuration files...")

@@ -1546,7 +1739,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	var predastoreNodeID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1583,17 +1776,19 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(allNodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(allNodes, node),

 		OperatorEmail: email,

 		EncryptionKeyFile: viperblockKeyPath,

-		// Init node runs ovn-central locally
-		OVNNBAddr: "tcp:127.0.0.1:6641",
-		OVNSBAddr: "tcp:127.0.0.1:6642",
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum; the init node's
+		// own address leads, the rest provide failover.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}

 	if networkConfig != nil {
@@ -1655,6 +1850,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	configDir, _ := cmd.Flags().GetString("config-dir")
 	clusterBind, _ := cmd.Flags().GetString("cluster-bind")
 	services, _ := cmd.Flags().GetStringSlice("services")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")

 	email, _ := cmd.Flags().GetString("email")
 	email = strings.TrimSpace(email)
@@ -1677,13 +1873,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	// Extract leader IP for OVN NB/SB DB address (strip port from host:port)
-	leaderIP, _, err := net.SplitHostPort(leaderHost)
-	if err != nil {
-		// leaderHost might be an IP without port
-		leaderIP = leaderHost
-	}
-
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1959,7 +2148,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}

 	// Generate server cert signed by CA with this node's bind IP
-	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
+	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSInternalSuffix); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -1986,6 +2175,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Build cluster topology from formation data
 	clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes)
 	predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(statusResp.Nodes)

 	fmt.Println("📝 Creating configuration files...")

@@ -2002,7 +2192,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	hasPredastoreConfig := len(predastoreNodes) >= 2

 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -2043,17 +2233,19 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(statusResp.Nodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node),

 		OperatorEmail: email,

 		EncryptionKeyFile: viperblockKeyPath,

-		// Joining nodes connect to the init node's OVN NB/SB DB
-		OVNNBAddr: fmt.Sprintf("tcp:%s:6641", leaderIP),
-		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum so the client
+		// fails over instead of pinning to a single init node.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}

 	if statusResp.NetworkConfig != nil {
@@ -2322,7 +2514,7 @@ func runCertRenew(cmd *cobra.Command, _ []string) {
 	serverCertPath := filepath.Join(configDir, "server.pem")
 	serverKeyPath := filepath.Join(configDir, "server.key")

-	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+	if err := admin.GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
 		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2398,15 +2590,19 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region
 func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) {
 	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
-	settings.PoolName = nc.PoolName
-	settings.PoolSource = nc.PoolSource
-	settings.PoolBindBridge = nc.PoolBindBridge
-	settings.PoolStart = nc.PoolStart
-	settings.PoolEnd = nc.PoolEnd
-	settings.PoolGateway = nc.PoolGateway
-	settings.PoolGatewayIP = nc.PoolGatewayIP
-	settings.PoolPrefixLen = nc.PoolPrefixLen
-	settings.PoolDNSServers = nc.PoolDNSServers
+	if nc.ExternalMode != "" {
+		settings.Pools = []admin.PoolData{{
+			Name:       nc.PoolName,
+			Source:     nc.PoolSource,
+			BindBridge: nc.PoolBindBridge,
+			Start:      nc.PoolStart,
+			End:        nc.PoolEnd,
+			Gateway:    nc.PoolGateway,
+			GatewayIP:  nc.PoolGatewayIP,
+			PrefixLen:  nc.PoolPrefixLen,
+			DNSServers: nc.PoolDNSServers,
+		}}
+	}

 	settings.BootstrapAccountId = nc.BootstrapAccountId
 	settings.BootstrapVpcId = nc.BootstrapVpcId
@@ -2564,6 +2760,66 @@ func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []by
 	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }

+// isNonBridgeableUplink reports whether the interface name indicates an uplink
+// that cannot be enslaved to an L2 bridge: WiFi (wl*), cellular (ww*), PPP.
+func isNonBridgeableUplink(name string) bool {
+	return strings.HasPrefix(name, "wl") || strings.HasPrefix(name, "ww") || strings.HasPrefix(name, "ppp")
+}
+
+// resolvePublicPoolFlags validates the public-pool flag set shared by pool
+// mode and nat-with-public-pool. Returns the resolved source, static range
+// start/end, and bind bridge. Source defaults to dhcp when no range is given;
+// defaultBindBridge fills --external-bind-bridge for dhcp (br-wan in pool
+// mode, the uplink interface in nat mode).
+func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindBridge string) (resolvedSource, start, end, resolvedBindBridge string, err error) {
+	if source == "" {
+		if poolRange == "" {
+			source = "dhcp"
+		} else {
+			source = "static"
+		}
+	}
+	switch source {
+	case "dhcp":
+		if poolRange != "" {
+			return "", "", "", "", fmt.Errorf("--external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)")
+		}
+		if bindBridge == "" {
+			if defaultBindBridge == "" {
+				return "", "", "", "", fmt.Errorf("--external-bind-bridge is required with --external-source=dhcp (no uplink interface detected)")
+			}
+			bindBridge = defaultBindBridge
+		}
+	case "static":
+		if bindBridge != "" {
+			return "", "", "", "", fmt.Errorf("--external-bind-bridge only valid with --external-source=dhcp")
+		}
+		if poolRange == "" {
+			return "", "", "", "", fmt.Errorf("--external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)")
+		}
+		if gateway == "" {
+			return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
+		}
+		parts := strings.SplitN(poolRange, "-", 2)
+		if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+			return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
+		}
+		start, end = parts[0], parts[1]
+	default:
+		return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
+	}
+	return source, start, end, bindBridge, nil
+}
+
+// bridgeModeFor returns the vpcd bridge_mode admin init persists for the
+// external mode: only nat mode is pinned; bridged modes stay auto-detected.
+func bridgeModeFor(externalMode string) string {
+	if externalMode == "nat" {
+		return "nat"
+	}
+	return ""
+}
+
 // detectDNSServers auto-detects DNS servers from the host for the specified
 // interface. Uses resolvectl (systemd-resolved) first, then falls back to
 // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.
@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCABakeCmd_Args(t *testing.T) {
+	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
+	args := cmd.Args
+
+	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
+	assert.Contains(t, args, "-a")
+	assert.Contains(t, args, "/img/disk.raw")
+	assert.Contains(t, args, "--upload")
+	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
+	assert.Contains(t, args, "--run-command")
+
+	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
+	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
+	assert.Contains(t, args, "--no-selinux-relabel")
+
+	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
+	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
+	// ca-certificates bundle when no updater is present. The RHEL branch relabels
+	// only the touched trust paths so the cert is labelled without a full relabel.
+	joined := strings.Join(args, " ")
+	assert.Contains(t, joined, "update-ca-certificates")
+	assert.Contains(t, joined, "update-ca-trust")
+	assert.Contains(t, joined, "restorecon")
+	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
+}
+
+func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
+	defer func() { caBakeRunner = orig }()
+
+	// A missing CA must not invoke virt-customize and must not panic.
+	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
+	assert.False(t, called, "runner must not be called when the CA is absent")
+}
+
+func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
+	caPath := filepath.Join(t.TempDir(), "ca.pem")
+	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
+
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) {
+		called = true
+		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
+	}
+	defer func() { caBakeRunner = orig }()
+
+	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
+	bakeCACertIntoImage("/img/disk.raw", caPath)
+	assert.True(t, called, "runner must be invoked when the CA is present")
+}
@@ -0,0 +1,198 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `external_mode = "nat"`)
+	assert.Contains(t, content, `bridge_mode = "nat"`)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `gateway     = "100.127.0.1"`)
+	assert.NotContains(t, content, "range_start", "nat mode has no public IP range")
+}
+
+func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode:  "pool",
+		ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "bridge_mode", "bridged modes keep runtime auto-detect")
+}
+
+func TestIsNonBridgeableUplink(t *testing.T) {
+	for _, name := range []string{"wlan0", "wlp3s0", "wlx001122334455", "wwan0", "wwp0s20f0u6", "ppp0"} {
+		assert.True(t, isNonBridgeableUplink(name), name)
+	}
+	for _, name := range []string{"eth0", "enp0s3", "eno1", "br-wan", "veth-wan-br"} {
+		assert.False(t, isNonBridgeableUplink(name), name)
+	}
+}
+
+func TestBridgeModeFor(t *testing.T) {
+	assert.Equal(t, "nat", bridgeModeFor("nat"))
+	assert.Equal(t, "", bridgeModeFor("pool"))
+	assert.Equal(t, "", bridgeModeFor(""))
+}
+
+func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24},
+			{
+				Name: "wan", Source: "dhcp", BindBridge: "wlan0",
+				DHCPMAC: "interface", PrefixLen: 24,
+			},
+		},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `name        = "wan"`)
+	assert.Contains(t, content, `source      = "dhcp"`)
+	assert.Contains(t, content, `bind_bridge = "wlan0"`)
+	assert.Contains(t, content, `dhcp_mac    = "interface"`)
+	transitIdx := strings.Index(content, `name        = "nat-transit"`)
+	wanIdx := strings.Index(content, `name        = "wan"`)
+	assert.Less(t, transitIdx, wanIdx, "transit pool must render first (IGW allocator picks it)")
+}
+
+func TestResolvePublicPoolFlags(t *testing.T) {
+	tests := []struct {
+		name              string
+		source, poolRange string
+		bindBridge, gw    string
+		defaultBridge     string
+		wantSource        string
+		wantStart         string
+		wantEnd           string
+		wantBridge        string
+		wantErr           string
+	}{
+		{
+			name: "static pool range", source: "", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1",
+			wantSource: "static", wantStart: "192.168.1.150", wantEnd: "192.168.1.250",
+		},
+		{
+			name: "dhcp defaults bind bridge", source: "dhcp", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "wlan0",
+		},
+		{
+			name: "dhcp explicit bind bridge", source: "dhcp", bindBridge: "br-wan", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "br-wan",
+		},
+		{
+			name: "dhcp rejects pool range", source: "dhcp", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-pool not allowed",
+		},
+		{
+			name: "static requires gateway", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-gateway is required",
+		},
+		{
+			name: "static rejects bind bridge", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1", bindBridge: "br-wan",
+			wantErr: "--external-bind-bridge only valid",
+		},
+		{
+			name: "static requires range", source: "static", gw: "192.168.1.1",
+			wantErr: "--external-pool is required",
+		},
+		{
+			name: "bad range", poolRange: "not-a-range", gw: "192.168.1.1",
+			wantErr: "start-end IPs",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src, start, end, bb, err := resolvePublicPoolFlags(tt.source, tt.poolRange, tt.bindBridge, tt.gw, tt.defaultBridge)
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantSource, src)
+			assert.Equal(t, tt.wantStart, start)
+			assert.Equal(t, tt.wantEnd, end)
+			assert.Equal(t, tt.wantBridge, bb)
+		})
+	}
+}
@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/utils"
+	toml "github.com/pelletier/go-toml/v2"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -206,7 +209,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 		{ID: 3, Host: "10.0.0.3"},
 	}
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content, `nats_url = "nats://localhost:4222"`)
@@ -214,7 +217,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {

 	// Specific bind IP → stays as-is.
 	content2, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
@@ -379,20 +382,19 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "spinifex.toml")
 	settings := admin.ConfigSettings{
-		Node:           "node1",
-		Az:             "ap-southeast-2a",
-		Port:           "4432",
-		Region:         "ap-southeast-2",
-		BindIP:         "10.11.12.1",
-		ConfigDir:      dir,
-		OVNNBAddr:      "tcp:127.0.0.1:6641",
-		OVNSBAddr:      "tcp:127.0.0.1:6642",
-		ExternalMode:   "pool",
-		ExternalIface:  "eth0",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		Node:          "node1",
+		Az:            "ap-southeast-2a",
+		Port:          "4432",
+		Region:        "ap-southeast-2",
+		BindIP:        "10.11.12.1",
+		ConfigDir:     dir,
+		OVNNBAddr:     "tcp:127.0.0.1:6641",
+		OVNSBAddr:     "tcp:127.0.0.1:6642",
+		ExternalMode:  "pool",
+		ExternalIface: "eth0",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -421,12 +423,11 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 		OVNSBAddr:     "tcp:127.0.0.1:6642",
 		ExternalMode:  "pool",
 		ExternalIface: "eth0",
-		PoolName:      "wan",
-		PoolSource:    "static",
-		PoolStart:     "192.168.1.150",
-		PoolEnd:       "192.168.1.250",
-		PoolGateway:   "192.168.1.1",
-		PoolPrefixLen: 24,
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -451,6 +452,94 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 		PoolPrefixLen:  24,
 	}
 	applyNetworkConfig(settings, nc)
-	assert.Equal(t, "dhcp", settings.PoolSource)
-	assert.Equal(t, "br-wan", settings.PoolBindBridge)
+	require.Len(t, settings.Pools, 1)
+	assert.Equal(t, "dhcp", settings.Pools[0].Source)
+	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
+}
+
+func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "predastore.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, predastoreTomlTemplate, settings))
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
+	return []admin.PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"},
+	}
+}
+
+// Unset knob must leave production config byte-identical: no [compaction] block
+// and the original trailing bytes unchanged (single-node has no trailing
+// newline, multinode keeps one).
+func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok",
+	})
+	assert.NotContains(t, single, "[compaction]")
+	assert.True(t, strings.HasSuffix(single, `access_keys_bucket = "spinifex-iam-access-keys"`),
+		"unset single-node tail changed: %q", single[len(single)-40:])
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0,
+	)
+	require.NoError(t, err)
+	assert.NotContains(t, multi, "[compaction]")
+	assert.True(t, strings.HasSuffix(multi, "access_keys_bucket = \"spinifex-iam-access-keys\"\n"),
+		"unset multinode tail changed: %q", multi[len(multi)-40:])
+}
+
+func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
+	const interval = 30
+
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok", CompactionIntervalSeconds: interval,
+	})
+	assert.Contains(t, single, "[compaction]")
+	var singleCfg compactionConfig
+	require.NoError(t, toml.Unmarshal([]byte(single), &singleCfg))
+	assert.Equal(t, interval, singleCfg.Compaction.IntervalSeconds)
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", interval,
+	)
+	require.NoError(t, err)
+	var multiCfg compactionConfig
+	require.NoError(t, toml.Unmarshal([]byte(multi), &multiCfg))
+	assert.Equal(t, interval, multiCfg.Compaction.IntervalSeconds)
+}
+
+// compactionConfig parses just the rendered [compaction] block. Predastore's
+// s3.Config dropped its Compaction field, so this asserts the spinifex template
+// still emits a valid, parseable block carrying the interval.
+type compactionConfig struct {
+	Compaction struct {
+		IntervalSeconds int `toml:"interval_seconds"`
+	} `toml:"compaction"`
+}
+
+// The init/join flag must register with an unset (0) default and flow into the
+// rendered config via ConfigSettings.
+func TestAdminInitFlag_CompactionIntervalReachesRenderedConfig(t *testing.T) {
+	for _, c := range []*cobra.Command{adminInitCmd, adminJoinCmd} {
+		f := c.Flags().Lookup("predastore-compaction-interval")
+		require.NotNil(t, f, "%s missing --predastore-compaction-interval", c.Name())
+		assert.Equal(t, "0", f.DefValue)
+	}
+
+	require.NoError(t, adminInitCmd.Flags().Set("predastore-compaction-interval", "45"))
+	defer func() { _ = adminInitCmd.Flags().Set("predastore-compaction-interval", "0") }()
+
+	v, _ := adminInitCmd.Flags().GetInt("predastore-compaction-interval")
+	require.Equal(t, 45, v)
+
+	out := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", CompactionIntervalSeconds: v,
+	})
+	assert.Contains(t, out, "interval_seconds = 45")
 }
@@ -161,6 +161,93 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 	fmt.Printf("Cluster shutdown complete (%s)\n", time.Since(start).Round(time.Millisecond))
 }

+// runNodeDrainLocal runs the GATE and DRAIN shutdown phases against the local
+// node only, leaving STORAGE/PERSIST/INFRA to systemd's ordered unit teardown.
+// It is the ExecStop of spinifex-shutdown.service: guests are powered down and
+// their volumes unmounted while every service is still up, so the subsequent
+// service stops never tear storage out from under a running guest.
+func runNodeDrainLocal(cmd *cobra.Command, args []string) {
+	local, _ := cmd.Flags().GetBool("local")
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	if !local {
+		fmt.Fprintln(os.Stderr, "Error: node drain currently supports only --local")
+		os.Exit(1)
+	}
+
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	node := cfg.Node
+	fmt.Printf("Draining local node %q (gate -> drain), timeout %s/phase\n", node, timeout)
+
+	for _, phase := range []string{"gate", "drain"} {
+		topic := "spinifex.cluster.shutdown." + phase
+		req := daemon.ShutdownRequest{Phase: phase, Timeout: int(timeout.Seconds())}
+		reqData, err := json.Marshal(req)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshaling request: %v\n", err)
+			os.Exit(1)
+		}
+
+		ack, err := collectLocalShutdownACK(nc, topic, reqData, node, timeout)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[%s] %v\n", strings.ToUpper(phase), err)
+			os.Exit(1)
+		}
+		if ack.Error != "" {
+			fmt.Fprintf(os.Stderr, "[%s] %s: ERROR - %s\n", strings.ToUpper(phase), ack.Node, ack.Error)
+			os.Exit(1)
+		}
+		if len(ack.Stopped) > 0 {
+			fmt.Printf("[%s] %s: stopped %s\n", strings.ToUpper(phase), ack.Node, strings.Join(ack.Stopped, ", "))
+		} else {
+			fmt.Printf("[%s] %s: OK\n", strings.ToUpper(phase), ack.Node)
+		}
+	}
+
+	fmt.Printf("Local node %q drained; systemd may now stop storage services\n", node)
+}
+
+// collectLocalShutdownACK publishes a shutdown-phase request and returns the ACK
+// from the local node, ignoring ACKs from any other node. Returns an error if
+// the local node does not respond within timeout.
+func collectLocalShutdownACK(nc *nats.Conn, topic string, reqData []byte, node string, timeout time.Duration) (daemon.ShutdownACK, error) {
+	inbox := nats.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return daemon.ShutdownACK{}, fmt.Errorf("failed to subscribe to inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
+		return daemon.ShutdownACK{}, fmt.Errorf("failed to publish request: %w", err)
+	}
+	nc.Flush()
+
+	deadline := time.Now().Add(timeout)
+	for {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
+		}
+		var ack daemon.ShutdownACK
+		if err := json.Unmarshal(msg.Data, &ack); err != nil {
+			continue
+		}
+		if ack.Node == node {
+			return ack, nil
+		}
+	}
+}
+
 // runClusterDrainDHCP asks every vpcd to DHCPRELEASE all external-pool leases
 // it currently holds, returning them to the upstream DHCP server. Intended for
 // the teardown path: an env reset otherwise strands held leases until TTL
@@ -1,26 +1,12 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -206,11 +192,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 	for name := range respondedNodes {
 		nodeSet[name] = struct{}{}
 	}
-	nodeNames := make([]string, 0, len(nodeSet))
-	for name := range nodeSet {
-		nodeNames = append(nodeNames, name)
-	}
-	sort.Strings(nodeNames)
+	nodeNames := slices.Sorted(maps.Keys(nodeSet))

 	for _, name := range nodeNames {
 		nodeCfg := cfg.Nodes[name]
@@ -291,7 +273,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 	})

 	tableData := pterm.TableData{
-		{"INSTANCE", "STATUS", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
+		{"INSTANCE", "STATUS", "HEALTH", "CRASHES", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
 	}

 	for _, v := range allVMs {
@@ -299,9 +281,15 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 		if v.LaunchTime > 0 {
 			age = formatUptime(time.Now().Unix() - v.LaunchTime)
 		}
+		health := v.Health
+		if health == "" {
+			health = "-"
+		}
 		tableData = append(tableData, []string{
 			v.InstanceID,
 			v.Status,
+			health,
+			strconv.Itoa(v.CrashCount),
 			v.InstanceType,
 			strconv.Itoa(v.VCPU),
 			formatMemGB(v.MemoryGB),
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -1,32 +1,20 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"os"
 	"strings"

 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/network/external"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/service"
 	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
+	"github.com/mulgadc/spinifex/spinifex/services/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
 	"github.com/mulgadc/spinifex/spinifex/services/viperblockd"
 	"github.com/mulgadc/spinifex/spinifex/vpcd"
@@ -39,6 +27,28 @@ var serviceCmd = &cobra.Command{
 	Short: "Manage Spinifex services",
 }

+// initTelemetry installs the JSON slog default (trace-stamping) and the OTel
+// providers for a service process. The returned func flushes exporters and
+// must be deferred; with no OTLP endpoint configured both are no-ops.
+func initTelemetry(serviceName string, debug bool) func() {
+	level := slog.LevelInfo
+	if debug {
+		level = slog.LevelDebug
+	}
+	otelsetup.SetDefaultJSONLogger(level)
+
+	shutdown, err := otelsetup.Init(context.Background(), serviceName)
+	if err != nil {
+		slog.Warn("otel init", "service", serviceName, "error", err)
+		return func() {}
+	}
+	return func() {
+		if err := shutdown(context.Background()); err != nil {
+			slog.Warn("otel shutdown", "service", serviceName, "error", err)
+		}
+	}
+}
+
 var predastoreCmd = &cobra.Command{
 	Use:   "predastore",
 	Short: "Manage the predastore service",
@@ -126,6 +136,8 @@ var predastoreStartCmd = &cobra.Command{
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")

+		defer initTelemetry("predastore", debug)()
+
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
@@ -288,6 +300,8 @@ var viperblockStartCmd = &cobra.Command{
 			encryptionKeyFile = envKey
 		}

+		defer initTelemetry("viperblockd", false)()
+
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -366,6 +380,8 @@ var natsStartCmd = &cobra.Command{

 		cfgFile := viper.GetString("config")

+		defer initTelemetry("nats", debug)()
+
 		service, err := service.New("nats", &nats.Config{
 			ConfigFile: cfgFile,
 			Port:       port,
@@ -459,6 +475,8 @@ var spinifexStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("spinifex-daemon", false)()
+
 		svc, err := service.New("spinifex", clusterConfig)

 		if err != nil {
@@ -564,6 +582,8 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("awsgw", viper.GetBool("debug"))()
+
 		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)

@@ -621,6 +641,8 @@ var spinifexUIStartCmd = &cobra.Command{
 		tlsCert := viper.GetString("spinifex-ui-tls-cert")
 		tlsKey := viper.GetString("spinifex-ui-tls-key")

+		defer initTelemetry("spinifex-ui", false)()
+
 		svc, err := service.New("spinifex-ui", &spinifexui.Config{
 			Port:    port,
 			Host:    host,
@@ -730,12 +752,13 @@ var vpcdStartCmd = &cobra.Command{
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]

 		// Map cluster-wide external pools to vpcd config
-		var extPools []vpcd.ExternalPoolConfig
+		var extPools []external.ExternalPoolConfig
 		for _, p := range clusterConfig.Network.ExternalPools {
-			extPools = append(extPools, vpcd.ExternalPoolConfig{
+			extPools = append(extPools, external.ExternalPoolConfig{
 				Name:            p.Name,
 				Source:          p.Source,
 				BindBridge:      p.BindBridge,
+				DHCPMAC:         p.DHCPMAC,
 				RangeStart:      p.RangeStart,
 				RangeEnd:        p.RangeEnd,
 				Gateway:         p.Gateway,
@@ -767,6 +790,8 @@ var vpcdStartCmd = &cobra.Command{
 			nodeConfig.BaseDir = baseDir
 		}

+		defer initTelemetry("vpcd", false)()
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -781,6 +806,7 @@ var vpcdStartCmd = &cobra.Command{
 			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
 			BridgeMode:        nodeConfig.VPCD.BridgeMode,
 			AZ:                nodeConfig.AZ,
+			NATExemptCIDRs:    clusterConfig.Network.NATExemptCIDRs,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)
@@ -823,6 +849,77 @@ var vpcdStatusCmd = &cobra.Command{
 	},
 }

+var qmpCollectorCmd = &cobra.Command{
+	Use:   "qmp-collector",
+	Short: "Manage the qmp-collector (guest metrics) service",
+}
+
+var qmpCollectorStartCmd = &cobra.Command{
+	Use:   "start",
+	Short: "Start the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Starting qmp-collector service...")
+
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			fmt.Println("Config file is not set")
+			return
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			fmt.Println("Error loading config file:", err)
+			return
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+
+		defer initTelemetry("qmp-collector", false)()
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{
+			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
+			NatsCACert: nodeConfig.NATS.CACert,
+			BaseDir:    nodeConfig.BaseDir,
+			NodeName:   clusterConfig.Node,
+		})
+		if err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			return
+		}
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service started")
+	},
+}
+
+var qmpCollectorStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Stopping qmp-collector service...")
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
+		if err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			return
+		}
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service stopped")
+	},
+}
+
+var qmpCollectorStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Get status of the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("qmp-collector service status: ...")
+	},
+}
+
 func init() {
 	viper.SetEnvPrefix("SPINIFEX") // Prefix for environment variables
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -1022,4 +1119,11 @@ func init() {
 	vpcdCmd.AddCommand(vpcdStartCmd)
 	vpcdCmd.AddCommand(vpcdStopCmd)
 	vpcdCmd.AddCommand(vpcdStatusCmd)
+
+	// qmp-collector
+	serviceCmd.AddCommand(qmpCollectorCmd)
+
+	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }
@@ -0,0 +1,24 @@
+package cmd
+
+import "testing"
+
+func TestInitTelemetryNoEndpoint(t *testing.T) {
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+
+	tests := []struct {
+		name  string
+		debug bool
+	}{
+		{"info level", false},
+		{"debug level", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			flush := initTelemetry("test-svc", tt.debug)
+			if flush == nil {
+				t.Fatal("initTelemetry returned nil flush func")
+			}
+			flush()
+		})
+	}
+}
@@ -1,5 +1,5 @@
 # AWS Gateway configuration file in TOML format.
-version = "2"
+version = "3"
 region = "{{.Region}}"

 # Enable verbose debugging
@@ -8,8 +8,8 @@ debug = false
 # API request throttling (per-account, per-action token bucket)
 [ratelimit]
 enabled = false
-rate = 20           # Default sustained requests per second per account per action
-burst = 100         # Default maximum burst capacity
+rate = 20 # Default sustained requests per second per account per action
+burst = 100 # Default maximum burst capacity

 [ratelimit.action.RunInstances]
 rate = 2
@@ -26,3 +26,12 @@ burst = 40
 [ratelimit.action.StopInstances]
 rate = 2
 burst = 40
+
+# Per-account service quotas (default off)
+[quota]
+enabled = false
+vcpus = 8
+vpcs = 8
+subnets = 16
+eips = 4
+volumes_gib = 100
@@ -75,3 +75,8 @@ nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -103,4 +103,9 @@ nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}
 nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
-access_keys_bucket = "spinifex-iam-access-keys"
+access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -8,32 +8,41 @@ node = "{{.Node}}"
 email = "{{.OperatorEmail}}"
 {{- end}}

+[aws]
+region = "{{.Region}}"
+internal_suffix = "spinifex.internal"
+
 [network]
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
+{{- range .Pools}}

 [[network.external_pools]]
-name        = "{{.PoolName}}"
-{{- if .PoolSource}}
-source      = "{{.PoolSource}}"
+name        = "{{.Name}}"
+{{- if .Source}}
+source      = "{{.Source}}"
 {{- end}}
-{{- if .PoolBindBridge}}
-bind_bridge = "{{.PoolBindBridge}}"
+{{- if .BindBridge}}
+bind_bridge = "{{.BindBridge}}"
 {{- end}}
-{{- if .PoolStart}}
-range_start = "{{.PoolStart}}"
-range_end   = "{{.PoolEnd}}"
+{{- if .DHCPMAC}}
+dhcp_mac    = "{{.DHCPMAC}}"
 {{- end}}
-{{- if .PoolGateway}}
-gateway     = "{{.PoolGateway}}"
+{{- if .Start}}
+range_start = "{{.Start}}"
+range_end   = "{{.End}}"
 {{- end}}
-{{- if .PoolGatewayIP}}
-gateway_ip  = "{{.PoolGatewayIP}}"
+{{- if .Gateway}}
+gateway     = "{{.Gateway}}"
+{{- end}}
+{{- if .GatewayIP}}
+gateway_ip  = "{{.GatewayIP}}"
+{{- end}}
+prefix_len  = {{if .PrefixLen}}{{.PrefixLen}}{{else}}24{{end}}
+{{- if .DNSServers}}
+dns_servers = [{{range $i, $s := .DNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
-prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
-{{- if .PoolDNSServers}}
-dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
 {{- end}}
 {{- if .BootstrapVpcId}}
@@ -103,6 +112,9 @@ ovn_sb_addr = "{{.OVNSBAddr}}"
 {{- if .ExternalIface}}
 external_interface = "{{.ExternalIface}}"
 {{- end}}
+{{- if .BridgeMode}}
+bridge_mode = "{{.BridgeMode}}"
+{{- end}}

 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"
--- a/Show More
+++ b/Show More