release: update go.mod for v1.10.0

chore: rebuild ui
chore: bump the npm-dependencies group (#437 )
2026-10-24 05:04:03 +00:00 · 2026-06-30 10:30:07 +10:00 · 2026-06-30 09:50:03 +10:00 · 2026-06-30 09:48:38 +10:00 · 2026-06-30 09:21:07 +10:00 · 2026-06-30 09:06:21 +10:00
1230 changed files with 117471 additions and 113246 deletions
@@ -0,0 +1,102 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="gw-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">ECS · EC2 LAUNCH TYPE ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">container instances from spinifex-ecs-node AMI · awsvpc + ALB</text>
+
+  <!-- Internet -->
+  <rect x="120" y="52" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="220" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
+
+  <!-- connector to ALB -->
+  <line x1="220" y1="92" x2="220" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="216,108 224,108 220,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- ALB -->
+  <g transform="translate(90, 116)">
+    <rect width="260" height="60" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">Application Load Balancer</text>
+    <line x1="16" y1="32" x2="244" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="50" font-family="'Segoe UI', sans-serif" font-size="10" fill="#7ee787">target group · target_type = ip</text>
+  </g>
+
+  <!-- Spinifex Gateway (control plane) -->
+  <g transform="translate(560, 110)">
+    <rect width="316" height="72" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">Spinifex Gateway :9999</text>
+    <text x="296" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">control plane</text>
+    <line x1="16" y1="32" x2="300" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <text x="20" y="48" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">ECS agent registers · TLS + SigV4 (IMDS creds)</text>
+    <text x="20" y="63" font-family="'Segoe UI', sans-serif" font-size="10" fill="#FCA5A5">task role creds → container at 169.254.170.2</text>
+  </g>
+
+  <!-- ALB fan-out to subnets (traffic) -->
+  <line x1="220" y1="176" x2="220" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="650" y2="200" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="220" y1="200" x2="220" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="650" y1="200" x2="650" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="435" y="195" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">balances to each task's ENI</text>
+
+  <!-- agent path: subnets up to gateway -->
+  <line x1="718" y1="228" x2="718" y2="182" stroke="#F87171" stroke-width="1.1" stroke-dasharray="3,3" opacity="0.6"/>
+  <polygon points="714,186 722,186 718,178" fill="#F87171" opacity="0.8"/>
+
+  <!-- subnet_a -->
+  <g transform="translate(60, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- container instance -->
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="225" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(520, 228)">
+    <rect width="320" height="156" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="98" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">container instance</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: spinifex-ecs-node</text>
+    <text x="34" y="74" font-family="'Segoe UI', sans-serif" font-size="8.5" fill="#a1a1aa">ecsInstanceRole · ECS agent</text>
+    <rect x="34" y="82" width="118" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="93" y="102" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">task (web)</text>
+    <text x="93" y="116" font-family="'Segoe UI', sans-serif" font-size="8" fill="#a1a1aa" text-anchor="middle">awsvpc ENI</text>
+    <rect x="166" y="82" width="118" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="225" y="103" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">image pull</text>
+    <text x="225" y="117" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">registry / ECR</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="410" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">No ECS launch API — you add capacity by booting EC2 instances from the spinifex-ecs-node AMI (the console's "Provision capacity" wraps RunInstances)</text>
+  <text x="450" y="426" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">The service keeps the desired task count running and registers each task's ENI IP with the ALB target group</text>
+</svg>
@@ -0,0 +1,96 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="900" height="470" viewBox="0 0 900 470">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0a0a0a"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+    <linearGradient id="inet-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb95022"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="pub-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3fb9501f"/>
+      <stop offset="100%" style="stop-color:#3fb95008"/>
+    </linearGradient>
+    <linearGradient id="cp-grad" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#DC26261f"/>
+      <stop offset="100%" style="stop-color:#DC262608"/>
+    </linearGradient>
+  </defs>
+  <rect width="900" height="470" fill="url(#bg)" rx="12"/>
+
+  <text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">EKS · MANAGED KUBERNETES ON SPINIFEX</text>
+  <text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">control plane VM + NLB · workers from eks-node AMI</text>
+
+  <!-- Internet / kubectl -->
+  <rect x="330" y="52" width="240" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
+  <text x="450" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="13" fill="#3fb950" text-anchor="middle" font-weight="700">Internet · kubectl + aws eks</text>
+
+  <!-- connector to control plane -->
+  <line x1="450" y1="92" x2="450" y2="112" stroke="#71717a" stroke-width="1" opacity="0.5"/>
+  <polygon points="446,108 454,108 450,116" fill="#71717a" opacity="0.7"/>
+
+  <!-- Control plane (Spinifex-managed) -->
+  <g transform="translate(290, 116)">
+    <rect width="320" height="76" rx="8" fill="url(#cp-grad)" stroke="#DC2626" stroke-width="1.2" stroke-opacity="0.65"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#F87171" font-weight="700">EKS Control Plane</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#FCA5A5" text-anchor="end">Spinifex-managed</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#DC2626" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="87" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">API NLB (public)</text>
+    <rect x="165" y="42" width="135" height="22" rx="4" fill="#DC26261a" stroke="#DC2626" stroke-opacity="0.5" stroke-width="0.6"/>
+    <text x="232" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#FCA5A5" text-anchor="middle" font-weight="600">apiserver VM</text>
+  </g>
+
+  <!-- authn note -->
+  <rect x="624" y="138" width="172" height="32" rx="6" fill="#0a0a0a" stroke="#71717a" stroke-width="0.6" stroke-opacity="0.5"/>
+  <text x="710" y="151" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">authn: API · access entries</text>
+  <text x="710" y="163" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">aws eks get-token (IAM → RBAC)</text>
+
+  <!-- fan-out to node-group subnets -->
+  <line x1="450" y1="192" x2="450" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="670" y2="210" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="230" y1="210" x2="230" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <line x1="670" y1="210" x2="670" y2="228" stroke="#3fb950" stroke-width="1" stroke-dasharray="3,3" opacity="0.55"/>
+  <text x="450" y="206" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="middle">managed node group · workers join the control plane</text>
+
+  <!-- subnet_a -->
+  <g transform="translate(108, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_a</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <!-- worker node -->
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+  </g>
+
+  <!-- subnet_b -->
+  <g transform="translate(472, 228)">
+    <rect width="320" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
+    <text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">subnet_b</text>
+    <text x="300" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
+    <line x1="16" y1="32" x2="304" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
+    <rect x="20" y="42" width="280" height="92" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.55" stroke-width="0.8"/>
+    <text x="34" y="60" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#c084fc" font-weight="700">worker node</text>
+    <text x="286" y="60" font-family="'Segoe UI', sans-serif" font-size="9" fill="#d8b4fe" text-anchor="end">AMI: eks-node</text>
+    <rect x="34" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="72" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="122" y="72" width="76" height="50" rx="4" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="160" y="100" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#d8b4fe" text-anchor="middle">pod</text>
+    <rect x="210" y="72" width="76" height="50" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.45" stroke-width="0.6"/>
+    <text x="248" y="93" font-family="'SF Mono', ui-monospace, monospace" font-size="9.5" fill="#7ee787" text-anchor="middle">ECR</text>
+    <text x="248" y="107" font-family="'Segoe UI', sans-serif" font-size="8" fill="#7ee787" text-anchor="middle">pull</text>
+  </g>
+
+  <!-- footer note -->
+  <text x="450" y="404" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">You provide the VPC, subnets, and IAM roles · Spinifex provisions the control plane, node-group VMs, and security groups</text>
+  <text x="450" y="420" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Node groups always boot the registered eks-node image · workers need egress (IGW / NAT) to pull images</text>
+</svg>
@@ -0,0 +1,7 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Changes
+
+<!-- The notable changes, as bullets. -->
@@ -27,16 +27,16 @@ jobs:
      cancel-in-progress: false
    env:
      BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      ARTIFACT_DIR: /tmp/spinifex-e2e-baremetal-artifacts-${{ github.run_id }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
    steps:
      - name: Clean Workspace
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
-          # When called (nightly/release) the caller picks the ref (dev on
-          # schedule, the tag on release); standalone dispatch defaults to the
-          # triggering ref.
          ref: ${{ inputs.ref || github.ref }}

      - uses: actions/create-github-app-token@v3
@@ -46,7 +46,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -104,7 +104,6 @@ jobs:
            echo "Ubuntu 26.04 cloud image cached ($(du -h "$UBUNTU_IMG" | cut -f1))"
          fi

-
      - name: Install go-junit-report on runner
        run: go install github.com/jstemmer/go-junit-report/v2@latest

@@ -117,10 +116,6 @@ jobs:
          SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
        run: |
          mkdir -p "${ARTIFACT_DIR}"
-          # Capture combined bootstrap + e2e output so failures produce a
-          # downloadable, analyzable artifact (the suites run go test -v on the
-          # init node and stream back over SSH). pipefail keeps the script's
-          # real exit code through the tee.
          set -o pipefail
          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
            --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
@@ -142,12 +137,12 @@ jobs:
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "Bare-Metal E2E failure analysis"
+          title: "E2E failure analysis (cell 20)"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
-          name: e2e-baremetal-artifacts-${{ github.run_id }}
+          name: e2e-artifacts-cell-20-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14
@@ -21,10 +21,6 @@ jobs:
  ddil:
    name: DDIL Real Multi-Node
    runs-on: [ self-hosted, ci-multi ]
-    # Budget: ~45m go test + tofu provision/bootstrap/teardown on ci-multi.
-    # In Phase 1 the scenarios all t.Skip, so the real-world runtime is
-    # dominated by provisioning; the 60m ceiling leaves headroom for
-    # Phase 2/3 when assertions come online.
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
@@ -39,16 +35,12 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      # Full spinifex checkout: the Go harness must run on the runner
-      # (outside the cluster) so its SSH path survives scenario partitions
-      # that drop peer-to-peer cluster traffic. Running inside the cluster
-      # would self-sever when Scenario C isolates a node.
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Spinifex
        with:
          path: spinifex

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Mulga (tofu-cluster)
        with:
          repository: mulgadc/mulga
@@ -88,7 +80,6 @@ jobs:
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
-          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
@@ -140,10 +131,6 @@ jobs:
            exit 1
          fi

-          # Guest SSH key: bootstrap-install.sh imports spinifex-key into EC2;
-          # the matching private material was generated on the primary node at
-          # ~/.ssh/spinifex-key. Pull it to the runner so the harness can SSH
-          # the witness via its MapPublicIpOnLaunch-assigned EIP.
          GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
          scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
            "${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
@@ -151,11 +138,6 @@ jobs:

          mkdir -p "${RUNNER_TEMP}/ddil-results"
          cd "${GITHUB_WORKSPACE}/spinifex"
-          # gotestsum: human-readable PASS/FAIL/SKIP lines in Actions stdout,
-          # plus machine-readable jsonfile + JUnit XML artifacts. The JUnit
-          # file feeds the junit-report composite action below so failures
-          # become summary table rows + workflow annotations.
-          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          banner "DDIL scenarios"
          DDIL_NODES="${NODE_IPS}" \
@@ -185,10 +167,6 @@ jobs:
        if: failure()
        working-directory: mulga/scripts/tofu-cluster
        run: |
-          # Pull spinifex-* journals + QEMU process listings from every
-          # node into the artifact bundle so failures can be triaged
-          # without re-provisioning the cluster. Best-effort — a missing
-          # node should not mask the original test failure.
          set +e
          INVENTORY=$(tofu output -json inventory)
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -197,7 +175,6 @@ jobs:
          LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
          mkdir -p "${LOG_DIR}"

-          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
          for ip in "${NODE_IPS[@]}"; do
@@ -221,7 +198,6 @@ jobs:
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
-          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
@@ -40,6 +40,7 @@ jobs:
    timeout-minutes: 35
    env:
      TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}

    steps:
      # blank or scheduled run = full matrix.
@@ -60,7 +61,10 @@ jobs:

      - name: Clean Workspace
        if: steps.gate.outputs.selected == 'true'
-        run: sudo rm -rf "$GITHUB_WORKSPACE"/*
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        if: steps.gate.outputs.selected == 'true'
@@ -70,7 +74,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          repository: mulgadc/mulga
@@ -78,6 +82,11 @@ jobs:
          sparse-checkout: scripts/tofu-cluster
          path: mulga

+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.selected == 'true'
+        with:
+          path: spinifex
+
      - name: Build Install Artifacts
        if: steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
@@ -132,8 +141,18 @@ jobs:
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in single cert lb; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"

      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
@@ -144,32 +163,57 @@ jobs:
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

-          NODE_IPS=""
+          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
-            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

-          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
+             for SUITE in multinode cert lb; do \
+               echo \"::group::suite \$SUITE\"; \
+               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+                 SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+                 SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
+                 ./\$SUITE.test -test.v -test.timeout=30m \
+                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
+               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
+             done; exit \$FAIL"

      - name: Run Tofu Workbooks
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
        working-directory: mulga/scripts/tofu-cluster
+        # The driver emits go test -v markers per workbook; teeing to
+        # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
+        # chain the Go suites use, so cell 17 produces an RCA bundle too.
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh"
+            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
+            2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc

      - name: Run Reboot Resilience E2E
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
        working-directory: mulga/scripts/tofu-cluster
-        # Driver runs from the runner (NOT via ssh-and-run) so it survives the
+        # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
        # systemctl-reboot-induced ssh disconnect mid-run.
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -178,31 +222,85 @@ jobs:
          REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
          mkdir -p "$REBOOT_DIR"
          tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
-          chmod +x "$REBOOT_DIR/run-reboot-e2e.sh"
+          chmod +x "$REBOOT_DIR/_bin/reboot.test"

-          WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
+          CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
+          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"

-      - name: Dump Cluster Journals on Failure
-        if: failure() && steps.gate.outputs.selected == 'true'
+          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
+          GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+            SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
+            SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
+            SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
+            ARTIFACT_DIR="${ARTIFACT_DIR}" \
+            "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
+            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
+          rc=${PIPESTATUS[0]}
+          if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
+          exit $rc
+
+      - name: Collect e2e artifacts
+        if: always() && steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+
          for i in $(seq 0 $((NODE_COUNT - 1))); do
-            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
-            echo "===== node$((i+1)) ($IP) ====="
-            for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd dhcpcd; do
-              echo "--- $svc ---"
-              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-                "sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)"
-            done
-            echo "--- DHCP client state ---"
-            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
-              "systemctl is-enabled dhcpcd.service 2>/dev/null; systemctl is-active dhcpcd.service 2>/dev/null; sudo ss -lupn 'sport = 68' 2>/dev/null; pgrep -laf dhcpcd 2>/dev/null" 2>&1 || echo "(node unreachable)"
+            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
+            mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+              | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
          done

+          cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Install go-junit-report
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Convert test logs to JUnit XML
+        if: always() && steps.gate.outputs.selected == 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.selected == 'true'
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E failure analysis (cell ${{ matrix.cell }})"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.selected == 'true'
+        with:
+          name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
@@ -217,8 +315,7 @@ jobs:
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

-  # Bare-metal single-node suite (PXE-reimaged physical hardware). Doesn't fit
-  # the tofu matrix shape (different runner/provisioning/120m timeout), so it
+  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
  # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
@@ -0,0 +1,799 @@
+# E2E (per-suite blocks) — EXPERIMENTAL, workflow_dispatch only.
+#
+# Restructures the looped e2e.yml into env-grouped job blocks so each suite
+# surfaces as its own node in the Actions graph (mulga-siv-283):
+#
+#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
+#                     └─ single_teardown (if: always)
+#   multi_provision  ─┬─ multi_suite  (matrix: multinode)
+#                     └─ multi_teardown (if: always)
+#
+# Constraint: tofu state is runner-filesystem-local, so *_provision and
+# *_teardown must execute on the same self-hosted runner (one runner per label
+# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
+# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach
+# the already-provisioned VM over its WAN IP using the key path the provision
+# job emits; they share the runner home dir, so the key file persists.
+#
+# This file does NOT post the gating `e2e` commit status — it runs alongside
+# e2e.yml for validation. Once green across a few manual runs, fold the
+# triggers from e2e.yml here and retire the loop.
+
+name: E2E (suite blocks)
+
+on:
+  workflow_dispatch:
+    inputs:
+      single_suites:
+        description: "Single-node suites to run as matrix legs"
+        required: false
+        default: "single cert lb eks iam"
+      run_multinode:
+        description: "Also run the multinode env block"
+        type: boolean
+        required: false
+        default: true
+      test_run:
+        description: "Optional -test.run regex (empty = all tests in suite)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Single-node env
+  # ---------------------------------------------------------------------------
+  single_provision:
+    name: Single env · provision
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 40
+    outputs:
+      wan_ip: ${{ steps.vm.outputs.wan_ip }}
+      ssh_key: ${{ steps.vm.outputs.ssh_key }}
+      suites: ${{ steps.suites.outputs.list }}
+      matrix: ${{ steps.suites.outputs.matrix }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      - name: Compute suite matrix
+        id: suites
+        run: |
+          LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
+          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
+          JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
+          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "VM provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve VM SSH details
+        id: vm
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(steps.suites.outputs.list, 'eks')
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build eks-node AMI (libguestfs — no host devices)"
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          group_open "make build-eks-node-image"
+          make build-eks-node-image
+          group_close
+          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
+
+      - name: Import eks-node AMI on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/eks-node/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "import eks-node AMI to VM image store"
+          scp "${SSH_OPTS[@]}" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
+            "$VM":/tmp/eks-node-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/eks-node-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+          ok "eks-node AMI imported"
+
+      - name: Install kubectl on VM
+        if: contains(steps.suites.outputs.list, 'eks')
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          banner "install kubectl on VM (eks data-plane subtest)"
+          ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
+          set -euo pipefail
+          KVER=v1.32.5
+          case "$(uname -m)" in
+            x86_64)  ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
+          esac
+          cd /tmp
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
+          sudo install -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+          EOF
+          ok "kubectl installed on VM"
+
+      - name: Build E2E test binaries on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # teardown always builds: the run-scoped sweep binary runs in the
+          # teardown job, it is never a selectable suite leg.
+          SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
+          banner "build ${SUITES_BUILD// /, }"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
+          group_close
+          for f in _bin/*.test; do
+            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
+          done
+
+      - name: Ship test binaries to VM
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on VM br-wan
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+            "$VM":/tmp/start-net-capture.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
+
+  single_suite:
+    name: Single env · suite
+    needs: single_provision
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+    env:
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          VM_HOST="tf-user@${WAN_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
+             printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from VM
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: ./spinifex/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "E2E ${{ matrix.suite }} results"
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  single_teardown:
+    name: Single env · teardown
+    needs: [single_provision, single_suite]
+    if: always()
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }}
+      WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
+      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
+    steps:
+      # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
+      # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
+      # again would wipe the local tofu state and orphan the VM.
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Capture network/IPsec diagnostics on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            "$VM":/tmp/capture-net-diag.sh 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \
+            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
+          mkdir -p "${ARTIFACT_DIR}/lb-console"
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+            | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
+          true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-single-diag-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: ignore
+          retention-days: 14
+
+      # Reclaim any resource leaked by a crashed/interrupted suite before the VM
+      # is destroyed. Scoped to this run's tag, best-effort, never fails the job.
+      - name: Sweep leaked run resources on VM
+        if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${WAN_IP}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown VM
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+              tf-user@${WAN_IP} \
+              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+              2>/dev/null || true
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+  # ---------------------------------------------------------------------------
+  # Multinode env
+  # ---------------------------------------------------------------------------
+  multi_provision:
+    name: Multi env · provision
+    if: ${{ inputs.run_multinode }}
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 45
+    outputs:
+      node1_ip: ${{ steps.cluster.outputs.node1_ip }}
+      ssh_key: ${{ steps.cluster.outputs.ssh_key }}
+      nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 3; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      - name: Reclaim docker build cache
+        run: |
+          docker container prune -f || true
+          docker image prune -f || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          ./build-install-artifacts.sh "${GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      - name: Build multinode binary on runner
+        working-directory: spinifex/tests/e2e
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          banner "build multinode, teardown"
+          group_open "go test -c"
+          make e2e-build SUITES_BUILD="multinode teardown"
+          group_close
+          ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
+
+      - name: Ship test binaries to node1
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
+          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
+          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
+          ok "shipped binaries to $VM"
+
+      - name: Start packet capture on all nodes
+        run: |
+          set +e
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            scp "${SSH_OPTS[@]}" \
+              "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
+              tf-user@"$ip":/tmp/start-net-capture.sh
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
+          done
+
+  multi_suite:
+    name: Multi env · suite
+    needs: multi_provision
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [multinode]
+    env:
+      NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Install go-junit-report
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Run ${{ matrix.suite }}
+        id: e2e
+        run: |
+          set +e
+          SUITE="${{ matrix.suite }}"
+          VM_HOST="tf-user@${NODE1_IP}"
+          TEST_RUN_FLAG=""
+          [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          cleanup_remote() {
+            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
+              "pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
+              2>/dev/null || true
+          }
+          trap cleanup_remote EXIT INT TERM
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
+             printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
+                    | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
+             exit \${PIPESTATUS[0]}"
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect suite artifacts from node1
+        if: always()
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
+          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          true
+
+      - name: Convert test log to JUnit XML
+        if: always()
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always()
+        uses: ./spinifex/.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "Multi-Node E2E ${{ matrix.suite }} results"
+
+      - name: Analyze e2e failures
+        if: always()
+        uses: ./spinifex/.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Fail job if suite failed
+        if: always()
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
+            exit 1
+          fi
+
+  multi_teardown:
+    name: Multi env · teardown
+    needs: [multi_provision, multi_suite]
+    if: always() && inputs.run_multinode
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 20
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
+      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
+      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Reclaim leaked run-tagged resources before the cluster is destroyed.
+      - name: Sweep leaked run resources on node1
+        if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }}
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
+          VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }}
+          banner "sweep run-tagged resources (run ${{ github.run_id }})"
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
+             SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+             SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
+             ./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
+            2>&1 || true
+          true
+
+      - name: Teardown cluster
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
+            IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
+            for ip in "${NODE_LIST[@]}"; do
+              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
+                tf-user@"$ip" \
+                'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
+                2>/dev/null || true
+            done
+          fi
+          UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
+            done
+          }
+          reap TERM; sleep 2; reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+  e2e_suites_status:
+    name: E2E suite blocks · summary
+    if: always()
+    needs: [single_suite, multi_suite]
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Summarize
+        run: |
+          echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY"
+          echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "multi_suite:  ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
+          case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
+          case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
@@ -14,13 +14,10 @@ on:
      - .github/workflows/e2e.yml
  workflow_dispatch:
    inputs:
-      sha:
-        description: "PR head SHA to report status on"
-        required: false
      suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
        required: false
-        default: "e2e-single e2e-multinode e2e-cert e2e-lb"
+        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
      test_run:
        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
        required: false
@@ -32,19 +29,55 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}

 jobs:
+  # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
+  # Does NOT gate or override the real SUITES env yet — it runs alongside the full matrix so we
+  # can compare predicted vs actual before flipping it on.
+  select_suites:
+    name: Select Suites (shadow)
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+          cache: false
+      - name: Compute targeted suite selection
+        run: |
+          set -euo pipefail
+          BASE="${{ github.event.before }}"
+          HEAD="${{ github.sha }}"
+          # First push to a branch reports an all-zero before-SHA; fall back to
+          # the first parent so the diff is still meaningful.
+          if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then
+            BASE="${HEAD}~1"
+          fi
+          echo "## Targeted suite selection (shadow)" >> "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          echo "base=$BASE head=$HEAD" >> "$GITHUB_STEP_SUMMARY"
+          go run ./tests/e2e/select-suites/cmd/select-suites \
+            --manifest docs/service-interfaces.yaml \
+            --base-sha "$BASE" --head-sha "$HEAD" --repo-root . \
+            2>>"$GITHUB_STEP_SUMMARY" \
+            | tee -a "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          echo "Shadow only — actual run uses SUITES='${SUITES}'." >> "$GITHUB_STEP_SUMMARY"
+
  e2e_single:
    name: Single-Node E2E
    # Skip when the dispatch input requests no in-VM suites (e.g.
-    # suites='e2e-reboot' only). Push triggers and the default SUITES list
+    # suites='e2e-multinode' only). Push triggers and the default SUITES list
    # both include single/cert/lb so the inputs.suites null-check keeps the
    # job running on plain PR pushes.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
-    runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 45
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-iam')
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 50
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -71,14 +104,13 @@ jobs:

      - name: Clean Workspace
        run: |
-          # Detach any leaked mounts under the workspace before deleting. A
-          # killed image build can leave a `mount --bind /dev` under its build
-          # dir; if `rm -rf` traverses it as root it wipes the host's /dev
-          # (deletes /dev/null and wedges the box). umount deepest-first, then
-          # forbid rm from crossing any mount boundary as a hard backstop.
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -87,14 +119,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -106,10 +138,10 @@ jobs:
      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
        run: |
          docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

@@ -184,7 +216,6 @@ jobs:
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          # Pull import args straight from the manifest so they never drift.
          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
            spinifex/scripts/images/eks-node/manifest.conf)"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
@@ -193,13 +224,10 @@ jobs:
          scp "${SSH_OPTS[@]}" \
            /tmp/eks-node-image-build/eks-node-alpine.raw \
            "$VM":/tmp/eks-node-alpine.raw
-          # Drop the import progress bar (pterm redraws to stdout — thousands of
-          # lines over the SSH pipe). Errors go to stderr, so they survive; the
-          # exit code still gates the step.
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -238,7 +266,7 @@ jobs:
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
+          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
@@ -280,7 +308,7 @@ jobs:
        id: e2e
        run: |
          set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -436,16 +464,16 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

-  e2e_real_multi:
-    name: Real Multi-Node E2E
+  e2e_multi:
+    name: Multi-Node E2E
    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
@@ -473,14 +501,13 @@ jobs:

      - name: Clean Workspace
        run: |
-          # Detach any leaked mounts under the workspace before deleting. A
-          # killed image build can leave a `mount --bind /dev` under its build
-          # dir; if `rm -rf` traverses it as root it wipes the host's /dev
-          # (deletes /dev/null and wedges the box). umount deepest-first, then
-          # forbid rm from crossing any mount boundary as a hard backstop.
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -489,14 +516,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -508,10 +535,10 @@ jobs:
      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
        run: |
          docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

@@ -568,12 +595,15 @@ jobs:
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # single + iam are single-node-only suites — their fixtures skip
+          # unless SPINIFEX_MODE=single, so building/running them on multinode
+          # is wasted work.
          # eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
          # stream and create-cluster needs an elected leader, but multinode
          # JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
          # validated single-node only for now; re-enable here once clustered
          # JetStream + multinode EKS are a deliberate target.
-          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
+          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
@@ -635,9 +665,6 @@ jobs:
            echo "=== systemctl is-active spinifex-* ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
            echo
-            echo "=== which spx + version ==="
-            ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true'
-            echo
            echo "=== spx get nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
            echo
@@ -656,7 +683,7 @@ jobs:
        id: e2e
        run: |
          set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -851,303 +878,33 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

-  e2e_reboot:
-    name: Reboot Resilience E2E
-    # Opt-in: only fires when the SUITES dispatch input lists e2e-reboot.
-    # The reboot test reboots the spinifex VM, so it must run on the runner
-    # (not shipped to the VM like the cert/lb/single suites). Runs in
-    # parallel with e2e_single — both target [self-hosted, ci-single]
-    # and GitHub Actions serialises on the underlying runner allocation,
-    # so no explicit needs: ordering is required.
-    if: contains(github.event.inputs.suites, 'e2e-reboot')
-    runs-on: [ self-hosted, ci-single ]
-    timeout-minutes: 60
-    env:
-      TOFU_LOG: /tmp/e2e-tofu-reboot-${{ github.run_id }}.log
-      ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }}
-    steps:
-      - name: Reap Orphan Procs
-        run: |
-          set +e
-          UID_SELF=$(id -u)
-          WS="${GITHUB_WORKSPACE}"
-          reap() {
-            local sig="$1"
-            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
-              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
-              case "$cwd" in
-                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
-              esac
-            done
-          }
-          reap TERM
-          sleep 3
-          reap KILL
-          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
-          true
-
-      - name: Clean Workspace
-        run: |
-          # Detach any leaked mounts under the workspace before deleting. A
-          # killed image build can leave a `mount --bind /dev` under its build
-          # dir; if `rm -rf` traverses it as root it wipes the host's /dev
-          # (deletes /dev/null and wedges the box). umount deepest-first, then
-          # forbid rm from crossing any mount boundary as a hard backstop.
-          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
-            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
-          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
-
-      - uses: actions/create-github-app-token@v3
-        id: app-token
-        with:
-          client-id: ${{ secrets.APP_ID }}
-          private-key: ${{ secrets.APP_PRIVATE_KEY }}
-          repositories: mulga
-
-      - uses: actions/checkout@v6
-        with:
-          repository: mulgadc/mulga
-          token: ${{ steps.app-token.outputs.token }}
-          sparse-checkout: scripts/tofu-cluster
-          path: mulga
-
-      - uses: actions/checkout@v6
-        with:
-          path: spinifex
-
-      - uses: actions/setup-go@v6
-        with:
-          go-version-file: spinifex/go.mod
-          cache: false
-
-      - name: Re-enable Go toolchain auto-download
-        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
-
-      - name: Prune rootless docker cache
-        run: |
-          docker container prune -f || true
-          docker image prune -af || true
-          docker buildx prune -af --filter "until=24h" || true
-          docker system df || true
-
-      - name: Build Install Artifacts
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          ./build-install-artifacts.sh "${GIT_BRANCH}" \
-            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
-            "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
-
-      - name: Provision VM
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          set -euo pipefail
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          export TF_VAR_git_branch="${GIT_BRANCH}"
-          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
-          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
-          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
-          ok "VM provisioned"
-
-      - name: Bootstrap via Binary Install
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
-          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
-            "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
-            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
-
-      - name: Resolve VM SSH details
-        id: vm
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          INVENTORY=$(tofu output -json inventory)
-          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
-          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
-          SSH_KEY="${SSH_KEY/#\~/$HOME}"
-          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
-          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
-
-      - name: Build reboot.test on runner
-        working-directory: spinifex/tests/e2e
-        run: |
-          set -euo pipefail
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build reboot"
-          group_open "go test -c"
-          make e2e-build SUITES_BUILD="reboot"
-          group_close
-          ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)"
-
-      - name: Fetch AWS credentials from VM
-        id: creds
-        run: |
-          set -euo pipefail
-          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-                    -i "${{ steps.vm.outputs.ssh_key }}")
-          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials')
-          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
-          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
-          if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then
-            echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}"
-            exit 1
-          fi
-          echo "::add-mask::${AWS_AK}"
-          echo "::add-mask::${AWS_SK}"
-          echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT"
-          echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT"
-
-      - name: Run reboot suite on runner
-        id: e2e
-        run: |
-          set +e
-          mkdir -p "${ARTIFACT_DIR}"
-          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
-          R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m'
-          printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R"
-
-          if [ -n "${TEST_RUN}" ]; then
-            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
-          else
-            TEST_RUN_FLAG=""
-          fi
-          GITHUB_ACTIONS=true FORCE_COLOR=1 \
-            SPINIFEX_E2E=1 SPINIFEX_MODE=single \
-            SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \
-            SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \
-            SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \
-            SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \
-            SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \
-            ARTIFACT_DIR="${ARTIFACT_DIR}" \
-            ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
-            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \
-                 | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'
-          rc=${PIPESTATUS[0]}
-          echo "rc=${rc}" >> "$GITHUB_OUTPUT"
-          exit 0
-
-      - name: Collect VM journals
-        if: always()
-        run: |
-          set +e
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          mkdir -p "${ARTIFACT_DIR}"
-          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \
-                    -i "${{ steps.vm.outputs.ssh_key }}")
-          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
-            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
-          ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
-
-      - name: Install go-junit-report
-        if: always()
-        run: |
-          go install github.com/jstemmer/go-junit-report/v2@latest
-          echo "$HOME/go/bin" >> "$GITHUB_PATH"
-
-      - name: Convert test log to JUnit XML
-        if: always()
-        run: |
-          shopt -s nullglob
-          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
-            BASE=$(basename "$LOG" .log)
-            SUITE=${BASE#test-}
-            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
-          done
-
-      - name: Publish test summary
-        if: always()
-        uses: ./spinifex/.github/actions/junit-report
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          title: "E2E reboot results"
-
-      - name: Analyze e2e failures
-        if: always()
-        uses: ./spinifex/.github/actions/e2e-analyze
-        with:
-          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "Reboot E2E failure analysis"
-
-      - uses: actions/upload-artifact@v7
-        if: always()
-        with:
-          name: e2e-reboot-artifacts-${{ github.run_id }}
-          path: ${{ env.ARTIFACT_DIR }}
-          if-no-files-found: warn
-          retention-days: 14
-
-      - name: Teardown
-        if: always()
-        working-directory: mulga/scripts/tofu-cluster
-        run: |
-          set +e
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          UID_SELF=$(id -u)
-          WS="${GITHUB_WORKSPACE}"
-          reap() {
-            local sig="$1"
-            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
-              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
-              case "$cwd" in
-                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
-              esac
-            done
-          }
-          reap TERM
-          sleep 2
-          reap KILL
-          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
-          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
-          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
-          ok "teardown complete"
-
-      - name: Fail job if reboot suite failed
-        if: always()
-        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
-            echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})"
-            exit 1
-          fi
-
  e2e_status:
    name: Report E2E Status
    if: always()
-    needs: [ e2e_single, e2e_real_multi, e2e_reboot ]
-    runs-on: ubuntu-latest
+    needs: [e2e_single, e2e_multi]
+    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
        env:
          GH_TOKEN: ${{ github.token }}
-          REPORT_SHA: ${{ inputs.sha || github.sha }}
+          REPORT_SHA: ${{ github.sha }}
        run: |
-          if [[ -z "${REPORT_SHA}" ]]; then
-            echo "::error::no SHA available (inputs.sha and github.sha both empty)"
-            exit 1
-          fi
-          single_ok=true; multi_ok=true; reboot_ok=true
+          single_ok=true; multi_ok=true
          case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
-          case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
-          case "${{ needs.e2e_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac
-          if $single_ok && $multi_ok && $reboot_ok; then
+          case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
+          if $single_ok && $multi_ok; then
            STATE="success"
            DESC="E2E suites passed"
          else
            STATE="failure"
-            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }} reboot=${{ needs.e2e_reboot.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
            -f state="$STATE" \
@@ -26,23 +26,23 @@ concurrency:
 jobs:
  publish:
    name: Build + publish eks-node AMI
-    runs-on: ubuntu-24.04
+    runs-on: [ self-hosted, ci-single ]

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -51,8 +51,12 @@ jobs:
      - name: Setup Go
        uses: actions/setup-go@v6
        with:
-          cache-dependency-path: "spinifex/go.sum"
          go-version-file: "spinifex/go.mod"
+          # Self-hosted runner: the Go module + build cache already persists on
+          # local disk between runs. setup-go's cache-save ("Post Setup Go")
+          # otherwise re-uploads hundreds of MB to the Actions cache backend,
+          # taking longer than the build itself.
+          cache: false

      - name: Setup Go Workspace
        run: go work init ./spinifex ./viperblock ./predastore
@@ -9,21 +9,21 @@ permissions:

 jobs:
  distro-amd64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -34,7 +34,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v6
        with:
-          go-version: '1.26.4'
+          go-version: "1.26.4"

      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
        run: cd spinifex && make build-microvm-image
@@ -54,21 +54,21 @@ jobs:
            spinifex/scripts/setup.sh

  distro-arm64:
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-26.04-arm
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -91,7 +91,7 @@ jobs:

  release:
    needs: [ distro-amd64, distro-arm64 ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Download amd64 Artifacts
        uses: actions/download-artifact@v8
@@ -132,12 +132,6 @@ jobs:
    steps:
      - name: Clean Workspace
        run: |
-          # This runner (ci-single) is shared with the iso job, which mounts a
-          # kernel-global devtmpfs at <workspace>/mulga/scripts/iso-builder/.cache/rootfs/dev.
-          # If that job is killed its EXIT-trap unmount never runs and the bind
-          # leaks; a plain rm -rf then traverses it and deletes the host's /dev
-          # (wipes /dev/null, wedges the box). Detach any leaked mounts under the
-          # workspace first, then forbid rm from crossing a mount boundary.
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
@@ -149,7 +143,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -186,10 +180,6 @@ jobs:
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

-  # Bare-metal single-node suite on the real ISO/squashfs + PXE-boot path —
-  # the only release check that exercises hardware install. Runs in parallel
-  # with `verify` (both need: release). Promotion and ISO upload both hard-gate
-  # on this passing.
  baremetal:
    needs: release
    permissions:
@@ -198,11 +188,9 @@ jobs:
    uses: ./.github/workflows/e2e-baremetal.yml
    secrets: inherit

-  # Promote the draft only after BOTH the tofu install (`verify`) and the
-  # bare-metal hardware boot (`baremetal`) pass — runs only if both succeed.
  promote:
    needs: [ verify, baremetal ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -223,7 +211,7 @@ jobs:
  mark-failed:
    needs: [ verify, baremetal ]
    if: failure()
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -248,8 +236,6 @@ jobs:
      - name: Clean Workspace
        run: |
          ROOTFS="${GITHUB_WORKSPACE}/mulga/scripts/iso-builder/.cache/rootfs"
-          # Mirror make clean's umount sequence unconditionally — lazy unmount
-          # detaches immediately even if the mount is busy, so rm -rf is safe.
          sudo umount -l -R "${ROOTFS}" 2>/dev/null || true
          sudo umount -l "${ROOTFS}/dev/pts" 2>/dev/null || true
          sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
@@ -265,25 +251,25 @@ jobs:
          repositories: mulga

      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          ref: ${{ github.ref }}

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock

      - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -335,9 +321,6 @@ jobs:
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
-          # Gate the R2 upload on BOTH the tofu install (`verify`) and the
-          # real-hardware boot (`baremetal`, a reusable workflow whose job shows
-          # as "baremetal / ..."). Upload only if both conclude success.
          echo "Waiting for verify + baremetal jobs to complete..."
          while true; do
            JOBS=$(gh api \
@@ -23,24 +23,24 @@ permissions:
 jobs:
  build_and_test:
    name: Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          fetch-depth: 2

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -58,6 +58,9 @@ jobs:
      - name: Build Spinifex
        run: make -C spinifex build

+      - name: Manifest check + drift lint
+        run: make -C spinifex manifest-check manifest-lint
+
      - name: Run Tests with Coverage
        run: make -C spinifex test-cover

@@ -69,23 +72,23 @@ jobs:

  race_detection:
    name: Race Detection
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -105,23 +108,23 @@ jobs:

  lint_and_security:
    name: Lint and Security
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -23,14 +23,14 @@ permissions:
 jobs:
  ui_build_and_test:
    name: UI Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    defaults:
      run:
        working-directory: spinifex/services/spinifexui/frontend

    steps:
      - name: Checkout Code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          sparse-checkout: spinifex/services/spinifexui/frontend

@@ -37,13 +37,3 @@ Thumbs.db

 # Test reports
 *-report.txt
-
-# Internal docs (moved to private mulga repo)
-docs/INTERNAL.md
-docs/PRIVATE.md
-docs/DEVNOTES.md
-docs/PLAN.md
-docs/TODO.md
-docs/development/
-docs/internal/
-scripts/iac/aws/spinifex-test.pem
@@ -57,7 +57,6 @@ linters:
      check-type-assertions: true
      exclude-functions:
        - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable
-        - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable

    govet:
      enable-all: true
@@ -1,3 +0,0 @@
-# Spinifex Installation
-
-For documentation on using and installing Spinifex see [https://docs.mulgadc.com](https://docs.mulgadc.com/)
@@ -6,17 +6,12 @@ ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
  GO_ARCH := amd64
  AWS_ARCH := x86_64
-  # On x86, we can run ARM VMs via emulation
-  QEMU_PACKAGES := qemu-system-x86 qemu-system-arm
 else ifeq ($(ARCH),aarch64)
  GO_ARCH := arm64
  AWS_ARCH := aarch64
-  # On ARM, we can run x86 VMs via emulation
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else ifeq ($(ARCH),arm64)
  GO_ARCH := arm64
  AWS_ARCH := aarch64
-  QEMU_PACKAGES := qemu-system-aarch64 qemu-system-x86
 else
  $(error Unsupported architecture: $(ARCH). Only x86_64 and aarch64/arm64 are supported.)
 endif
@@ -73,6 +68,10 @@ build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
 	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go

+build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
+	@echo -e "\n....Building ecs-agent (static)"
+	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
 	$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
@@ -95,6 +94,12 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build

+build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=ecs-agent
+
+import-ecs-node-image: ## Build + register the ecs-node AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=ecs-agent IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -115,14 +120,10 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
 	sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz

-go_run:
-	@echo -e "\n....Running $(GO_PROJECT_NAME)...."
-	$(GOPATH)/bin/$(GO_PROJECT_NAME)
-
 # Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
 # Use this before committing to catch CI failures locally.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check lint govulncheck test-cover diff-coverage test-race test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness
 	@echo -e "\n ✅ Preflight passed — safe to commit."

 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
@@ -139,6 +140,17 @@ manifest-check:
 	@echo -e "\n....Checking service-interfaces.yaml...."
 	@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml

+# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
+# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
+# drift beyond the baseline.
+manifest-lint:
+	@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
+
+# Accept current drift into the baseline. Run after an intentional change.
+manifest-lint-update:
+	@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
+
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
@@ -168,13 +180,8 @@ diff-coverage: test-cover

 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
-	$(MAKE) easyjson
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...

-run:
-	$(MAKE) go_build
-	$(MAKE) go_run
-
 # Fast iteration: build + install binary + restart all services.
 # Microvm artifacts are reinstalled when they already exist on disk — the rule's
 # input timestamps drive a rebuild only if anything actually changed. On a fresh
@@ -201,11 +208,10 @@ clean:

 install-system:
 	@echo -e "\n....Installing system dependencies for $(ARCH)...."
-	@echo "QEMU packages: $(QEMU_PACKAGES)"
 	sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		-o Dpkg::Options::="--force-confdef" \
 		-o Dpkg::Options::="--force-confold" \
-		nbdkit nbdkit-plugin-dev pkg-config $(QEMU_PACKAGES) qemu-utils qemu-kvm \
+		nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
 		ovmf qemu-efi-aarch64 \
 		libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
 		iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
@@ -234,19 +240,15 @@ install-aws:
 quickinstall: install-system install-go install-aws
 	@echo -e "\n✅ Quickinstall complete for $(ARCH)."
 	@echo "   Please ensure /usr/local/go/bin is in your PATH."
-	@echo "   Installed: Go ($(GO_ARCH)), AWS CLI ($(AWS_ARCH)), QEMU ($(QEMU_PACKAGES))"

-# Lint all Go code via golangci-lint (replaces check-format, vet, gosec, staticcheck)
 lint:
 	@echo "Running golangci-lint..."
 	$(_Q)golangci-lint run ./...
 	@echo "  golangci-lint ok"

-# Auto-fix all linter issues that have fixers
 fix:
 	golangci-lint run --fix ./...

-# Govulncheck — dependency vulnerability scanning (not covered by golangci-lint)
 govulncheck:
 	@echo "Running govulncheck..."
 	$(_Q)go tool govulncheck ./...
@@ -295,65 +297,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/

-# Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh).
-# See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md.
-#
-# Variable overrides: pass EXTRA_VARS="key=val key2=val2" to forward as
-# `--extra-vars` to ansible-playbook. Plain `-e` on the make command line is
-# make's own flag and does NOT reach ansible.
-#   make ansible-dev-install EXTRA_VARS="spinifex_external_pool_start=192.168.1.90 spinifex_external_pool_end=192.168.1.99"
-EXTRA_VARS ?=
-_ANSIBLE_EXTRA = $(if $(strip $(EXTRA_VARS)),--extra-vars "$(EXTRA_VARS)",)
-
-ansible-dev-preflight:
-	cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-teardown:
-	cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-install:
-	cd scripts/ansible && ansible-playbook playbooks/dev-install.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-reset:
-	cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-deploy:
-	cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml $(_ANSIBLE_EXTRA)
-
-ansible-dev-status:
-	cd scripts/ansible && ansible-playbook playbooks/dev-status.yml
-
-ansible-dev-logs:
-	cd scripts/ansible && ansible-playbook playbooks/dev-logs.yml
-
-# Snapshot / restore require an explicit -e snapshot_name=<name>. The
-# wrapper passes ANSIBLE_EXTRA through so devs can run
-# `make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'`.
-ansible-dev-snapshot:
-	cd scripts/ansible && ansible-playbook playbooks/dev-snapshot.yml $(ANSIBLE_EXTRA)
-
-ansible-dev-restore:
-	cd scripts/ansible && ansible-playbook playbooks/dev-restore.yml $(ANSIBLE_EXTRA)
-
-ansible-dev-version:
-	cd scripts/ansible && ansible-playbook playbooks/dev-version.yml
-
-ansible-dev-vpc:
-	cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
-
-# Multi-node cluster bootstrap against a tofu env (env1, env2, bryn, etc).
-# Inventory is generated from scripts/tofu-cluster/envs/<env>.tfvars.
-#   make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
-ENV ?=
-POOL ?=
-ansible-cluster-bootstrap:
-	@test -n "$(ENV)" || { echo "ENV=<env> required (e.g. env1, env2)"; exit 64; }
-	cd scripts/ansible && CLUSTER_ENV=$(ENV) ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \
-		-e cluster_env=$(ENV) \
-		$(if $(POOL),-e cluster_external_pool=$(POOL),) \
-		$(_ANSIBLE_EXTRA)
-
-.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck \
@@ -0,0 +1,50 @@
+#!/bin/sh
+# JetStream liveness watchdog for spinifex-nats.
+#
+# NATS can disable JetStream at runtime (e.g. a filesystem permission error while
+# flushing stream state) and keep the process alive serving KV as read-only. That
+# is not a process failure, so Restart=on-failure never fires and KV-backed
+# services (awsgw IAM bucket init, etc.) stall until NATS is restarted by hand.
+#
+# This probes the NATS monitoring endpoint's JS health and restarts the service
+# only when JetStream is sustainedly unhealthy while the service is running. The
+# multi-sample window avoids restarting during transient startup/quorum 503s.
+
+set -eu
+
+HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
+SERVICE="spinifex-nats.service"
+SAMPLES=3        # consecutive failing samples required before acting
+SAMPLE_GAP=2     # seconds between samples
+
+# Only act on a running service: a stopped/activating unit is systemd's to manage,
+# and try-restart on it would either no-op or fight an in-flight (re)start.
+if ! systemctl is-active --quiet "$SERVICE"; then
+    exit 0
+fi
+
+# curl is the probe transport; if it is unavailable, do nothing rather than
+# false-restart a healthy server.
+if ! command -v curl >/dev/null 2>&1; then
+    echo "nats-js-watchdog: curl not found, skipping JS health probe" >&2
+    exit 0
+fi
+
+# A single 200 means JetStream is healthy — exit fast on the common case.
+js_healthy() {
+    curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
+}
+
+i=1
+while [ "$i" -le "$SAMPLES" ]; do
+    if js_healthy; then
+        exit 0
+    fi
+    if [ "$i" -lt "$SAMPLES" ]; then
+        sleep "$SAMPLE_GAP"
+    fi
+    i=$((i + 1))
+done
+
+echo "nats-js-watchdog: JetStream unhealthy on $SAMPLES consecutive probes ($HEALTH_URL); restarting $SERVICE" >&2
+systemctl try-restart "$SERVICE"
@@ -6,6 +6,7 @@ Wants=spinifex-nats.service

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-gw
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
@@ -2,66 +2,29 @@
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
 After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
-# Wants=openvswitch-ipsec.service so systemd brings the IPsec userspace
-# daemon up before spinifex flips other_config:ipsec_encapsulation=true.
-# Without that daemon, OVS holds the cert pointers unused and strongSwan
-# never gets IKE config — tunnel traffic is encrypted with no SAs and
-# silently dropped. Wants= (not Requires=) keeps daemon bootable on
-# single-node / lab hosts where the package is intentionally absent.
+# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
 Wants=openvswitch-ipsec.service
-# No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
-# `systemctl restart spinifex-daemon`, which would mask the standalone-startup
-# path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers
-# cold boot.

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
-# DDIL Tier 1 (siv-17): no ExecStartPre wait-for-nats. The daemon starts in
-# standalone mode and reconnects to NATS in the background — process-exiting
-# on a missing NATS would kill the local VM management plane.
-#
-# Opt-in strict-startup (§1d-strict, siv-44): set SPINIFEX_REQUIRE_NATS=1 to
-# restore pre-DDIL fail-fast UX. First NATS connect uses a ~30s bounded retry
-# and the daemon aborts on timeout. Intended for dev/test/single-node deploys
-# where silent NATS misconfig is more confusing than helpful. Do NOT set this
-# in the shipped unit — it re-introduces the SPOF that Tier 1 removed.
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5
-# DDIL Tier 1: KillMode=process so systemd only signals the
-# daemon's main PID, not the cgroup. QEMU children stay alive across
-# `systemctl restart spinifex-daemon` and the new daemon reattaches to them
-# via the persisted local state file + pidfile + QMP socket. With the
-# default control-group mode, restart would terminate every guest VM —
-# defeating Scenario B's "daemon restart preserves running instances".
+# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)
 KillMode=process
-# 15s upper bound on the SIGTERM grace window. The new SIGTERM handler
-# skips VM teardown (it now just persists state and exits), so 15s is
-# plenty of slack for the cluster-manager + NATS unsubscribe path.
 TimeoutStopSec=15
+# RG-4: infra tier — reaped only after every guest
 OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
 Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/
-# DDIL Tier 1: QEMU pidfiles, QMP sockets, and the
-# shared qemu.log all resolve via utils.pidPath() — which falls back
-# to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With
-# PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so
-# pidfiles written by QEMU (running in the OLD daemon's namespace)
-# are invisible to the new daemon. The /local/instances handler then
-# reports PID=0 and Scenario B fails its "live QEMU PID" assertion.
-# Pin XDG_RUNTIME_DIR to the host-wide tmpfs at /run/spinifex (created
-# by tmpfiles.d, already in ReadWritePaths) so pidPath() returns a
-# stable, namespace-independent path that both daemon incarnations and
-# QEMU children resolve identically.
+# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)
 Environment=XDG_RUNTIME_DIR=/run/spinifex

-# Filesystem access — NoNewPrivileges omitted (daemon uses sudo for ip, ovs-vsctl, dhcpcd)
-# CAP_SYS_ADMIN: required for GPU passthrough sysfs operations (vfio group management).
-# CAP_DAC_OVERRIDE: sysfs driver bind/unbind paths (/sys/bus/pci/drivers/vfio-pci/bind,
-# unbind) are root:root 0200; non-root processes need CAP_DAC_OVERRIDE to open them.
+# RG-9: daemon tier — privileged by necessity (GPU vfio). NoNewPrivileges omitted (RG-10: sudo ip/ovs-vsctl/dhcpcd)
 AmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE
 ProtectSystem=full
 PrivateDevices=no
@@ -71,9 +34,11 @@ DeviceAllow=/dev/net/tun rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd
+# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
+# ProtectKernelTunables=yes would otherwise keep read-only.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf

-# Hardening
+# RG-9: hardening baseline
 ProtectHome=yes
 PrivateTmp=yes
 RestrictSUIDSGID=yes
@@ -88,8 +53,6 @@ ProtectControlGroups=yes
 ProtectProc=invisible
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
-# DHCP lease acquisition moved to spinifex-vpcd (see dhcp-allocate-error.md);
-# the daemon no longer needs AF_PACKET or the softened kernel protections.
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
 ProtectHostname=yes
 KeyringMode=private
@@ -97,8 +60,7 @@ RemoveIPC=yes
 LimitCORE=0
 LimitNOFILE=65536
 LimitNPROC=4096
-# VFIO GPU passthrough requires locking all guest RAM into physical memory.
-# Without this, QEMU fails with RLIMIT_MEMLOCK exceeded when pinning guest pages.
+# LimitMEMLOCK=infinity: VFIO pins all guest RAM
 LimitMEMLOCK=infinity

 [Install]
@@ -0,0 +1,8 @@
+[Unit]
+Description=Spinifex guest VM tier
+Before=slices.target
+
+[Slice]
+# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of
+# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice
+# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.
@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog
+PartOf=spinifex.target
+After=spinifex-nats.service
+
+[Service]
+Type=oneshot
+# Runs as root: the probe is a local curl to the NATS monitoring port, and the
+# remedy is `systemctl try-restart spinifex-nats` when JetStream is disabled while
+# the process stays alive (which Restart=on-failure cannot catch).
+ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
+
+# Hardening (kept compatible with talking to PID1 over /run/systemd).
+NoNewPrivileges=yes
+ProtectHome=yes
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+LockPersonality=yes
+RestrictSUIDSGID=yes
@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex NATS JetStream liveness watchdog timer
+PartOf=spinifex.target
+
+[Timer]
+# Delay the first probe so NATS startup + JetStream cluster quorum settle (a
+# forming cluster reports JS not-current transiently, which is not a fault).
+OnBootSec=60s
+OnUnitActiveSec=30s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target
@@ -5,6 +5,7 @@ Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.servic

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-nats
 Group=spinifex
 ExecStart=/usr/local/bin/spx service nats start
@@ -6,6 +6,7 @@ Wants=spinifex-nats.service

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-storage
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
@@ -0,0 +1,18 @@
+[Unit]
+Description=Spinifex graceful guest drain
+PartOf=spinifex.target
+# After= all services so it stops FIRST on target/host shutdown: the drain runs
+# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).
+After=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/true
+# ExecStop only runs when the target/host goes down, not on a single-service
+# restart — so a daemon restart still leaves guests running (DDIL).
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s
+TimeoutStopSec=180
+
+[Install]
+WantedBy=spinifex.target
@@ -0,0 +1,9 @@
+[Unit]
+Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
+Before=slices.target
+
+[Slice]
+# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the
+# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU
+# moves to the sibling spinifex-guests.slice (per-instance placement lands in
+# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.
@@ -5,6 +5,7 @@ After=spinifex-awsgw.service

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-ui
 Group=spinifex
 ExecStart=/usr/local/bin/spx service spinifex-ui start
@@ -6,6 +6,7 @@ Wants=spinifex-nats.service

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-viperblock
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
@@ -6,6 +6,7 @@ Wants=spinifex-nats.service ovn-controller.service

 [Service]
 Type=simple
+Slice=spinifex-system.slice
 User=spinifex-vpcd
 Group=spinifex
 ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
@@ -16,48 +17,34 @@ OOMScoreAdjust=-500
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/

-# vpcd MUST share the HOST mount namespace. IMDS binds one listener per VPC inside
-# a per-VPC netns entered in-process via setns(CLONE_NEWNET); the /run/netns/<vpc>
-# bind-mount that `ip netns add` creates must be host-visible (for diagnostics) and
-# must survive a vpcd restart (so the listener can rebind). ANY filesystem-sandbox
-# directive — ProtectSystem / ProtectHome / PrivateTmp / ProtectKernel* /
-# ProtectControlGroups / ProtectProc / Read{Only,Write}Paths — forks a PRIVATE
-# mount namespace that traps that bind-mount inside vpcd, leaving the host a stale
-# handle that fails setns(2) with EINVAL (CI runs 26920490454, 26922587471). Those
-# directives are therefore omitted by design, and so is the MountFlags=shared /
-# spinifex-netns.service propagation workaround they required.
-#
-# Boundary that REMAINS: DAC (User=spinifex-vpcd) plus the seccomp / capability /
-# namespace / address-family restrictions below. The durable fix that restores
-# filesystem sandboxing AND drops CAP_SYS_ADMIN is the process-per-netns model
-# (`ip netns exec` helper instead of in-process setns) — see
-# docs/development/improvements/imds-l2-datapath.md Phase 5.
-PrivateDevices=no
+# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no
+# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem
+# sandbox is restored. NoNewPrivileges stays off (RG-10: sudo ip/ovs-vsctl/dhcpcd).
+AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
+NoNewPrivileges=no
+ProtectSystem=full
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex

-# Hardening that does NOT fork a mount namespace (kept)
-RestrictSUIDSGID=no
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
 LockPersonality=yes
 RestrictRealtime=yes
+RestrictNamespaces=yes
 ProtectClock=yes
-# IMDS enters the per-VPC netns via setns(CLONE_NEWNET); the `ip -n <netns>` /
-# `ip netns exec` plumbing also unshares a mount namespace (CLONE_NEWNS, to give
-# the netns its own /sys), so `mnt` must be allowlisted alongside `net` or the
-# first in-netns command fails with EPERM.
-RestrictNamespaces=net mnt
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
 SystemCallArchitectures=native
-MemoryDenyWriteExecute=no
-# AF_PACKET: required by the in-process DHCP client (github.com/insomniacslk/dhcp
-# / nclient4) for raw L2 BOOTP on br-wan before an IP is configured on the
-# interface. See docs/development/bugs/dhcp-allocate-error.md.
+MemoryDenyWriteExecute=yes
+# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
-# CAP_NET_RAW: open AF_PACKET sockets.
-# CAP_NET_BIND_SERVICE: covers unicast RENEW to UDP :68.
-# CAP_NET_ADMIN: already used for ovs-vsctl / ip link wiring on the bridge.
-# CAP_SYS_ADMIN: setns(CLONE_NEWNET) into per-VPC IMDS netns to bind the listener.
-# Note, disabled for now, sudo issue with ovs
-AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
-NoNewPrivileges=no
 ProtectHostname=yes
 KeyringMode=private
 RemoveIPC=yes
@@ -0,0 +1,13 @@
+[Unit]
+Description=Spinifex resource-bounded slice (host ceiling)
+Before=slices.target
+
+[Slice]
+# RG-1: no CPUWeight — slice roots are siblings; capping below the default would
+# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on
+# system.slice (system.slice.d/spinifex-reserve.conf).
+# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).
+MemoryHigh=80%
+MemoryMax=90%
+# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.
+MemorySwapMax=8G
@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service

 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,205 @@
+// ecr-credential-provider is a kubelet exec credential provider that lets a
+// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
+//
+// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
+// reading a CredentialProviderResponse on STDOUT. The provider fetches the
+// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
+// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
+// for the requested registry host.
+//
+// Static node config (gateway URL, CA, region) is read from the cloud-init env
+// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+const (
+	credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
+	defaultCacheDuration   = "10h"
+	// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
+	// before the JWT actually expires mid-run.
+	expiryMargin = 30 * time.Minute
+
+	defaultEnvFile   = "/etc/spinifex-eks/agent.env"
+	defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
+	defaultIMDSBase  = "http://169.254.169.254/latest"
+)
+
+// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
+type CredentialProviderRequest struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Image      string `json:"image"`
+}
+
+// AuthConfig is a single registry credential entry.
+type AuthConfig struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
+type CredentialProviderResponse struct {
+	APIVersion    string                `json:"apiVersion"`
+	Kind          string                `json:"kind"`
+	CacheKeyType  string                `json:"cacheKeyType"`
+	CacheDuration string                `json:"cacheDuration"`
+	Auth          map[string]AuthConfig `json:"auth"`
+}
+
+// config holds the static node settings the provider needs.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	IMDSBase   string
+}
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
+	if err != nil {
+		slog.Error("ecr-credential-provider: build gateway client", "err", err)
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+
+	if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
+		slog.Error("ecr-credential-provider: run failed", "err", err)
+		// Still emit a valid empty-auth response so kubelet has a parseable reply.
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+}
+
+// run is the testable core: it reads a CredentialProviderRequest from in,
+// resolves credentials via IMDS + the gateway, and writes a
+// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
+func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
+	var req CredentialProviderRequest
+	if err := json.NewDecoder(in).Decode(&req); err != nil {
+		return fmt.Errorf("decode request: %w", err)
+	}
+	if req.Image == "" {
+		return fmt.Errorf("request image is empty")
+	}
+	imageHost := ecrauth.HostFromImage(req.Image)
+
+	creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
+	if err != nil {
+		return fmt.Errorf("fetch IMDS credentials: %w", err)
+	}
+
+	tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
+		creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
+	if err != nil {
+		return fmt.Errorf("get authorization token: %w", err)
+	}
+
+	auth := AuthConfig{Username: tok.Username, Password: tok.Password}
+	authMap := map[string]AuthConfig{imageHost: auth}
+	if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
+		authMap[tok.ProxyHost] = auth
+	}
+
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: cacheDurationFrom(tok.ExpiresAt),
+		Auth:          authMap,
+	}
+	return json.NewEncoder(out).Encode(&resp)
+}
+
+// cacheDurationFrom derives a Go-duration string from the token expiry minus a
+// safety margin, falling back to the default when expiry is unknown or too near.
+func cacheDurationFrom(expiresAt time.Time) string {
+	if expiresAt.IsZero() {
+		return defaultCacheDuration
+	}
+	d := time.Until(expiresAt) - expiryMargin
+	if d <= 0 {
+		return defaultCacheDuration
+	}
+	return d.Round(time.Second).String()
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override. The
+// IMDS base is fixed (not in the env file) but overridable for tests.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL: get("EKS_GATEWAY_URL"),
+		GatewayCA:  get("EKS_GATEWAY_CA"),
+		Region:     get("EKS_REGION"),
+		IMDSBase:   get("EKS_IMDS_BASE"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
+
+// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
+func emitEmpty(out io.Writer) {
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: defaultCacheDuration,
+		Auth:          map[string]AuthConfig{},
+	}
+	if err := json.NewEncoder(out).Encode(&resp); err != nil {
+		slog.Error("ecr-credential-provider: encode empty response", "err", err)
+	}
+}
@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
+// ECR GetAuthorizationToken on one TLS server, routed by request path.
+func fakeBackend(t *testing.T) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+
+	// IMDSv2 token.
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("imds-v2-token"))
+	})
+	// IMDS role name.
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		// Role credentials.
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"Type":            "AWS-HMAC",
+			"AccessKeyId":     "AKIATEST",
+			"SecretAccessKey": "secrettest",
+			"Token":           "sessiontoken",
+			"AccountId":       "123456789012",
+		})
+	})
+	// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": authToken,
+				"proxyEndpoint":      "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
+			}},
+		})
+	})
+
+	srv := httptest.NewTLSServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func clientTrusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestRun_ReturnsBasicAuth(t *testing.T) {
+	srv := fakeBackend(t)
+	client := clientTrusting(srv)
+
+	cfg := config{
+		GatewayURL: srv.URL,
+		Region:     "us-east-1",
+		IMDSBase:   srv.URL + "/latest",
+	}
+
+	const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
+	reqBody, err := json.Marshal(CredentialProviderRequest{
+		APIVersion: credProviderAPIVersion,
+		Kind:       "CredentialProviderRequest",
+		Image:      image,
+	})
+	if err != nil {
+		t.Fatalf("marshal request: %v", err)
+	}
+
+	var out bytes.Buffer
+	if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	var resp CredentialProviderResponse
+	if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
+		t.Fatalf("unmarshal response: %v\n%s", err, out.String())
+	}
+
+	if resp.Kind != "CredentialProviderResponse" {
+		t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
+	}
+	if resp.CacheKeyType != "Registry" {
+		t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
+	}
+
+	host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
+	auth, ok := resp.Auth[host]
+	if !ok {
+		t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
+	}
+	if auth.Username != "AWS" {
+		t.Errorf("username = %q, want AWS", auth.Username)
+	}
+	if auth.Password != "testjwt" {
+		t.Errorf("password = %q, want testjwt", auth.Password)
+	}
+}
@@ -0,0 +1,188 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	goruntime "runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// version is the agent build version, reported in the register message.
+// Overridable via -ldflags "-X main.version=...".
+var version = "dev"
+
+// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
+// container runtime, and an ECR resolver. It registers the host as a container
+// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
+// task assignments, and runs them through containerd, reporting state back over
+// the gateway. It never connects to NATS — the bus stays host-internal.
+type Agent struct {
+	cfg      config
+	id       identity
+	cp       controlPlane
+	puller   ctrruntime.ImagePuller
+	runner   ctrruntime.Runner
+	resolver ctrruntime.Resolver
+
+	reg *registrar
+	hb  *heartbeater
+
+	// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
+	netns *taskNetns
+
+	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
+	cred *credEndpoint
+
+	closers []func() error
+}
+
+// newAgent assembles an Agent from already-built seams. Tests use this directly
+// with fakes; New builds the production seams and delegates here. runner may be
+// nil when containerd is unavailable; the assign path then reports the task
+// STOPPED rather than crashing.
+func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImagePuller, runner ctrruntime.Runner, resolver ctrruntime.Resolver) *Agent {
+	return &Agent{
+		cfg:      cfg,
+		id:       id,
+		cp:       cp,
+		puller:   puller,
+		runner:   runner,
+		resolver: resolver,
+		reg:      newRegistrar(cp, id),
+		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
+		netns:    newTaskNetns(execNetRunner{}),
+	}
+}
+
+// New builds an Agent from config: it resolves the host identity from IMDS,
+// builds the gateway control-plane client, the ECR resolver and (best-effort)
+// the containerd runtime. A containerd connect failure is logged, not fatal —
+// registration and heartbeat still run so the instance is visible while the
+// runtime recovers. The ECR gateway client is built lazily on first image pull
+// (not here), so a missing or malformed gateway CA does not stop registration.
+func New(cfg config) (*Agent, error) {
+	imdsClient := &http.Client{Timeout: 5 * time.Second}
+
+	meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
+	if err != nil {
+		return nil, fmt.Errorf("instance metadata: %w", err)
+	}
+	host, _ := os.Hostname()
+	id := identity{
+		AccountID:    meta.AccountID,
+		ClusterName:  cfg.ClusterName,
+		InstanceID:   meta.InstanceID,
+		AZ:           meta.AZ,
+		Hostname:     host,
+		Capacity:     detectCapacity(),
+		AgentVersion: version,
+	}
+
+	creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
+	resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
+
+	var puller ctrruntime.ImagePuller
+	var runner ctrruntime.Runner
+	if rt, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
+		slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
+	} else {
+		puller = rt
+		runner = rt
+	}
+
+	cp, err := newGatewayControlPlane(cfg, creds)
+	if err != nil {
+		return nil, fmt.Errorf("build gateway control-plane: %w", err)
+	}
+
+	a := newAgent(cfg, id, cp, puller, runner, resolver)
+	a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
+		cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
+	if puller != nil {
+		a.closers = append(a.closers, puller.Close)
+	}
+	return a, nil
+}
+
+// Run registers the instance, starts the heartbeat and assignment-poll loops,
+// and blocks until ctx is cancelled, then tears down.
+func (a *Agent) Run(ctx context.Context) error {
+	if err := a.reg.Register(); err != nil {
+		slog.Error("ecs-agent: initial registration failed", "err", err)
+	} else {
+		slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
+	}
+
+	if a.cred != nil {
+		if err := a.cred.Start(); err != nil {
+			slog.Error("ecs-agent: credential endpoint start failed", "err", err)
+		}
+	}
+
+	go a.hb.Run(ctx)
+
+	// Re-adopt containers still running from before this restart, then poll with
+	// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
+	go a.pollAssignments(ctx, a.reconcile(ctx))
+
+	<-ctx.Done()
+	return a.Stop()
+}
+
+// Stop closes the runtime. Safe to call once.
+func (a *Agent) Stop() error {
+	var firstErr error
+	if a.cred != nil {
+		_ = a.cred.Stop()
+	}
+	for _, c := range a.closers {
+		if err := c(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
+func detectCapacity() bus.InstanceCapacity {
+	return bus.InstanceCapacity{
+		CPU:       goruntime.NumCPU() * 1024,
+		MemoryMiB: memTotalMiB(),
+	}
+}
+
+// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
+func memTotalMiB() int {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "MemTotal:") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			return 0
+		}
+		kb, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return 0
+		}
+		return kb / 1024
+	}
+	return 0
+}
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+)
+
+func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
+	puller := &ctrruntime.FakePuller{}
+	a := newAgent(cfg, testIdentity(), cp, puller, puller, nil)
+	a.closers = append(a.closers, puller.Close)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	// Give it time to register + re-register (heartbeat) a few times.
+	time.Sleep(30 * time.Millisecond)
+	cancel()
+
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("Run returned error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+
+	// One boot register + at least one heartbeat re-register.
+	if cp.registerCount() < 2 {
+		t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
+	}
+	if !puller.Closed {
+		t.Error("Stop did not close the puller")
+	}
+}
+
+func TestDetectCapacity_Positive(t *testing.T) {
+	c := detectCapacity()
+	if c.CPU <= 0 {
+		t.Errorf("CPU = %d, want > 0", c.CPU)
+	}
+	// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
+	if c.MemoryMiB < 0 {
+		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
+	}
+}
@@ -0,0 +1,119 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	defaultEnvFile          = "/etc/spinifex-ecs/agent.env"
+	defaultGatewayCA        = "/etc/spinifex-ecs/gateway-ca.pem"
+	defaultIMDSBase         = "http://169.254.169.254/latest"
+	defaultContainerdSocket = "/run/containerd/containerd.sock"
+	defaultHeartbeat        = 30 * time.Second
+	defaultPollInterval     = 5 * time.Second
+)
+
+// config holds the static settings the agent reads at boot. Cluster identity
+// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
+// only the cluster *name* the instance was launched into is static. AccessKey /
+// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
+// gateway calls with instance-role credentials from IMDS; the agent never holds
+// a NATS token.
+type config struct {
+	GatewayURL       string
+	GatewayCA        string
+	Region           string
+	ClusterName      string
+	AccessKey        string
+	SecretKey        string
+	IMDSBase         string
+	ContainerdSocket string
+	Heartbeat        time.Duration
+	PollInterval     time.Duration
+	CredEndpointIP   string
+	CredEndpointPort int
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL:       get("ECS_GATEWAY_URL"),
+		GatewayCA:        get("ECS_GATEWAY_CA"),
+		Region:           get("ECS_REGION"),
+		ClusterName:      get("ECS_CLUSTER"),
+		AccessKey:        get("ECS_ACCESS_KEY"),
+		SecretKey:        get("ECS_SECRET_KEY"),
+		IMDSBase:         get("ECS_IMDS_BASE"),
+		ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	if cfg.ContainerdSocket == "" {
+		cfg.ContainerdSocket = defaultContainerdSocket
+	}
+	if cfg.ClusterName == "" {
+		cfg.ClusterName = "default"
+	}
+	cfg.Heartbeat = defaultHeartbeat
+	if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.Heartbeat = d
+		}
+	}
+	cfg.PollInterval = defaultPollInterval
+	if v := get("ECS_POLL_INTERVAL"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.PollInterval = d
+		}
+	}
+	cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
+	if cfg.CredEndpointIP == "" {
+		cfg.CredEndpointIP = defaultCredEndpointIP
+	}
+	cfg.CredEndpointPort = defaultCredEndpointPort
+	if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
+		if p, err := strconv.Atoi(v); err == nil && p > 0 {
+			cfg.CredEndpointPort = p
+		}
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
@@ -0,0 +1,88 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestLoadConfig_Defaults(t *testing.T) {
+	// Ensure env vars don't leak in from the host.
+	for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
+		"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
+		"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
+		t.Setenv(k, "")
+	}
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	if cfg.GatewayCA != defaultGatewayCA {
+		t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
+	}
+	if cfg.IMDSBase != defaultIMDSBase {
+		t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
+	}
+	if cfg.ContainerdSocket != defaultContainerdSocket {
+		t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
+	}
+	if cfg.ClusterName != "default" {
+		t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
+	}
+	if cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
+	}
+	if cfg.PollInterval != defaultPollInterval {
+		t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
+	}
+}
+
+func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
+	dir := t.TempDir()
+	envFile := filepath.Join(dir, "agent.env")
+	body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
+	if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
+	t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
+
+	cfg := loadConfig(envFile)
+	if cfg.GatewayURL != "https://gw.env" {
+		t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
+	}
+	if cfg.Region != "us-west-2" {
+		t.Errorf("Region = %q, want from file", cfg.Region)
+	}
+	if cfg.ClusterName != "prod" {
+		t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
+	}
+	if cfg.AccessKey != "AKIAENV" {
+		t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
+	}
+}
+
+func TestLoadConfig_HeartbeatOverride(t *testing.T) {
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
+		t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
+	}
+	// Garbage value falls back to the default, not zero.
+	t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
+	if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
+		t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
+	}
+}
+
+func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "f.env")
+	if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	m := parseEnvFile(p)
+	if m["A"] != "1" || m["B"] != "2" {
+		t.Errorf("parse mismatch: %#v", m)
+	}
+	if _, ok := m["nokeyval"]; ok {
+		t.Errorf("line without = should be skipped")
+	}
+}
@@ -0,0 +1,129 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ecs"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecsgw"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
+// NATS publisher: every method is a SigV4-signed gateway request, so the NATS bus
+// stays host-internal. Tests inject a fake; production uses gatewayControlPlane.
+type controlPlane interface {
+	// Register (re-)registers the container instance. Called at boot and on every
+	// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
+	// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
+	Register(id identity) error
+	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
+	SubmitTaskState(st bus.TaskState) error
+	// PollAssignments drains this instance's assignment inbox, acking the taskIDs
+	// accepted on the previous poll. Returns the still-pending assignments.
+	PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error)
+}
+
+// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
+type gatewayControlPlane struct {
+	client *ecsgw.Client
+}
+
+var _ controlPlane = (*gatewayControlPlane)(nil)
+
+// pollTimeout bounds a single PollAssignments request; larger than the register
+// timeout to leave room for a future server-side long-poll.
+const pollTimeout = 30 * time.Second
+
+// newGatewayControlPlane builds the SigV4 client signing with the instance-role
+// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
+func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
+	client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return &gatewayControlPlane{client: client}, nil
+}
+
+// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
+// per-call credentials hook.
+func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
+	return func(ctx context.Context) (string, string, string, error) {
+		c, err := p.Retrieve(ctx)
+		if err != nil {
+			return "", "", "", err
+		}
+		return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
+	}
+}
+
+func (g *gatewayControlPlane) Register(id identity) error {
+	in := &ecs.RegisterContainerInstanceInput{
+		Cluster:                  aws.String(id.ClusterName),
+		InstanceIdentityDocument: aws.String(id.InstanceID),
+		TotalResources: []*ecs.Resource{
+			{Name: aws.String("CPU"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.CPU))},
+			{Name: aws.String("MEMORY"), Type: aws.String("INTEGER"), IntegerValue: aws.Int64(int64(id.Capacity.MemoryMiB))},
+		},
+		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal register: %w", err)
+	}
+	if _, err := g.client.Call("RegisterContainerInstance", body); err != nil {
+		return fmt.Errorf("register: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
+	in := &ecs.SubmitTaskStateChangeInput{
+		Cluster: aws.String(st.ClusterName),
+		Task:    aws.String(st.TaskID),
+		Status:  aws.String(st.LastStatus),
+		Reason:  aws.String(st.Reason),
+	}
+	for _, c := range st.Containers {
+		csc := &ecs.ContainerStateChange{
+			ContainerName: aws.String(c.Name),
+			Status:        aws.String(c.Status),
+			RuntimeId:     aws.String(c.ContainerID),
+		}
+		if c.ExitCode != nil {
+			csc.ExitCode = aws.Int64(int64(*c.ExitCode))
+		}
+		in.Containers = append(in.Containers, csc)
+	}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal task-state: %w", err)
+	}
+	if _, err := g.client.Call("SubmitTaskStateChange", body); err != nil {
+		return fmt.Errorf("submit task-state: %w", err)
+	}
+	return nil
+}
+
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error) {
+	in := &handlers_ecs.PollAssignmentsInput{Cluster: cluster, ContainerInstance: instance, AckTaskIDs: ack}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return nil, fmt.Errorf("marshal poll: %w", err)
+	}
+	resp, err := g.client.Call("PollAssignments", body)
+	if err != nil {
+		return nil, fmt.Errorf("poll: %w", err)
+	}
+	var out handlers_ecs.PollAssignmentsOutput
+	if err := json.Unmarshal(resp, &out); err != nil {
+		return nil, fmt.Errorf("decode poll: %w", err)
+	}
+	return out.Assignments, nil
+}
@@ -0,0 +1,361 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+const (
+	// defaultCredEndpointIP is the AWS-reserved link-local address tasks reach for
+	// container credentials; defaultCredEndpointPort is the standard HTTP port.
+	defaultCredEndpointIP   = "169.254.170.2"
+	defaultCredEndpointPort = 80
+	// defaultCredProxyPort is the high port the listener actually binds; a nat
+	// DNAT rule rewrites 169.254.170.2:80 to it so nothing holds a socket on :80
+	// and host/bridge tasks sharing the netns can bind :80 themselves.
+	defaultCredProxyPort = 51679
+	// credRefreshMargin refreshes a cached credential set this long before expiry
+	// so a container never starts a request with a token about to lapse.
+	credRefreshMargin = 5 * time.Minute
+	// credDummyIface is the dummy interface the endpoint IP is added to in the host
+	// netns so the agent can bind it.
+	credDummyIface = "ecs-cred"
+)
+
+// credResponse is the ECS task-credentials JSON the container's AWS SDK expects
+// at AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.
+type credResponse struct {
+	RoleArn         string `json:"RoleArn"`
+	AccessKeyID     string `json:"AccessKeyId"`
+	SecretAccessKey string `json:"SecretAccessKey"`
+	Token           string `json:"Token"`
+	Expiration      string `json:"Expiration"`
+}
+
+// credEntry caches one task's assumed-role credentials alongside the role it was
+// assumed from.
+type credEntry struct {
+	roleARN string
+	creds   stsauth.Credentials
+}
+
+// credEndpoint is the host-netns HTTP server that hands a task its IAM role
+// credentials. It maps credID -> taskRoleARN (registered as tasks start) and
+// mints/caches credentials on demand by assuming the role over the gateway with
+// the agent's instance credentials.
+type credEndpoint struct {
+	provider   credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+	ip         string
+	port       int
+	proxyPort  int
+	run        netCmdRunner
+
+	// assume mints fresh credentials for a role; defaults to AssumeRole over the
+	// gateway and is overridable in tests.
+	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
+
+	mu         sync.Mutex
+	roles      map[string]string    // credID -> taskRoleARN
+	cache      map[string]credEntry // credID -> cached creds
+	httpClient *http.Client
+
+	srv *http.Server
+	ln  net.Listener
+}
+
+// newCredEndpoint builds the endpoint. An empty ip/port falls back to the
+// AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
+// adds the link-local address to the dummy interface (real or fake).
+func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
+	if ip == "" {
+		ip = defaultCredEndpointIP
+	}
+	if port == 0 {
+		port = defaultCredEndpointPort
+	}
+	c := &credEndpoint{
+		provider:   provider,
+		region:     region,
+		gatewayURL: gatewayURL,
+		caPath:     caPath,
+		ip:         ip,
+		port:       port,
+		proxyPort:  defaultCredProxyPort,
+		run:        run,
+		roles:      map[string]string{},
+		cache:      map[string]credEntry{},
+	}
+	c.assume = c.assumeOverGateway
+	return c
+}
+
+// Register associates a credID with the task role it serves; Deregister removes
+// it and drops any cached credentials. Both are no-ops for an empty credID.
+func (c *credEndpoint) Register(credID, roleARN string) {
+	if credID == "" || roleARN == "" {
+		return
+	}
+	c.mu.Lock()
+	c.roles[credID] = roleARN
+	c.mu.Unlock()
+}
+
+func (c *credEndpoint) Deregister(credID string) {
+	if credID == "" {
+		return
+	}
+	c.mu.Lock()
+	delete(c.roles, credID)
+	delete(c.cache, credID)
+	c.mu.Unlock()
+}
+
+// relativeURI is the value injected into a container's
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI for the given credID.
+func credRelativeURI(credID string) string { return "/v2/credentials/" + credID }
+
+// ServeHTTP serves GET /v2/credentials/{credID}. Unknown paths/credIDs return
+// 404; non-GET returns 405; an AssumeRole failure returns 502.
+func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	credID, ok := strings.CutPrefix(r.URL.Path, "/v2/credentials/")
+	if !ok || credID == "" || strings.Contains(credID, "/") {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	creds, roleARN, err := c.fetch(r.Context(), credID)
+	if err != nil {
+		if err == errUnknownCredID {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		http.Error(w, "credentials unavailable", http.StatusBadGateway)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(credResponse{
+		RoleArn:         roleARN,
+		AccessKeyID:     creds.AccessKeyID,
+		SecretAccessKey: creds.SecretAccessKey,
+		Token:           creds.SessionToken,
+		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
+	}); err != nil {
+		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+	}
+}
+
+// errUnknownCredID signals a credID with no registered task role.
+var errUnknownCredID = fmt.Errorf("unknown credId")
+
+// fetch returns valid credentials for credID, reusing the cache until within the
+// refresh margin and otherwise assuming the task role over the gateway.
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+	c.mu.Lock()
+	roleARN, known := c.roles[credID]
+	if !known {
+		c.mu.Unlock()
+		return stsauth.Credentials{}, "", errUnknownCredID
+	}
+	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+		c.mu.Unlock()
+		return e.creds, roleARN, nil
+	}
+	c.mu.Unlock()
+
+	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	if err != nil {
+		return stsauth.Credentials{}, "", err
+	}
+
+	c.mu.Lock()
+	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.mu.Unlock()
+	return creds, roleARN, nil
+}
+
+// assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
+// against the gateway with the agent's instance credentials.
+func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+	instCreds, err := c.provider.Retrieve(ctx)
+	if err != nil {
+		return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
+	}
+	client, err := c.client()
+	if err != nil {
+		return stsauth.Credentials{}, err
+	}
+	return stsauth.AssumeRole(c.region, c.gatewayURL, client,
+		instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
+		roleARN, sessionName)
+}
+
+// client builds the pinned-CA gateway HTTP client on first use and caches it.
+func (c *credEndpoint) client() (*http.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.httpClient != nil {
+		return c.httpClient, nil
+	}
+	hc, err := ecrauth.GatewayHTTPClient(c.caPath)
+	if err != nil {
+		return nil, err
+	}
+	c.httpClient = hc
+	return hc, nil
+}
+
+// Start adds the endpoint IP to a dummy interface in the host netns, serves HTTP
+// on the high proxy port, and DNATs the advertised :80 to it. A bind, address,
+// or nat failure is returned so the caller can log it without aborting
+// register/heartbeat.
+func (c *credEndpoint) Start() error {
+	if err := c.addEndpointAddr(); err != nil {
+		return err
+	}
+	ln, err := net.Listen("tcp", net.JoinHostPort(c.ip, strconv.Itoa(c.listenPort())))
+	if err != nil {
+		return fmt.Errorf("listen %s:%d: %w", c.ip, c.listenPort(), err)
+	}
+	if err := c.addRedirect(); err != nil {
+		_ = ln.Close()
+		return err
+	}
+	c.ln = ln
+	c.srv = &http.Server{Handler: c, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		if err := c.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
+			slog.Error("ecs-agent: credential endpoint serve", "err", err)
+		}
+	}()
+	slog.Info("ecs-agent: credential endpoint listening", "addr", ln.Addr().String())
+	return nil
+}
+
+// Stop shuts the server down and removes the dummy interface. Safe on a nil/
+// never-started endpoint.
+func (c *credEndpoint) Stop() error {
+	if c == nil {
+		return nil
+	}
+	if c.srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = c.srv.Shutdown(ctx)
+	}
+	c.delRedirect()
+	c.delEndpointAddr()
+	return nil
+}
+
+// listenPort is the port the socket binds. Loopback test mode binds the
+// caller-supplied (ephemeral) port directly; the real endpoint binds the high
+// proxy port and reaches :80 via the DNAT rules.
+func (c *credEndpoint) listenPort() int {
+	if c.ip == "127.0.0.1" {
+		return c.port
+	}
+	return c.proxyPort
+}
+
+// addRedirect installs nat DNAT rules rewriting the advertised :80 to the proxy
+// port, for both locally generated traffic (host/bridge task containers, OUTPUT)
+// and traffic arriving over the awsvpc credential veth (PREROUTING). The dst IP
+// is left on the endpoint IP so no route_localnet handling is needed. Skipped in
+// loopback test mode, which binds the port directly.
+func (c *credEndpoint) addRedirect() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		if _, err := c.run.Run("iptables", c.natArgs("-C", chain)...); err == nil {
+			continue // rule already present
+		}
+		if _, err := c.run.Run("iptables", c.natArgs("-A", chain)...); err != nil {
+			return fmt.Errorf("add nat redirect %s: %w", chain, err)
+		}
+	}
+	return nil
+}
+
+func (c *credEndpoint) delRedirect() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	for _, chain := range []string{"OUTPUT", "PREROUTING"} {
+		_, _ = c.run.Run("iptables", c.natArgs("-D", chain)...)
+	}
+}
+
+// natArgs builds the iptables nat rule for op (-A/-C/-D) on chain: DNAT the
+// advertised endpoint :80 to the proxy port on the same IP.
+func (c *credEndpoint) natArgs(op, chain string) []string {
+	return []string{
+		"-t", "nat", op, chain,
+		"-d", c.ip, "-p", "tcp", "--dport", strconv.Itoa(c.port),
+		"-j", "DNAT", "--to-destination", net.JoinHostPort(c.ip, strconv.Itoa(c.proxyPort)),
+	}
+}
+
+// addEndpointAddr brings up the dummy interface holding the endpoint IP. Skipped
+// for loopback (tests bind 127.0.0.1 directly, no interface plumbing needed).
+func (c *credEndpoint) addEndpointAddr() error {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return nil
+	}
+	_, _ = c.run.Run("ip", "link", "add", credDummyIface, "type", "dummy")
+	if _, err := c.run.Run("ip", "addr", "add", c.ip+"/32", "dev", credDummyIface); err != nil &&
+		!strings.Contains(err.Error(), "exists") {
+		return fmt.Errorf("add endpoint addr %s: %w", c.ip, err)
+	}
+	if _, err := c.run.Run("ip", "link", "set", credDummyIface, "up"); err != nil {
+		return fmt.Errorf("bring up %s: %w", credDummyIface, err)
+	}
+	return nil
+}
+
+func (c *credEndpoint) delEndpointAddr() {
+	if c.run == nil || c.ip == "127.0.0.1" {
+		return
+	}
+	_, _ = c.run.Run("ip", "link", "del", credDummyIface)
+}
+
+// credValid reports whether creds are usable now with margin to spare.
+func credValid(c stsauth.Credentials, margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// STS [\w+=,.@-]{2,64} constraint.
+func sessionName(credID string) string {
+	name := "ecs-" + credID
+	if len(name) > 64 {
+		name = name[:64]
+	}
+	return name
+}
@@ -0,0 +1,197 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/stsauth"
+)
+
+// newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
+// counting calls so cache behaviour is observable. run is nil (no iface plumbing).
+func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+	var calls int32
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		atomic.AddInt32(&calls, 1)
+		return creds, nil
+	}
+	return c, &calls
+}
+
+func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
+	exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
+	})
+	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String())
+	}
+	var got credResponse
+	if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || got.SecretAccessKey != "secret" || got.Token != "tok" {
+		t.Errorf("bad creds in response: %+v", got)
+	}
+	if got.RoleArn != "arn:aws:iam::111122223333:role/task" {
+		t.Errorf("RoleArn = %q", got.RoleArn)
+	}
+	if got.Expiration != exp.Format(time.RFC3339) {
+		t.Errorf("Expiration = %q, want %q", got.Expiration, exp.Format(time.RFC3339))
+	}
+}
+
+func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404", rr.Code)
+	}
+	if *calls != 0 {
+		t.Errorf("assume called %d times for unknown credID, want 0", *calls)
+	}
+}
+
+func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusMethodNotAllowed {
+		t.Fatalf("status = %d, want 405", rr.Code)
+	}
+}
+
+func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	for hit := range 3 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+		if rr.Code != http.StatusOK {
+			t.Fatalf("hit %d: status %d", hit, rr.Code)
+		}
+	}
+	if *calls != 1 {
+		t.Errorf("assume called %d times, want 1 (cached)", *calls)
+	}
+}
+
+func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
+	// Expiry inside the refresh margin -> every hit re-assumes.
+	c, calls := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
+	})
+	c.Register("cred-1", "arn:role")
+	for range 2 {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	}
+	if *calls != 2 {
+		t.Errorf("assume called %d times, want 2 (expiry within margin)", *calls)
+	}
+}
+
+func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
+	c.Register("cred-1", "arn:role")
+	c.Deregister("cred-1")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want 404 after deregister", rr.Code)
+	}
+}
+
+func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
+		return stsauth.Credentials{}, context.DeadlineExceeded
+	}
+	c.Register("cred-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
+	if rr.Code != http.StatusBadGateway {
+		t.Fatalf("status = %d, want 502", rr.Code)
+	}
+}
+
+func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
+	})
+	c.Register("cred-1", "arn:role")
+	c.port = 0 // bind an ephemeral port instead of privileged 80
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	url := "http://" + c.ln.Addr().String() + credRelativeURI("cred-1")
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+}
+
+func TestCredEndpoint_RedirectInstallsAndRemovesDNAT(t *testing.T) {
+	// failOn the -C existence check so addRedirect falls through to -A.
+	f := &fakeNetRunner{failOn: "-t nat -C"}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "169.254.170.2", 80, f)
+	if c.listenPort() != defaultCredProxyPort {
+		t.Fatalf("listenPort = %d, want %d (proxy)", c.listenPort(), defaultCredProxyPort)
+	}
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	target := "-d 169.254.170.2 -p tcp --dport 80 -j DNAT --to-destination 169.254.170.2:51679"
+	for _, chain := range []string{"-A OUTPUT", "-A PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing add rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+	c.delRedirect()
+	for _, chain := range []string{"-D OUTPUT", "-D PREROUTING"} {
+		if !f.sawAny("iptables -t nat " + chain + " " + target) {
+			t.Errorf("missing delete rule for %s; calls=%v", chain, f.joined())
+		}
+	}
+}
+
+func TestCredEndpoint_RedirectSkippedOnLoopback(t *testing.T) {
+	f := &fakeNetRunner{}
+	c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, f)
+	if err := c.addRedirect(); err != nil {
+		t.Fatalf("addRedirect: %v", err)
+	}
+	c.delRedirect()
+	if f.sawAny("iptables") {
+		t.Errorf("loopback mode must not touch iptables; calls=%v", f.joined())
+	}
+	if c.listenPort() != c.port {
+		t.Errorf("loopback listenPort = %d, want caller port %d", c.listenPort(), c.port)
+	}
+}
+
+func TestCredRelativeURI(t *testing.T) {
+	if got := credRelativeURI("abc"); got != "/v2/credentials/abc" {
+		t.Errorf("credRelativeURI = %q", got)
+	}
+}
@@ -0,0 +1,59 @@
+package credentials
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/imdscreds"
+)
+
+// refreshMargin is how long before expiry the cache is considered stale.
+const refreshMargin = 5 * time.Minute
+
+// IMDSProvider fetches instance-role credentials from IMDS and caches them until
+// they near expiry. Safe for concurrent use.
+type IMDSProvider struct {
+	client *http.Client
+	base   string
+
+	mu     sync.Mutex
+	cached Credentials
+}
+
+var _ CredentialsProvider = (*IMDSProvider)(nil)
+
+// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
+// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
+func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	return &IMDSProvider{client: client, base: base}
+}
+
+// Retrieve returns cached credentials when still valid, else fetches fresh ones.
+func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.cached.Valid(refreshMargin) {
+		return p.cached, nil
+	}
+	if err := ctx.Err(); err != nil {
+		return Credentials{}, err
+	}
+
+	raw, err := imdscreds.Fetch(p.client, p.base)
+	if err != nil {
+		return Credentials{}, err
+	}
+	p.cached = Credentials{
+		AccessKeyID:     raw.AccessKeyID,
+		SecretAccessKey: raw.SecretAccessKey,
+		SessionToken:    raw.SessionToken,
+		Expiration:      raw.Expiration,
+	}
+	return p.cached, nil
+}
@@ -0,0 +1,104 @@
+package credentials
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits != nil {
+			atomic.AddInt32(hits, 1)
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestRetrieve_FetchesAndCaches(t *testing.T) {
+	var hits int32
+	exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		"Expiration":      exp.Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	got, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("Retrieve: %v", err)
+	}
+	if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
+		t.Errorf("creds mismatch: %+v", got)
+	}
+
+	// Second call within validity window must not re-hit IMDS.
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatalf("Retrieve 2: %v", err)
+	}
+	if hits != 1 {
+		t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
+	}
+}
+
+func TestRetrieve_RefetchesWhenStale(t *testing.T) {
+	var hits int32
+	// Expiry inside the refresh margin → never cached, always refetched.
+	srv := imdsStub(t, &hits, map[string]string{
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Expiration":      time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
+	})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Retrieve(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if hits != 2 {
+		t.Errorf("hits = %d, want 2 (stale refetch)", hits)
+	}
+}
+
+func TestRetrieve_CancelledContext(t *testing.T) {
+	srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
+	p := NewIMDSProvider(srv.Client(), srv.URL)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := p.Retrieve(ctx); err == nil {
+		t.Fatal("expected error from cancelled context")
+	}
+}
+
+func TestCredentials_Valid(t *testing.T) {
+	if (Credentials{}).Valid(0) {
+		t.Error("empty creds should be invalid")
+	}
+	noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
+	if !noExp.Valid(time.Hour) {
+		t.Error("zero expiry should be treated valid")
+	}
+	soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
+	if soon.Valid(5 * time.Minute) {
+		t.Error("expiry inside margin should be invalid")
+	}
+}
@@ -0,0 +1,36 @@
+// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
+// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
+// with the host's instance-role credentials, fetched from IMDS and cached until
+// near expiry.
+package credentials
+
+import (
+	"context"
+	"time"
+)
+
+// Credentials is a SigV4 credential set with an expiry.
+type Credentials struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	SessionToken    string
+	Expiration      time.Time
+}
+
+// Valid reports whether the credentials are usable now, leaving a margin so a
+// caller does not start a request with a token about to expire mid-flight.
+func (c Credentials) Valid(margin time.Duration) bool {
+	if c.AccessKeyID == "" || c.SecretAccessKey == "" {
+		return false
+	}
+	if c.Expiration.IsZero() {
+		return true
+	}
+	return time.Now().Add(margin).Before(c.Expiration)
+}
+
+// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
+// must be safe for concurrent use.
+type CredentialsProvider interface {
+	Retrieve(ctx context.Context) (Credentials, error)
+}
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func TestTaskCredID(t *testing.T) {
+	cases := []struct {
+		name string
+		as   bus.Assign
+		want string
+	}{
+		{"no role", bus.Assign{TaskID: "t1"}, ""},
+		{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
+	}
+	for _, tc := range cases {
+		if got := taskCredID(&tc.as); got != tc.want {
+			t.Errorf("%s: taskCredID = %q, want %q", tc.name, got, tc.want)
+		}
+	}
+}
+
+func TestWithCredEnv(t *testing.T) {
+	base := map[string]string{"FOO": "bar"}
+
+	// No credID -> env returned unchanged (same map).
+	if got := withCredEnv(base, ""); got["FOO"] != "bar" || got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "" {
+		t.Errorf("blank credID mutated env: %v", got)
+	}
+
+	got := withCredEnv(base, "cred-1")
+	if got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "/v2/credentials/cred-1" {
+		t.Errorf("missing relative-uri: %v", got)
+	}
+	if got["FOO"] != "bar" {
+		t.Errorf("dropped existing env: %v", got)
+	}
+	// Original map must be untouched (copy semantics).
+	if _, ok := base["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]; ok {
+		t.Errorf("source env mutated: %v", base)
+	}
+}
+
+func TestSessionName(t *testing.T) {
+	if got := sessionName("abc"); got != "ecs-abc" {
+		t.Errorf("sessionName = %q", got)
+	}
+	long := sessionName(string(make([]byte, 100)))
+	if len(long) != 64 {
+		t.Errorf("sessionName not truncated to 64: len=%d", len(long))
+	}
+}
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/internal/ecrauth"
+)
+
+// ecrResolver implements runtime.Resolver by minting an ECR authorization token
+// per pull: it fetches the host's IMDS credentials, SigV4-signs
+// GetAuthorizationToken against the gateway, and returns the AWS:<jwt> as a
+// user/password pair for containerd.
+//
+// The gateway HTTP client is built lazily on first Authorize, not at agent boot:
+// a missing or malformed gateway CA only matters when an image is actually
+// pulled, so the agent can still register + heartbeat on a host with absent or
+// wrong ECR config.
+type ecrResolver struct {
+	creds      credentials.CredentialsProvider
+	region     string
+	gatewayURL string
+	caPath     string
+
+	mu         sync.Mutex
+	httpClient *http.Client
+}
+
+var _ runtime.Resolver = (*ecrResolver)(nil)
+
+// newECRResolver builds a resolver with an injected HTTP client (tests). The
+// client is used as-is and never rebuilt from caPath.
+func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
+}
+
+// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
+// from caPath on first use (production path).
+func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
+	return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
+}
+
+// client returns the gateway HTTP client, building it from caPath on first call
+// and caching the result. An injected client (newECRResolver) is returned as-is.
+func (e *ecrResolver) client() (*http.Client, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.httpClient != nil {
+		return e.httpClient, nil
+	}
+	c, err := ecrauth.GatewayHTTPClient(e.caPath)
+	if err != nil {
+		return nil, err
+	}
+	e.httpClient = c
+	return c, nil
+}
+
+// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
+// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
+// public registries (docker.io and friends) pull anonymously, so refs without
+// an ECR host must not drag in IMDS host credentials.
+func isECRRef(ref string) bool {
+	host, _, _ := strings.Cut(ref, "/")
+	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
+		return false // no registry host -> docker.io default
+	}
+	return strings.Contains(host, ".dkr.ecr.")
+}
+
+// Authorize returns the ECR token's user/password and proxy endpoint for ref.
+// Non-ECR refs resolve to anonymous pull (empty credentials).
+func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
+	if !isECRRef(ref) {
+		return "", "", "", nil
+	}
+	c, err := e.creds.Retrieve(ctx)
+	if err != nil {
+		return "", "", "", err
+	}
+	httpClient, err := e.client()
+	if err != nil {
+		return "", "", "", err
+	}
+	tok, err := ecrauth.GetAuthorizationToken(e.region, e.gatewayURL, httpClient,
+		c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
+	if err != nil {
+		return "", "", "", err
+	}
+	return tok.Username, tok.Password, tok.ProxyHost, nil
+}
@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+)
+
+// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
+type stubCreds struct {
+	c   credentials.Credentials
+	err error
+}
+
+func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
+	return s.c, s.err
+}
+
+func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if status != http.StatusOK {
+			w.WriteHeader(status)
+			return
+		}
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
+				"proxyEndpoint":      proxy,
+			}},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func trusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestECRResolver_Authorize(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	user, pass, endpoint, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+	if endpoint != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
+		t.Errorf("endpoint = %q", endpoint)
+	}
+}
+
+func TestECRResolver_CredsError(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("no imds")}, "us-east-1", "https://x", http.DefaultClient)
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected creds error to propagate")
+	}
+}
+
+func TestECRResolver_GatewayError(t *testing.T) {
+	srv := gatewayStub(t, "", "", http.StatusInternalServerError)
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected gateway error")
+	}
+}
+
+func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
+	creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
+	// Construction with a bogus CA path must not fail — only Authorize does.
+	r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
+	if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
+		t.Fatal("expected lazy gateway-client build to fail on missing CA")
+	}
+}
+
+// TestECRResolver_NonECRAnonymous asserts that a public-registry ref pulls
+// anonymously: Authorize returns empty creds without ever touching IMDS, even
+// when the credentials provider would error.
+func TestECRResolver_NonECRAnonymous(t *testing.T) {
+	r := newECRResolver(stubCreds{err: errors.New("imds must not be called")}, "us-east-1", "https://x", http.DefaultClient)
+	for _, ref := range []string{"nginx:alpine", "docker.io/library/nginx:alpine", "ghcr.io/org/app:1"} {
+		user, pass, endpoint, err := r.Authorize(context.Background(), ref)
+		if err != nil || user != "" || pass != "" || endpoint != "" {
+			t.Errorf("ref %q: got (%q,%q,%q,%v), want anonymous", ref, user, pass, endpoint, err)
+		}
+	}
+}
+
+func TestIsECRRef(t *testing.T) {
+	cases := map[string]bool{
+		"nginx:alpine":            false,
+		"docker.io/library/nginx": false,
+		"ghcr.io/org/app:1":       false,
+		"registry:5000/app:1":     false,
+		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
+		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	}
+	for ref, want := range cases {
+		if got := isECRRef(ref); got != want {
+			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+		}
+	}
+}
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"time"
+)
+
+// heartbeater keeps the container instance alive by periodically re-registering
+// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
+// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
+// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
+type heartbeater struct {
+	cp       controlPlane
+	id       identity
+	interval time.Duration
+}
+
+func newHeartbeater(cp controlPlane, id identity, interval time.Duration) *heartbeater {
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	return &heartbeater{cp: cp, id: id, interval: interval}
+}
+
+// beat re-registers the instance once, refreshing its LastSeen.
+func (h *heartbeater) beat() error {
+	return h.cp.Register(h.id)
+}
+
+// Run beats every interval until ctx is cancelled. A failed beat is logged and
+// retried on the next tick rather than killing the loop.
+func (h *heartbeater) Run(ctx context.Context) {
+	ticker := time.NewTicker(h.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := h.beat(); err != nil {
+				slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
+			}
+		}
+	}
+}
@@ -0,0 +1,39 @@
+// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
+// the ECS-AMI, with containerd baked in). It registers the host with the ECS
+// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
+// re-registering, polls the gateway for task assignments, runs them through
+// containerd, and reports state back over the gateway.
+//
+// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
+// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
+// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
+// comes from IMDS at boot.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	agent, err := New(cfg)
+	if err != nil {
+		slog.Error("ecs-agent: startup failed", "err", err)
+		os.Exit(1)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if err := agent.Run(ctx); err != nil {
+		slog.Error("ecs-agent: run failed", "err", err)
+		os.Exit(1)
+	}
+}
@@ -0,0 +1,97 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
+type instanceMetadata struct {
+	AccountID  string
+	InstanceID string
+	AZ         string
+}
+
+// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
+// base is the IMDS root, e.g. "http://169.254.169.254/latest".
+func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
+	token, err := imdsToken(client, base)
+	if err != nil {
+		return instanceMetadata{}, err
+	}
+
+	instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
+	}
+	az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
+	}
+
+	doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
+	if err != nil {
+		return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
+	}
+	var ident struct {
+		AccountID string `json:"accountId"`
+	}
+	if err := json.Unmarshal([]byte(doc), &ident); err != nil {
+		return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
+	}
+
+	return instanceMetadata{
+		AccountID:  ident.AccountID,
+		InstanceID: strings.TrimSpace(instanceID),
+		AZ:         strings.TrimSpace(az),
+	}, nil
+}
+
+// imdsToken fetches an IMDSv2 session token.
+func imdsToken(client *http.Client, base string) (string, error) {
+	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("imds token: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
+
+// imdsGet performs a token-authenticated IMDS GET.
+func imdsGet(client *http.Client, url, token string) (string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("status %d", resp.StatusCode)
+	}
+	return string(body), nil
+}
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	guard := func(w http.ResponseWriter, r *http.Request) bool {
+		if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("i-0abc123"))
+		}
+	})
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte("us-east-1a"))
+		}
+	})
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
+		if guard(w, r) {
+			_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
+		}
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestFetchInstanceMetadata_OK(t *testing.T) {
+	srv := metadataStub(t, true)
+	got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
+	if err != nil {
+		t.Fatalf("fetch: %v", err)
+	}
+	if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
+		t.Errorf("metadata mismatch: %+v", got)
+	}
+}
+
+func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
+	// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
+	srv := metadataStub(t, true)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
+		t.Fatalf("v2 path should succeed: %v", err)
+	}
+}
+
+func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
+	mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
+	mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
+	mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
+		t.Fatal("expected parse error on bad identity document")
+	}
+}
@@ -0,0 +1,167 @@
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// netnsRunDir is the iproute2 netns mount directory; a name here is the netns
+// path containerd joins via the OCI network namespace path.
+const netnsRunDir = "/var/run/netns"
+
+// Credentials path constants: each task netns gets a /30 veth into the host netns
+// from 169.254.172.0/22 and a host-routed /32 to the credential endpoint, keeping
+// the endpoint off the task ENI's VPC route. credTaskVeth is the task-side name
+// (unique per netns); the host side is per-task (credVethName).
+const (
+	credTaskVeth     = "ecsc0"
+	credEndpointCIDR = defaultCredEndpointIP + "/32"
+)
+
+// netCmdRunner executes a host networking command. Abstracted so unit tests can
+// assert the ip/netns sequence without touching the kernel.
+type netCmdRunner interface {
+	Run(name string, args ...string) (string, error)
+}
+
+// execNetRunner runs real commands.
+type execNetRunner struct{}
+
+func (execNetRunner) Run(name string, args ...string) (string, error) {
+	out, err := exec.Command(name, args...).CombinedOutput()
+	return string(out), err
+}
+
+// taskNetns builds and tears down per-task awsvpc network namespaces. The ENI is
+// already hot-plugged into the VM (scheduler did that); the agent finds the NIC
+// by MAC, moves it into a task netns, and DHCPs the OVN-assigned ENI IP — the
+// same way the primary NIC obtains its address.
+type taskNetns struct {
+	run     netCmdRunner
+	nicWait time.Duration
+	poll    time.Duration
+}
+
+func newTaskNetns(run netCmdRunner) *taskNetns {
+	return &taskNetns{run: run, nicWait: 30 * time.Second, poll: time.Second}
+}
+
+func netnsName(taskID string) string { return "ecs-" + taskID }
+func netnsPathFor(taskID string) string {
+	return netnsRunDir + "/" + netnsName(taskID)
+}
+
+// Setup creates the task netns, moves the ENI NIC (matched by MAC) into it, and
+// brings it up with a DHCP lease. It returns the netns path for containerd. On
+// any failure it tears the partial netns down so a retry starts clean.
+func (n *taskNetns) Setup(taskID, mac string) (string, error) {
+	iface, err := n.waitForNIC(mac)
+	if err != nil {
+		return "", err
+	}
+	name := netnsName(taskID)
+	if _, err := n.run.Run("ip", "netns", "add", name); err != nil {
+		return "", fmt.Errorf("netns add %s: %w", name, err)
+	}
+	hostVeth := credVethName(taskID)
+	hostIP, taskIP := credSubnet(taskID)
+	steps := [][]string{
+		{"ip", "link", "set", iface, "netns", name},
+		{"ip", "-n", name, "link", "set", "lo", "up"},
+		{"ip", "-n", name, "link", "set", iface, "up"},
+		{"ip", "netns", "exec", name, "udhcpc", "-i", iface, "-q", "-n"},
+		// Credentials path: a veth into the host netns plus a host-routed /32 to the
+		// credential endpoint, so the container reaches 169.254.170.2 without it
+		// leaking onto the task ENI's VPC route.
+		{"ip", "link", "add", hostVeth, "type", "veth", "peer", "name", credTaskVeth, "netns", name},
+		{"ip", "addr", "add", hostIP + "/30", "dev", hostVeth},
+		{"ip", "link", "set", hostVeth, "up"},
+		{"ip", "-n", name, "addr", "add", taskIP + "/30", "dev", credTaskVeth},
+		{"ip", "-n", name, "link", "set", credTaskVeth, "up"},
+		{"ip", "-n", name, "route", "add", credEndpointCIDR, "via", hostIP},
+	}
+	for _, s := range steps {
+		if _, err := n.run.Run(s[0], s[1:]...); err != nil {
+			_ = n.Teardown(taskID)
+			return "", fmt.Errorf("netns setup %v: %w", s, err)
+		}
+	}
+	return netnsPathFor(taskID), nil
+}
+
+// credVethName is the host-side veth name for a task's credential path. Derived
+// from the taskID and kept within the 15-char interface-name limit.
+func credVethName(taskID string) string {
+	return fmt.Sprintf("ecsv%08x", crc32.ChecksumIEEE([]byte(taskID)))
+}
+
+// credSubnet derives a unique /30 in 169.254.172.0/22 for a task's credential
+// veth, returning the host-side and task-side addresses. The host side is the
+// container's gateway to the credential endpoint.
+func credSubnet(taskID string) (hostIP, taskIP string) {
+	slot := crc32.ChecksumIEEE([]byte(taskID)) % 256 // 256 /30s in the /22
+	third := 172 + (slot*4)/256
+	base := (slot * 4) % 256
+	hostIP = fmt.Sprintf("169.254.%d.%d", third, base+1)
+	taskIP = fmt.Sprintf("169.254.%d.%d", third, base+2)
+	return hostIP, taskIP
+}
+
+// Teardown deletes the task netns (releasing the moved NIC). Idempotent enough
+// for the stop path: a missing netns is reported but not fatal to the caller.
+func (n *taskNetns) Teardown(taskID string) error {
+	name := netnsName(taskID)
+	// Deleting the netns destroys the task-side veth and its host peer; remove the
+	// host side explicitly too so a leaked half from a partial setup is reaped.
+	_, _ = n.run.Run("ip", "link", "del", credVethName(taskID))
+	if _, err := n.run.Run("ip", "netns", "del", name); err != nil {
+		return fmt.Errorf("netns del %s: %w", name, err)
+	}
+	return nil
+}
+
+// waitForNIC polls until a NIC with the given MAC appears in the host netns,
+// returning its interface name. The hot-plug pipeline is async, so the device
+// may lag the assign by a few seconds.
+func (n *taskNetns) waitForNIC(mac string) (string, error) {
+	want := strings.ToLower(strings.TrimSpace(mac))
+	deadline := time.Now().Add(n.nicWait)
+	for {
+		iface, err := n.findNICByMAC(want)
+		if err == nil {
+			return iface, nil
+		}
+		if time.Now().After(deadline) {
+			return "", fmt.Errorf("nic with mac %s not found within %s", want, n.nicWait)
+		}
+		time.Sleep(n.poll)
+	}
+}
+
+// findNICByMAC parses `ip -o link` and returns the interface whose link/ether
+// matches mac.
+func (n *taskNetns) findNICByMAC(mac string) (string, error) {
+	out, err := n.run.Run("ip", "-o", "link")
+	if err != nil {
+		return "", fmt.Errorf("ip -o link: %w", err)
+	}
+	for line := range strings.SplitSeq(out, "\n") {
+		if !strings.Contains(strings.ToLower(line), mac) {
+			continue
+		}
+		// "<idx>: <ifname>: <flags> ... link/ether <mac> ..."
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		name := strings.TrimSuffix(strings.TrimSuffix(fields[1], ":"), "@")
+		if i := strings.IndexByte(name, '@'); i >= 0 {
+			name = name[:i]
+		}
+		return name, nil
+	}
+	return "", fmt.Errorf("no interface with mac %s", mac)
+}
@@ -0,0 +1,154 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// fakeNetRunner records every command and serves canned `ip -o link` output. A
+// failOn substring forces the matching command to error so teardown paths run.
+type fakeNetRunner struct {
+	mu      sync.Mutex
+	calls   [][]string
+	linkOut string
+	failOn  string
+}
+
+func (f *fakeNetRunner) Run(name string, args ...string) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	cmd := append([]string{name}, args...)
+	f.calls = append(f.calls, cmd)
+	joined := strings.Join(cmd, " ")
+	if f.failOn != "" && strings.Contains(joined, f.failOn) {
+		return "", errors.New("boom: " + joined)
+	}
+	if name == "ip" && len(args) >= 2 && args[0] == "-o" && args[1] == "link" {
+		return f.linkOut, nil
+	}
+	return "", nil
+}
+
+func (f *fakeNetRunner) joined() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]string, len(f.calls))
+	for i, c := range f.calls {
+		out[i] = strings.Join(c, " ")
+	}
+	return out
+}
+
+func (f *fakeNetRunner) sawAny(sub string) bool {
+	for _, j := range f.joined() {
+		if strings.Contains(j, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+const linkWithENI = `1: lo: <LOOPBACK,UP> mtu 65536 ... link/loopback 00:00:00:00:00:00 ...
+2: eth0: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:aa:bb:cc ...
+3: eth1: <BROADCAST,UP> mtu 1500 ... link/ether 52:54:00:de:ad:01 ...`
+
+func newTestNetns(f *fakeNetRunner) *taskNetns {
+	n := newTaskNetns(f)
+	n.nicWait = 50 * time.Millisecond
+	n.poll = 5 * time.Millisecond
+	return n
+}
+
+func TestNetns_FindNICByMAC(t *testing.T) {
+	n := newTestNetns(&fakeNetRunner{linkOut: linkWithENI})
+	iface, err := n.findNICByMAC("52:54:00:de:ad:01")
+	if err != nil || iface != "eth1" {
+		t.Fatalf("want eth1, got %q err=%v", iface, err)
+	}
+	if _, err := n.findNICByMAC("52:54:00:ff:ff:ff"); err == nil {
+		t.Fatal("want error for absent mac")
+	}
+}
+
+func TestNetns_SetupSequence(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	path, err := n.Setup("t-001", "52:54:00:DE:AD:01")
+	if err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	if path != netnsPathFor("t-001") {
+		t.Fatalf("want %q, got %q", netnsPathFor("t-001"), path)
+	}
+	hostIP, taskIP := credSubnet("t-001")
+	hostVeth := credVethName("t-001")
+	want := []string{
+		"ip netns add ecs-t-001",
+		"ip link set eth1 netns ecs-t-001",
+		"ip -n ecs-t-001 link set lo up",
+		"ip -n ecs-t-001 link set eth1 up",
+		"ip netns exec ecs-t-001 udhcpc -i eth1 -q -n",
+		"ip link add " + hostVeth + " type veth peer name ecsc0 netns ecs-t-001",
+		"ip addr add " + hostIP + "/30 dev " + hostVeth,
+		"ip -n ecs-t-001 addr add " + taskIP + "/30 dev ecsc0",
+		"ip -n ecs-t-001 route add 169.254.170.2/32 via " + hostIP,
+	}
+	for _, w := range want {
+		if !f.sawAny(w) {
+			t.Errorf("missing command %q in %v", w, f.joined())
+		}
+	}
+}
+
+func TestCredSubnet_UniquePer30(t *testing.T) {
+	hostA, taskA := credSubnet("task-a")
+	hostB, taskB := credSubnet("task-b")
+	if hostA == hostB {
+		t.Errorf("distinct tasks share a host IP: %s", hostA)
+	}
+	for _, ip := range []string{hostA, taskA, hostB, taskB} {
+		if !strings.HasPrefix(ip, "169.254.17") {
+			t.Errorf("cred IP %s outside 169.254.172.0/22", ip)
+		}
+	}
+	// host and task ends of one task sit in the same /30 (differ in last octet).
+	if hostA[:strings.LastIndexByte(hostA, '.')] != taskA[:strings.LastIndexByte(taskA, '.')] {
+		t.Errorf("host %s and task %s not in same /30", hostA, taskA)
+	}
+}
+
+func TestNetns_SetupNICNotFound(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:00:00:99"); err == nil {
+		t.Fatal("want timeout error for absent NIC")
+	}
+	if f.sawAny("netns add") {
+		t.Error("should not create netns when NIC never appears")
+	}
+}
+
+func TestNetns_SetupTearsDownOnStepFailure(t *testing.T) {
+	f := &fakeNetRunner{linkOut: linkWithENI, failOn: "udhcpc"}
+	n := newTestNetns(f)
+	if _, err := n.Setup("t-001", "52:54:00:de:ad:01"); err == nil {
+		t.Fatal("want error when udhcpc fails")
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected teardown after step failure, got %v", f.joined())
+	}
+}
+
+func TestNetns_Teardown(t *testing.T) {
+	f := &fakeNetRunner{}
+	n := newTestNetns(f)
+	if err := n.Teardown("t-001"); err != nil {
+		t.Fatalf("teardown: %v", err)
+	}
+	if !f.sawAny("ip netns del ecs-t-001") {
+		t.Errorf("expected netns del, got %v", f.joined())
+	}
+}
@@ -0,0 +1,72 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// reconcile re-adopts containers that survived an agent restart. It lists the
+// runtime's containers and, for each task still running in this cluster,
+// re-registers its IAM credentials, re-attaches the exit-wait, and refreshes its
+// RUNNING state on the gateway. The returned set seeds pollAssignments so the
+// gateway's re-delivered assignments for these tasks are acked but not re-run.
+//
+// It is best-effort: a nil runner or a List error returns an empty set and the
+// agent falls back to its pre-reconcile behaviour (poll + run).
+func (a *Agent) reconcile(ctx context.Context) map[string]bool {
+	adopted := map[string]bool{}
+	if a.runner == nil {
+		return adopted
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: reconcile list failed", "err", err)
+		return adopted
+	}
+
+	// Group this cluster's running containers back into their tasks. The Assign is
+	// reconstructed from labels — enough to re-register creds and tear down netns.
+	type task struct {
+		as         *bus.Assign
+		containers []ctrruntime.Container
+	}
+	tasks := map[string]*task{}
+	for _, c := range containers {
+		if !c.Running || c.Labels[labelClusterName] != a.id.ClusterName {
+			continue
+		}
+		taskID := c.Labels[labelTaskID]
+		if taskID == "" {
+			continue
+		}
+		t := tasks[taskID]
+		if t == nil {
+			t = &task{as: &bus.Assign{
+				TaskID:        taskID,
+				ClusterName:   a.id.ClusterName,
+				CredID:        c.Labels[labelCredID],
+				TaskRoleARN:   c.Labels[labelTaskRoleARN],
+				ENIMacAddress: c.Labels[labelENIMac],
+			}}
+			tasks[taskID] = t
+		}
+		t.containers = append(t.containers, c)
+	}
+
+	for taskID, t := range tasks {
+		a.registerTaskCreds(t.as)
+		statuses := make([]bus.ContainerStatus, 0, len(t.containers))
+		for _, c := range t.containers {
+			name := c.Labels[labelContainerName]
+			go a.waitContainer(ctx, t.as, name, c.ID)
+			statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
+		}
+		a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
+		adopted[taskID] = true
+		slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.containers))
+	}
+	return adopted
+}
@@ -0,0 +1,181 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+const testTaskRole = "arn:aws:iam::111122223333:role/task"
+
+// adoptedContainer builds a runtime.Container as the reconciler would discover it,
+// stamping only the mulga.ecs.* labels that are non-empty.
+func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
+	labels := map[string]string{
+		labelTaskID:        taskID,
+		labelContainerName: name,
+		labelClusterName:   cluster,
+	}
+	if credID != "" {
+		labels[labelCredID] = credID
+	}
+	if role != "" {
+		labels[labelTaskRoleARN] = role
+	}
+	if mac != "" {
+		labels[labelENIMac] = mac
+	}
+	return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
+}
+
+func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
+	t.Helper()
+	deadline := time.After(time.Second)
+	for !slices.Contains(rt.Waits(), id) {
+		select {
+		case <-deadline:
+			t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// A running, correctly-labeled container is re-adopted: its credentials are
+// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
+func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+	a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if !adopted["t-001"] {
+		t.Fatalf("t-001 not adopted; got %v", adopted)
+	}
+	if got := a.cred.roles["t-001"]; got != testTaskRole {
+		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
+	}
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
+	}
+	waitedFor(t, rt, containerID("t-001", "web"))
+}
+
+// pollAssignments seeded with an adopted task acks its re-delivered assignment
+// but does not re-run it, while a genuinely new task still dispatches.
+func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
+	mk := func(id string) bus.Assign {
+		return bus.Assign{
+			AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
+			Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
+		}
+	}
+	adopted, fresh := mk("t-001"), mk("t-002")
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{adopted, fresh},
+		{adopted, fresh},
+		nil,
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{"t-001": true})
+
+	deadline := time.After(time.Second)
+	for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
+		select {
+		case <-deadline:
+			t.Fatal("fresh task t-002 was never dispatched")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
+		t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
+	}
+	if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
+		t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
+	}
+	var t1Acked bool
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == "t-001" {
+				t1Acked = true
+			}
+		}
+	}
+	if !t1Acked {
+		t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
+	}
+}
+
+// Containers for another cluster or with a stopped task are not adopted.
+func TestReconcile_FiltersOtherClusterAndStopped(t *testing.T) {
+	rt := &ctrruntime.FakePuller{
+		WaitErr: errors.New("blocked"),
+		Containers: []ctrruntime.Container{
+			adoptedContainer("t-other", "web", "other", "", "", "", true),
+			adoptedContainer("t-stop", "web", "default", "", "", "", false),
+			adoptedContainer("t-ok", "web", "default", "", "", "", true),
+		},
+	}
+	a := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+
+	adopted := a.reconcile(context.Background())
+
+	if len(adopted) != 1 || !adopted["t-ok"] {
+		t.Fatalf("adopted = %v, want only t-ok", adopted)
+	}
+}
+
+// A nil runner or a List error degrades to an empty set without panicking.
+func TestReconcile_DegradesGracefully(t *testing.T) {
+	noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil, nil)
+	if got := noRunner.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("nil runner: adopted = %v, want empty", got)
+	}
+
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
+	listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, rt, nil)
+	if got := listErr.reconcile(context.Background()); len(got) != 0 {
+		t.Errorf("list error: adopted = %v, want empty", got)
+	}
+}
+
+// taskLabels emits the optional cred/role/MAC labels only when set.
+func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
+	full := taskLabels(&bus.Assign{
+		TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
+	}, "web")
+	for k, want := range map[string]string{
+		labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
+	} {
+		if full[k] != want {
+			t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
+		}
+	}
+
+	bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
+	if len(bare) != 3 {
+		t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
+	}
+	for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
+		if _, ok := bare[k]; ok {
+			t.Errorf("bare labels unexpectedly carry %q", k)
+		}
+	}
+}
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// identity is the container instance's stable identity, assembled at boot from
+// IMDS (account, instance, AZ) and config (cluster). It is the join key the
+// scheduler uses to track this instance.
+type identity struct {
+	AccountID    string
+	ClusterName  string
+	InstanceID   string
+	AZ           string
+	Hostname     string
+	Capacity     bus.InstanceCapacity
+	AgentVersion string
+}
+
+// registrar registers the host as a container instance through the gateway.
+type registrar struct {
+	cp controlPlane
+	id identity
+}
+
+func newRegistrar(cp controlPlane, id identity) *registrar {
+	return &registrar{cp: cp, id: id}
+}
+
+// Register calls the gateway's RegisterContainerInstance. The scheduler records
+// (or refreshes) the container instance on receipt.
+func (r *registrar) Register() error {
+	if err := r.cp.Register(r.id); err != nil {
+		return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
+	}
+	return nil
+}
@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// fakeCP is a test double for the gateway control-plane. It records registers and
+// task-state reports, and replays a scripted queue of poll responses.
+type fakeCP struct {
+	mu sync.Mutex
+
+	registers   int
+	states      []bus.TaskState
+	pollAcks    [][]string
+	pollReplies [][]bus.Assign
+	pollCalls   int
+
+	registerErr bool
+	submitErr   bool
+}
+
+var _ controlPlane = (*fakeCP)(nil)
+
+func (f *fakeCP) Register(_ identity) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.registerErr {
+		return errors.New("register boom")
+	}
+	f.registers++
+	return nil
+}
+
+func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.submitErr {
+		return errors.New("submit boom")
+	}
+	f.states = append(f.states, st)
+	return nil
+}
+
+func (f *fakeCP) PollAssignments(_, _ string, ack []string) ([]bus.Assign, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	cp := append([]string(nil), ack...)
+	f.pollAcks = append(f.pollAcks, cp)
+	var out []bus.Assign
+	if f.pollCalls < len(f.pollReplies) {
+		out = f.pollReplies[f.pollCalls]
+	}
+	f.pollCalls++
+	return out, nil
+}
+
+func (f *fakeCP) registerCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.registers
+}
+
+func (f *fakeCP) taskStates() []bus.TaskState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]bus.TaskState(nil), f.states...)
+}
+
+func (f *fakeCP) acks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollAcks...)
+}
+
+func testIdentity() identity {
+	return identity{
+		AccountID:    "123456789012",
+		ClusterName:  "default",
+		InstanceID:   "i-abc123",
+		AZ:           "us-east-1a",
+		Hostname:     "ecs-host",
+		Capacity:     bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
+		AgentVersion: "test",
+	}
+}
+
+func TestRegistrar_Register(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("registers = %d, want 1", cp.registerCount())
+	}
+}
+
+func TestRegistrar_RegisterError(t *testing.T) {
+	cp := &fakeCP{registerErr: true}
+	if err := newRegistrar(cp, testIdentity()).Register(); err == nil {
+		t.Fatal("expected register error")
+	}
+}
+
+func TestHeartbeater_Beat(t *testing.T) {
+	cp := &fakeCP{}
+	if err := newHeartbeater(cp, testIdentity(), time.Second).beat(); err != nil {
+		t.Fatalf("beat: %v", err)
+	}
+	if cp.registerCount() != 1 {
+		t.Errorf("beat should re-register once, got %d", cp.registerCount())
+	}
+}
+
+func TestHeartbeater_DefaultInterval(t *testing.T) {
+	h := newHeartbeater(&fakeCP{}, testIdentity(), 0)
+	if h.interval != defaultHeartbeat {
+		t.Errorf("interval = %v, want default", h.interval)
+	}
+}
+
+func TestHeartbeater_RunStopsOnContext(t *testing.T) {
+	cp := &fakeCP{}
+	h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	done := make(chan struct{})
+	go func() { h.Run(ctx); close(done) }()
+
+	time.Sleep(25 * time.Millisecond)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("Run did not stop on context cancel")
+	}
+	if cp.registerCount() == 0 {
+		t.Error("expected at least one re-register before cancel")
+	}
+}
@@ -0,0 +1,118 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/remotes"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/distribution/reference"
+)
+
+// ecsNamespace isolates ecs-agent images/containers in containerd, mirroring the
+// "moby"/"k8s.io" convention other runtimes use.
+const ecsNamespace = "ecs"
+
+// servingProbeTimeout bounds the boot-time daemon liveness check.
+const servingProbeTimeout = 3 * time.Second
+
+// containerdPuller drives a real containerd daemon over its unix socket.
+type containerdPuller struct {
+	client *containerd.Client
+}
+
+var _ ImagePuller = (*containerdPuller)(nil)
+
+// New connects to the containerd daemon at socket (e.g.
+// "/run/containerd/containerd.sock") and returns an ImagePuller. The containerd
+// client dials lazily, so New actively probes the daemon with IsServing and
+// returns an error if it is unreachable — otherwise a dead socket would only
+// surface on the first pull, and the agent's boot-time "containerd unavailable"
+// log would never fire.
+func New(socket string) (Runtime, error) {
+	client, err := containerd.New(socket, containerd.WithDefaultNamespace(ecsNamespace))
+	if err != nil {
+		return nil, fmt.Errorf("connect containerd %s: %w", socket, err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), servingProbeTimeout)
+	defer cancel()
+	serving, err := client.IsServing(ctx)
+	if err != nil || !serving {
+		_ = client.Close()
+		return nil, fmt.Errorf("containerd %s not serving: %w", socket, err)
+	}
+	return &containerdPuller{client: client}, nil
+}
+
+// Pull resolves credentials via r, then pulls and unpacks the image.
+func (p *containerdPuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Ref)
+	if err != nil {
+		return Image{}, err
+	}
+
+	resolver, err := newDockerResolver(ctx, ref, r)
+	if err != nil {
+		return Image{}, err
+	}
+
+	img, err := p.client.Pull(ctx, ref,
+		containerd.WithPullUnpack,
+		containerd.WithResolver(resolver),
+	)
+	if err != nil {
+		return Image{}, fmt.Errorf("pull %s: %w", ref, err)
+	}
+
+	target := img.Target()
+	return Image{
+		Ref:    ref,
+		Digest: target.Digest.String(),
+		Size:   target.Size,
+	}, nil
+}
+
+// normalizeRef expands a task-definition image string into the fully-qualified
+// reference containerd's resolver requires: a bare "nginx:alpine" becomes
+// "docker.io/library/nginx:alpine" and an untagged ref gets ":latest". Without
+// this, containerd parses the bare ref as "dummy://nginx:alpine" and rejects it.
+func normalizeRef(ref string) (string, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", fmt.Errorf("parse image ref %q: %w", ref, err)
+	}
+	return reference.TagNameOnly(named).String(), nil
+}
+
+// Close releases the containerd client connection.
+func (p *containerdPuller) Close() error {
+	if p.client == nil {
+		return nil
+	}
+	return p.client.Close()
+}
+
+// newDockerResolver builds a containerd remote resolver whose per-host
+// credentials come from r (the ECR token resolver). Credentials are fetched once
+// for ref and reused for every host containerd asks about during the pull.
+func newDockerResolver(ctx context.Context, ref string, r Resolver) (remotes.Resolver, error) {
+	creds := func(string) (string, string, error) { return "", "", nil }
+	if r != nil {
+		user, pass, _, err := r.Authorize(ctx, ref)
+		if err != nil {
+			return nil, fmt.Errorf("authorize %s: %w", ref, err)
+		}
+		creds = func(string) (string, string, error) { return user, pass, nil }
+	}
+
+	authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
+	return docker.NewResolver(docker.ResolverOptions{
+		Hosts: docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer)),
+	}), nil
+}
@@ -0,0 +1,142 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cio"
+	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+var _ Runner = (*containerdPuller)(nil)
+
+// Run creates a container from an already-pulled image and starts its task. The
+// container joins the host network namespace (bridge/CNI task networking is a
+// later sprint) and carries spec.Labels for the reboot reconciler. id must be
+// unique within the "ecs" namespace.
+func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (string, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+
+	ref, err := normalizeRef(spec.Image)
+	if err != nil {
+		return "", err
+	}
+	image, err := p.client.GetImage(ctx, ref)
+	if err != nil {
+		return "", fmt.Errorf("get image %s: %w", ref, err)
+	}
+
+	specOpts := []oci.SpecOpts{
+		oci.WithImageConfig(image),
+		oci.WithHostHostsFile,
+		oci.WithHostResolvconf,
+	}
+	if spec.NetnsPath != "" {
+		// awsvpc: join the task ENI netns built by the agent.
+		specOpts = append(specOpts, oci.WithLinuxNamespace(specs.LinuxNamespace{
+			Type: specs.NetworkNamespace,
+			Path: spec.NetnsPath,
+		}))
+	} else {
+		// bridge/host: share the VM (host) netns.
+		specOpts = append(specOpts, oci.WithHostNamespace(specs.NetworkNamespace))
+	}
+	if len(spec.Command) > 0 {
+		specOpts = append(specOpts, oci.WithProcessArgs(spec.Command...))
+	}
+	if len(spec.Env) > 0 {
+		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
+	}
+
+	container, err := p.client.NewContainer(ctx, id,
+		containerd.WithNewSnapshot(id+"-snapshot", image),
+		containerd.WithNewSpec(specOpts...),
+		containerd.WithContainerLabels(spec.Labels),
+	)
+	if err != nil {
+		return "", fmt.Errorf("create container %s: %w", id, err)
+	}
+
+	task, err := container.NewTask(ctx, cio.NullIO)
+	if err != nil {
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("create task %s: %w", id, err)
+	}
+	if err := task.Start(ctx); err != nil {
+		_, _ = task.Delete(ctx)
+		_ = container.Delete(ctx, containerd.WithSnapshotCleanup)
+		return "", fmt.Errorf("start task %s: %w", id, err)
+	}
+	return id, nil
+}
+
+// Wait blocks until the container's task exits and returns its exit code.
+func (p *containerdPuller) Wait(ctx context.Context, containerID string) (RunStatus, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("load container %s: %w", containerID, err)
+	}
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("attach task %s: %w", containerID, err)
+	}
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		return RunStatus{}, fmt.Errorf("wait task %s: %w", containerID, err)
+	}
+	status := <-statusC
+	code, _, err := status.Result()
+	return RunStatus{ExitCode: int(code)}, err
+}
+
+// List enumerates every container in the ecs namespace with its labels and
+// whether its task is running, so the agent can re-adopt live containers after a
+// restart. Per-container label/task errors are treated as "not running" rather
+// than failing the whole list.
+func (p *containerdPuller) List(ctx context.Context) ([]Container, error) {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	containers, err := p.client.Containers(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list containers: %w", err)
+	}
+	out := make([]Container, 0, len(containers))
+	for _, c := range containers {
+		labels, _ := c.Labels(ctx)
+		running := false
+		if task, terr := c.Task(ctx, nil); terr == nil {
+			if status, serr := task.Status(ctx); serr == nil {
+				running = status.Status == containerd.Running
+			}
+		}
+		out = append(out, Container{ID: c.ID(), Labels: labels, Running: running})
+	}
+	return out, nil
+}
+
+// Remove kills and deletes the container's task, then the container + snapshot.
+func (p *containerdPuller) Remove(ctx context.Context, containerID string) error {
+	ctx = namespaces.WithNamespace(ctx, ecsNamespace)
+	container, err := p.client.LoadContainer(ctx, containerID)
+	if err != nil {
+		return nil //nolint:nilerr // load failure means the container is already gone
+	}
+	if task, terr := container.Task(ctx, nil); terr == nil {
+		_, _ = task.Delete(ctx, containerd.WithProcessKill)
+	}
+	return container.Delete(ctx, containerd.WithSnapshotCleanup)
+}
+
+// envSlice renders an env map as sorted "K=V" entries for deterministic specs.
+func envSlice(env map[string]string) []string {
+	out := make([]string, 0, len(env))
+	for k, v := range env {
+		out = append(out, k+"="+v)
+	}
+	sort.Strings(out)
+	return out
+}
@@ -0,0 +1,82 @@
+// Package runtime is the ecs-agent's container-runtime seam. The agent pulls
+// task images and (from Sprint 4e) runs containers through an ImagePuller; the
+// real implementation drives containerd, but the interface lets tests and the
+// scheduler-only build run against a fake.
+package runtime
+
+import "context"
+
+// PullSpec describes one image to pull.
+type PullSpec struct {
+	// Ref is the full image reference, e.g.
+	// "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:latest".
+	Ref string
+}
+
+// Image identifies a pulled image in the runtime's content store.
+type Image struct {
+	Ref    string
+	Digest string
+	Size   int64
+}
+
+// Resolver supplies registry credentials for a given image reference. The
+// ecs-agent backs this with ECR GetAuthorizationToken (see ecrresolver.go).
+type Resolver interface {
+	// Authorize returns the username, password and registry endpoint to use for
+	// ref. An empty user/pass means anonymous pull.
+	Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error)
+}
+
+// ImagePuller pulls images into the local runtime. Close releases the runtime
+// client connection.
+type ImagePuller interface {
+	Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error)
+	Close() error
+}
+
+// RunSpec describes a single container the agent must create and start. v1 runs
+// the container in the host network namespace (bridge/CNI task networking lands
+// in a later sprint); Labels carry the mulga.ecs.* task identity for the reboot
+// reconciler.
+type RunSpec struct {
+	Image   string
+	Command []string
+	Env     map[string]string
+	Labels  map[string]string
+	// NetnsPath, when set, joins the container to a pre-built network namespace
+	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
+	// bridge/host mode behaviour.
+	NetnsPath string
+}
+
+// RunStatus is a finished container's outcome.
+type RunStatus struct {
+	ExitCode int
+}
+
+// Container is a container the runtime already knows about, discovered by List.
+// Labels carry the mulga.ecs.* task identity so the agent can re-adopt running
+// containers after a restart; Running reports whether its task is live.
+type Container struct {
+	ID      string
+	Labels  map[string]string
+	Running bool
+}
+
+// Runner creates and starts containers from already-pulled images. id is a
+// caller-unique container ID; Wait blocks until the container exits; Remove
+// tears down the container + its task; List enumerates known containers so the
+// reboot reconciler can re-adopt the ones still running.
+type Runner interface {
+	Run(ctx context.Context, id string, spec RunSpec) (containerID string, err error)
+	Wait(ctx context.Context, containerID string) (RunStatus, error)
+	Remove(ctx context.Context, containerID string) error
+	List(ctx context.Context) ([]Container, error)
+}
+
+// Runtime is the full container runtime the ecs-agent drives: pull + run.
+type Runtime interface {
+	ImagePuller
+	Runner
+}
@@ -0,0 +1,125 @@
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+// FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
+// It records pull requests and replays a programmed result or error.
+type FakePuller struct {
+	mu     sync.Mutex
+	Pulls  []PullSpec
+	Authzd []string
+	Result Image
+	Err    error
+	Closed bool
+	OnPull func(PullSpec) (Image, error)
+
+	// Run bookkeeping.
+	Runs     []RunSpec
+	RunErr   error
+	WaitCode int
+	WaitErr  error
+	Waited   []string
+	Removed  []string
+	OnRun    func(string, RunSpec) (string, error)
+
+	// List bookkeeping: Containers is replayed by List; ListErr forces a failure;
+	// Listed records that List was called.
+	Containers []Container
+	ListErr    error
+	Listed     bool
+}
+
+var (
+	_ ImagePuller = (*FakePuller)(nil)
+	_ Runner      = (*FakePuller)(nil)
+)
+
+// Pull records the spec, exercises the resolver, and returns the programmed
+// result. If OnPull is set it takes precedence over Result/Err.
+func (f *FakePuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
+	f.mu.Lock()
+	f.Pulls = append(f.Pulls, spec)
+	f.mu.Unlock()
+
+	if r != nil {
+		user, _, _, err := r.Authorize(ctx, spec.Ref)
+		if err != nil {
+			return Image{}, fmt.Errorf("authorize %s: %w", spec.Ref, err)
+		}
+		f.mu.Lock()
+		f.Authzd = append(f.Authzd, user)
+		f.mu.Unlock()
+	}
+
+	if f.OnPull != nil {
+		return f.OnPull(spec)
+	}
+	if f.Err != nil {
+		return Image{}, f.Err
+	}
+	res := f.Result
+	if res.Ref == "" {
+		res.Ref = spec.Ref
+	}
+	return res, nil
+}
+
+// Close marks the puller closed.
+func (f *FakePuller) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Closed = true
+	return nil
+}
+
+// Run records the spec and returns id (or the programmed result/error).
+func (f *FakePuller) Run(_ context.Context, id string, spec RunSpec) (string, error) {
+	f.mu.Lock()
+	f.Runs = append(f.Runs, spec)
+	f.mu.Unlock()
+	if f.OnRun != nil {
+		return f.OnRun(id, spec)
+	}
+	if f.RunErr != nil {
+		return "", f.RunErr
+	}
+	return id, nil
+}
+
+// Wait records the container ID and returns the programmed exit code/error.
+func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, error) {
+	f.mu.Lock()
+	f.Waited = append(f.Waited, containerID)
+	f.mu.Unlock()
+	return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
+}
+
+// Remove records the container ID.
+func (f *FakePuller) Remove(_ context.Context, containerID string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Removed = append(f.Removed, containerID)
+	return nil
+}
+
+// Waits returns a copy of the container IDs passed to Wait, for assertions.
+func (f *FakePuller) Waits() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.Waited...)
+}
+
+// List replays the programmed container set (or error).
+func (f *FakePuller) List(_ context.Context) ([]Container, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Listed = true
+	if f.ListErr != nil {
+		return nil, f.ListErr
+	}
+	return f.Containers, nil
+}
@@ -0,0 +1,89 @@
+package runtime
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// staticResolver is a test Resolver returning fixed creds or an error.
+type staticResolver struct {
+	user, pass, endpoint string
+	err                  error
+}
+
+func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
+	return s.user, s.pass, s.endpoint, s.err
+}
+
+func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
+	f := &FakePuller{}
+	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
+	if err != nil {
+		t.Fatalf("Pull: %v", err)
+	}
+	if img.Ref != "repo/app:1" {
+		t.Errorf("Ref defaulted wrong: %q", img.Ref)
+	}
+	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
+		t.Errorf("Pulls = %+v", f.Pulls)
+	}
+	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
+		t.Errorf("Authzd = %+v", f.Authzd)
+	}
+}
+
+func TestFakePuller_ResolverError(t *testing.T) {
+	f := &FakePuller{}
+	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
+	if err == nil {
+		t.Fatal("expected resolver error to propagate")
+	}
+}
+
+func TestFakePuller_ProgrammedError(t *testing.T) {
+	f := &FakePuller{Err: errors.New("pull failed")}
+	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
+		t.Fatal("expected programmed error")
+	}
+}
+
+func TestFakePuller_OnPullOverrides(t *testing.T) {
+	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
+	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
+		return want, nil
+	}}
+	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
+	if err != nil || got != want {
+		t.Fatalf("got %+v, %v; want %+v", got, err, want)
+	}
+}
+
+func TestFakePuller_Close(t *testing.T) {
+	f := &FakePuller{}
+	if err := f.Close(); err != nil || !f.Closed {
+		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
+	}
+}
+
+func TestNormalizeRef(t *testing.T) {
+	cases := map[string]string{
+		"nginx:alpine":      "docker.io/library/nginx:alpine",
+		"nginx":             "docker.io/library/nginx:latest",
+		"ghcr.io/org/app:1": "ghcr.io/org/app:1",
+		"123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1": "123456789012.dkr.ecr.ap-southeast-2.spinifex.internal/app:1",
+	}
+	for in, want := range cases {
+		got, err := normalizeRef(in)
+		if err != nil {
+			t.Errorf("normalizeRef(%q): %v", in, err)
+			continue
+		}
+		if got != want {
+			t.Errorf("normalizeRef(%q) = %q, want %q", in, got, want)
+		}
+	}
+	if _, err := normalizeRef("Bad::Ref"); err == nil {
+		t.Error("normalizeRef accepted invalid ref")
+	}
+}
@@ -0,0 +1,234 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"maps"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+// pollAssignments drains the instance's assignment inbox on a ticker until ctx is
+// cancelled. Each assign is dispatched to runTask exactly once; the taskIDs seen
+// on a poll are acked on the next poll so the gateway can drop them — a crash
+// before ack re-delivers (at-least-once), matching ACS. A failed poll is logged
+// and retried on the next tick rather than killing the loop. dispatched is seeded
+// by the startup reconcile with tasks already adopted from running containers, so
+// their re-delivered assignments are acked but not re-run.
+func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool) {
+	ticker := time.NewTicker(a.cfg.PollInterval)
+	defer ticker.Stop()
+
+	var ackNext []string
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			assigns, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackNext)
+			if err != nil {
+				slog.Warn("ecs-agent: poll assignments failed", "err", err)
+				continue
+			}
+			ackNext = ackNext[:0]
+			for i := range assigns {
+				as := assigns[i]
+				if !dispatched[as.TaskID] {
+					dispatched[as.TaskID] = true
+					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
+					go a.runTask(ctx, &as)
+				}
+				ackNext = append(ackNext, as.TaskID)
+			}
+		}
+	}
+}
+
+// runTask pulls each container image, starts the containers, and reports task
+// state on the bus. A missing runtime or any per-container failure reports the
+// task STOPPED with a reason; success reports RUNNING and waits for exit.
+func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
+	if a.puller == nil || a.runner == nil {
+		a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
+		return
+	}
+
+	// awsvpc: build the task netns from the hot-plugged ENI before any container.
+	netnsPath, err := a.setupTaskNetns(as)
+	if err != nil {
+		slog.Error("ecs-agent: task netns setup failed", "task", as.TaskID, "err", err)
+		a.reportTaskState(as, bus.TaskStatusStopped, "network setup failed: "+err.Error(), nil)
+		return
+	}
+
+	credID := a.registerTaskCreds(as)
+
+	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
+	for _, c := range as.Containers {
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, a.resolver); err != nil {
+			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
+			return
+		}
+
+		cid := containerID(as.TaskID, c.Name)
+		spec := ctrruntime.RunSpec{
+			Image:     c.Image,
+			Command:   c.Command,
+			Env:       withCredEnv(c.Environment, credID),
+			Labels:    taskLabels(as, c.Name),
+			NetnsPath: netnsPath,
+		}
+		id, err := a.runner.Run(ctx, cid, spec)
+		if err != nil {
+			slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
+			a.teardownTaskNetns(as)
+			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
+			return
+		}
+		statuses = append(statuses, bus.ContainerStatus{Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id})
+		go a.waitContainer(ctx, as, c.Name, id)
+	}
+
+	a.reportTaskState(as, bus.TaskStatusRunning, "", statuses)
+	slog.Info("ecs-agent: task running", "task", as.TaskID, "containers", len(statuses))
+}
+
+// waitContainer blocks until a container exits, then reports the task STOPPED
+// with the exit code. v1 stops the whole task when its first container exits.
+func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
+	status, err := a.runner.Wait(ctx, containerID)
+	if err != nil {
+		slog.Warn("ecs-agent: wait container failed", "task", as.TaskID, "container", name, "err", err)
+		return
+	}
+	exit := status.ExitCode
+	a.teardownTaskNetns(as)
+	statuses := []bus.ContainerStatus{{Name: name, Status: bus.TaskStatusStopped, ContainerID: containerID, ExitCode: &exit}}
+	a.reportTaskState(as, bus.TaskStatusStopped, fmt.Sprintf("container %s exited (%d)", name, exit), statuses)
+	slog.Info("ecs-agent: task stopped", "task", as.TaskID, "container", name, "exitCode", exit)
+}
+
+// setupTaskNetns builds the awsvpc task netns from the hot-plugged ENI. Bridge/
+// host tasks (no ENI MAC) and a nil controller are no-ops returning an empty path.
+func (a *Agent) setupTaskNetns(as *bus.Assign) (string, error) {
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return "", nil
+	}
+	return a.netns.Setup(as.TaskID, as.ENIMacAddress)
+}
+
+// teardownTaskNetns releases the task's IAM credential registration, then removes
+// the awsvpc task netns (the latter is a no-op for bridge/host tasks). It runs at
+// every task-stop path so credentials never outlive the task that owns them.
+func (a *Agent) teardownTaskNetns(as *bus.Assign) {
+	if a.cred != nil {
+		a.cred.Deregister(taskCredID(as))
+	}
+	if as.ENIMacAddress == "" || a.netns == nil {
+		return
+	}
+	if err := a.netns.Teardown(as.TaskID); err != nil {
+		slog.Warn("ecs-agent: task netns teardown", "task", as.TaskID, "err", err)
+	}
+}
+
+// registerTaskCreds registers the task's credID -> role mapping with the
+// credential endpoint and returns the credID (empty when the task has no role).
+func (a *Agent) registerTaskCreds(as *bus.Assign) string {
+	credID := taskCredID(as)
+	if credID == "" || a.cred == nil {
+		return credID
+	}
+	a.cred.Register(credID, as.TaskRoleARN)
+	return credID
+}
+
+// taskCredID is the credential ID a task's containers fetch credentials under:
+// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
+// carries no IAM role.
+func taskCredID(as *bus.Assign) string {
+	if as.TaskRoleARN == "" {
+		return ""
+	}
+	if as.CredID != "" {
+		return as.CredID
+	}
+	return as.TaskID
+}
+
+// withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for
+// credID, copying the map so the assign's container env is left untouched. A
+// blank credID (no task role) returns env unchanged.
+func withCredEnv(env map[string]string, credID string) map[string]string {
+	if credID == "" {
+		return env
+	}
+	out := make(map[string]string, len(env)+1)
+	maps.Copy(out, env)
+	out["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = credRelativeURI(credID)
+	return out
+}
+
+// reportTaskState reports a task transition through the gateway's
+// SubmitTaskStateChange action.
+func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
+	if a.cp == nil {
+		return
+	}
+	msg := bus.TaskState{
+		AccountID:   a.id.AccountID,
+		ClusterName: a.id.ClusterName,
+		InstanceID:  a.id.InstanceID,
+		TaskID:      as.TaskID,
+		LastStatus:  status,
+		Containers:  containers,
+		Reason:      reason,
+		ReportedAt:  time.Now().UTC(),
+	}
+	if err := a.cp.SubmitTaskState(msg); err != nil {
+		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
+	}
+}
+
+// containerID composes a containerd-valid container ID from a task + container.
+func containerID(taskID, name string) string {
+	return fmt.Sprintf("%s-%s", taskID, name)
+}
+
+// mulga.ecs.* label keys stamped on every container. The reboot reconciler reads
+// them back to re-associate running containers with their task on restart; the
+// cred/role/MAC labels make a container self-describing enough to re-register its
+// credentials and tear down its netns without the original assignment.
+const (
+	labelTaskID        = "mulga.ecs.taskID"
+	labelContainerName = "mulga.ecs.containerName"
+	labelClusterName   = "mulga.ecs.clusterName"
+	labelCredID        = "mulga.ecs.credID"
+	labelTaskRoleARN   = "mulga.ecs.taskRoleArn"
+	labelENIMac        = "mulga.ecs.eniMac"
+)
+
+// taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC
+// labels are omitted when empty so a container carries only what it needs.
+func taskLabels(as *bus.Assign, name string) map[string]string {
+	labels := map[string]string{
+		labelTaskID:        as.TaskID,
+		labelContainerName: name,
+		labelClusterName:   as.ClusterName,
+	}
+	if credID := taskCredID(as); credID != "" {
+		labels[labelCredID] = credID
+	}
+	if as.TaskRoleARN != "" {
+		labels[labelTaskRoleARN] = as.TaskRoleARN
+	}
+	if as.ENIMacAddress != "" {
+		labels[labelENIMac] = as.ENIMacAddress
+	}
+	return labels
+}
@@ -0,0 +1,229 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
+)
+
+func testAssign() *bus.Assign {
+	return &bus.Assign{
+		AccountID:   "123456789012",
+		ClusterName: "default",
+		InstanceID:  "i-abc123",
+		TaskID:      "t-001",
+		Containers: []bus.AssignContainer{
+			{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"},
+				Environment: map[string]string{"FOO": "bar"}},
+		},
+	}
+}
+
+func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
+	return newAgent(config{}, testIdentity(), cp, rt, rt, nil)
+}
+
+// runTask with no runtime reports the task STOPPED instead of crashing.
+func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED state, got %+v", st)
+	}
+	if st[0].Reason == "" {
+		t.Error("STOPPED state missing reason")
+	}
+}
+
+// Happy path: pull + run reports RUNNING with the container ID; Wait blocking
+// (forced error) means no STOPPED overwrite, so we can assert the RUNNING frame.
+func TestRunTask_PullRunReportsRunning(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if len(rt.Pulls) != 1 || rt.Pulls[0].Ref != "registry/web:1" {
+		t.Fatalf("expected one pull of registry/web:1, got %+v", rt.Pulls)
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].Image != "registry/web:1" {
+		t.Fatalf("expected one run, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].Labels["mulga.ecs.taskID"] != "t-001" {
+		t.Errorf("missing task label: %+v", rt.Runs[0].Labels)
+	}
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	if len(st[len(st)-1].Containers) != 1 || st[len(st)-1].Containers[0].ContainerID != "t-001-web" {
+		t.Errorf("RUNNING state container wrong: %+v", st[len(st)-1].Containers)
+	}
+}
+
+// A pull failure reports STOPPED and never starts the container.
+func TestRunTask_PullFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Err: errors.New("pull denied")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	if len(rt.Runs) != 0 {
+		t.Errorf("container should not run after pull failure, got %+v", rt.Runs)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// A run failure reports STOPPED.
+func TestRunTask_RunFailureReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{RunErr: errors.New("start boom")}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want STOPPED, got %+v", st)
+	}
+}
+
+// On container exit, waitContainer reports STOPPED with the exit code.
+func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitCode: 7}
+	a := newRunAgent(cp, rt)
+	a.runTask(context.Background(), testAssign())
+
+	deadline := time.After(time.Second)
+	for {
+		st := cp.taskStates()
+		if len(st) > 0 && st[len(st)-1].LastStatus == bus.TaskStatusStopped {
+			last := st[len(st)-1]
+			if len(last.Containers) != 1 || last.Containers[0].ExitCode == nil || *last.Containers[0].ExitCode != 7 {
+				t.Fatalf("want exit code 7, got %+v", last.Containers)
+			}
+			return
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("no STOPPED state after exit; got %+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+}
+
+// pollAssignments dispatches each assign once and acks it on the next poll.
+func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{pollReplies: [][]bus.Assign{
+		{*as}, // poll 1: one pending assign
+		{*as}, // poll 2: still present (not yet acked when poll 2 was issued)
+		nil,   // poll 3: gateway dropped it after the ack
+	}}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	// runTask reports RUNNING through the (mutex-guarded) control plane. With the
+	// assign returned on two consecutive polls, dedup means exactly one dispatch →
+	// exactly one RUNNING report.
+	deadline := time.After(time.Second)
+	for {
+		if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("assign was never dispatched (no RUNNING report)")
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 1 {
+		t.Errorf("task dispatched %d times, want exactly 1 (dedup on taskID)", running)
+	}
+	// The taskID must appear in a later poll's ack list.
+	acked := false
+	for _, ack := range cp.acks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("task %s was never acked; acks=%v", as.TaskID, cp.acks())
+	}
+}
+
+func countStatus(states []bus.TaskState, taskID, status string) int {
+	n := 0
+	for _, st := range states {
+		if st.TaskID == taskID && st.LastStatus == status {
+			n++
+		}
+	}
+	return n
+}
+
+// awsvpc: an assign carrying an ENI MAC builds the task netns and passes its
+// path to the container RunSpec.
+func TestRunTask_AwsvpcBuildsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	as := testAssign()
+	as.ENIMacAddress = "52:54:00:de:ad:01"
+	a.runTask(context.Background(), as)
+
+	if !f.sawAny("ip netns add ecs-t-001") {
+		t.Fatalf("expected task netns build, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != netnsPathFor("t-001") {
+		t.Fatalf("want NetnsPath %q on RunSpec, got %+v", netnsPathFor("t-001"), rt.Runs)
+	}
+}
+
+// bridge/host: no ENI MAC means no netns is built and the container runs in the
+// host (VM) netns (empty NetnsPath).
+func TestRunTask_NoMacSkipsNetns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	f := &fakeNetRunner{linkOut: linkWithENI}
+	a.netns = newTestNetns(f)
+
+	a.runTask(context.Background(), testAssign())
+
+	if len(f.joined()) != 0 {
+		t.Fatalf("expected no netns commands, got %v", f.joined())
+	}
+	if len(rt.Runs) != 1 || rt.Runs[0].NetnsPath != "" {
+		t.Fatalf("want empty NetnsPath, got %+v", rt.Runs)
+	}
+}
+
+func TestContainerID(t *testing.T) {
+	if got := containerID("t-1", "web"); got != "t-1-web" {
+		t.Errorf("containerID = %q, want t-1-web", got)
+	}
+}
@@ -0,0 +1,139 @@
+// eks-gateway-fetch runs inside the EKS K3s control-plane VM and fetches a
+// control-plane resource from the host through the AWS gateway, instead of
+// dialing core NATS directly. It is the read-side companion to
+// eks-gateway-publish.
+//
+// It SigV4-signs (service "eks") an HTTPS GET to the gateway, retrying with
+// backoff until the gateway returns 2xx or the attempt budget is exhausted (so
+// a degraded link surfaces as a non-zero exit), then emits the staged add-ons
+// as tab-separated lines the on-VM addon-sync shell agent consumes without a
+// JSON parser:
+//
+//	<addonName>\t<addonVersion>\t<serviceAccountRoleArn>\t<base64(configurationValues)>
+//
+// The configuration values are base64-encoded so embedded tabs/newlines cannot
+// break the line framing.
+//
+// Usage:
+//
+//	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
+//
+// Flags default to environment variables seeded by cloud-init:
+// EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
+// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
+package main
+
+import (
+	"bufio"
+	"encoding/base64"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/eksgw"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+// stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated
+// here to keep this tiny VM binary free of the handler package's dependency
+// graph.
+type stagedAddon struct {
+	AddonName             string `json:"addonName"`
+	AddonVersion          string `json:"addonVersion"`
+	ServiceAccountRoleArn string `json:"serviceAccountRoleArn"`
+	ConfigurationValues   string `json:"configurationValues"`
+}
+
+type internalAddonsResponse struct {
+	Addons []stagedAddon `json:"addons"`
+}
+
+const (
+	maxAttempts = 30
+	retryDelay  = 5 * time.Second
+)
+
+func main() {
+	var (
+		gatewayURL  string
+		gatewayCA   string
+		accessKey   string
+		secretKey   string
+		region      string
+		accountID   string
+		clusterName string
+		resource    string
+	)
+
+	flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
+	flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
+	flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
+	flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
+	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
+	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
+	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
+	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons")
+	flag.Parse()
+
+	switch {
+	case accountID == "":
+		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
+	case clusterName == "":
+		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
+	case resource != "addons":
+		fatal("--resource must be addons")
+	}
+
+	client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
+	if err != nil {
+		fatal(fmt.Sprintf("build gateway client: %v", err))
+	}
+	path := "/clusters/" + clusterName + "/internal-addons/" + accountID
+
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		body, err := client.Get(path)
+		if err != nil {
+			lastErr = err
+			slog.Warn("eks-gateway-fetch: attempt failed", "resource", resource, "attempt", attempt, "err", err)
+			if attempt < maxAttempts {
+				time.Sleep(retryDelay)
+			}
+			continue
+		}
+		if werr := emitAddonsTSV(os.Stdout, body); werr != nil {
+			fatal(fmt.Sprintf("render response: %v", werr))
+		}
+		return
+	}
+	fatal(fmt.Sprintf("fetch failed after %d attempts: %v", maxAttempts, lastErr))
+}
+
+// emitAddonsTSV decodes the internal-addons response and writes one TSV line per
+// staged add-on to w. An empty add-on set writes nothing (the agent then GCs
+// every locally-rendered manifest), which is the correct steady state for a
+// cluster with no managed add-ons.
+func emitAddonsTSV(out io.Writer, body []byte) error {
+	var resp internalAddonsResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-addons response: %w", err)
+	}
+	w := bufio.NewWriter(out)
+	for _, a := range resp.Addons {
+		cfg := base64.StdEncoding.EncodeToString([]byte(a.ConfigurationValues))
+		if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", a.AddonName, a.AddonVersion, a.ServiceAccountRoleArn, cfg); err != nil {
+			return err
+		}
+	}
+	return w.Flush()
+}
+
+func fatal(msg string) {
+	slog.Error("eks-gateway-fetch: " + msg)
+	os.Exit(1)
+}
@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"testing"
+)
+
+func TestEmitAddonsTSV(t *testing.T) {
+	body := []byte(`{"addons":[
+		{"addonName":"spinifex-noop","addonVersion":"0.1.0"},
+		{"addonName":"aws-load-balancer-controller","addonVersion":"2.8.1",
+		 "serviceAccountRoleArn":"arn:aws:iam::111122223333:role/alb",
+		 "configurationValues":"{\"replicaCount\":2}"}
+	]}`)
+
+	var buf bytes.Buffer
+	if err := emitAddonsTSV(&buf, body); err != nil {
+		t.Fatalf("emitAddonsTSV: %v", err)
+	}
+	cfg := base64.StdEncoding.EncodeToString([]byte(`{"replicaCount":2}`))
+	want := "spinifex-noop\t0.1.0\t\t\n" +
+		"aws-load-balancer-controller\t2.8.1\tarn:aws:iam::111122223333:role/alb\t" + cfg + "\n"
+	if buf.String() != want {
+		t.Fatalf("TSV mismatch:\n got: %q\nwant: %q", buf.String(), want)
+	}
+}
+
+func TestEmitAddonsTSV_Empty(t *testing.T) {
+	var buf bytes.Buffer
+	if err := emitAddonsTSV(&buf, []byte(`{"addons":[]}`)); err != nil {
+		t.Fatalf("emitAddonsTSV: %v", err)
+	}
+	if buf.Len() != 0 {
+		t.Fatalf("empty addon set must emit nothing, got %q", buf.String())
+	}
+}
+
+func TestEmitAddonsTSV_BadJSON(t *testing.T) {
+	var buf bytes.Buffer
+	if err := emitAddonsTSV(&buf, []byte(`{not-json`)); err == nil {
+		t.Fatal("expected error on malformed JSON")
+	}
+}
@@ -69,7 +69,7 @@ func main() {
 	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
 	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
-	flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state")
+	flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state|addon")
 	flag.StringVar(&kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
 	flag.Parse()

@@ -78,8 +78,8 @@ func main() {
 		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case clusterName == "":
 		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case channel != "bootstrap" && channel != "state":
-		fatal("--channel must be bootstrap or state")
+	case channel != "bootstrap" && channel != "state" && channel != "addon":
+		fatal("--channel must be bootstrap, state or addon")
 	case channel == "bootstrap" && kind == "":
 		fatal("--kind is required for the bootstrap channel")
 	}
@@ -0,0 +1,147 @@
+// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving
+// certificate for an in-cluster admission webhook and prints its base64 PEM so
+// the addon renderer can embed it as the webhook Secret + each caBundle.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent of the
+// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster,
+// cached on disk under -dir, stable across addon-sync ticks. The single cert is
+// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf
+// and the caBundle the apiserver trusts.
+//
+// Usage:
+//
+//	eks-webhook-cert -dir /var/lib/spinifex-eks/webhook-certs/<addon> \
+//	    -cn <svc>.<ns>.svc -dns <svc>.<ns>.svc,<svc>.<ns>.svc.cluster.local
+//
+// Output (one line): <ca_b64>\t<tls_crt_b64>\t<tls_key_b64>
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	cn := flag.String("cn", "", "certificate common name")
+	dnsCSV := flag.String("dns", "", "comma-separated DNS SANs")
+	flag.Parse()
+
+	if *dir == "" || *cn == "" || *dnsCSV == "" {
+		fatal("eks-webhook-cert: -dir, -cn and -dns are required")
+	}
+	dnsNames := splitCSV(*dnsCSV)
+	if len(dnsNames) == 0 {
+		fatal("eks-webhook-cert: -dns lists no names")
+	}
+
+	certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames)
+	if err != nil {
+		fatal("eks-webhook-cert: " + err.Error())
+	}
+	b := base64.StdEncoding.EncodeToString
+	fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM))
+}
+
+// ensureCert returns the cached cert/key under dir, minting and persisting a
+// fresh self-signed pair (for cn + dnsNames) when none is present or readable.
+func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if c, cerr := os.ReadFile(certPath); cerr == nil {
+		if k, kerr := os.ReadFile(keyPath); kerr == nil {
+			if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
+				return c, k, nil
+			}
+		}
+	}
+
+	certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPEM, keyPEM, nil
+}
+
+// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
+// a server cert, with the given common name and DNS SANs.
+func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		DNSNames:              dnsNames,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func splitCSV(s string) []string {
+	var out []string
+	for p := range strings.SplitSeq(s, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/x509"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const (
+	testCN  = "aws-load-balancer-webhook-service.kube-system.svc"
+	testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local"
+)
+
+func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
+	t.Helper()
+	leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
+	if err != nil {
+		t.Fatalf("parse cert: %v", err)
+	}
+	return leaf
+}
+
+func TestEnsureCertMintsServingCert(t *testing.T) {
+	dir := t.TempDir()
+	certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS})
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if len(keyPEM) == 0 {
+		t.Fatal("empty key PEM")
+	}
+	leaf := parseLeaf(t, certPEM)
+
+	// Both webhook service DNS names must be present, or the apiserver's TLS
+	// dial to the webhook fails SAN verification.
+	for _, want := range []string{testCN, testDNS} {
+		if err := leaf.VerifyHostname(want); err != nil {
+			t.Errorf("SAN %q missing: %v", want, err)
+		}
+	}
+	if !leaf.IsCA {
+		t.Error("cert must be a CA so the same PEM serves as the webhook caBundle")
+	}
+
+	// The single self-signed cert must verify against itself as a root — this is
+	// exactly what the apiserver does with caBundle == the serving cert.
+	roots := x509.NewCertPool()
+	roots.AddCert(leaf)
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   testCN,
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("self-signed cert does not verify as its own CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesCachedPair(t *testing.T) {
+	dir := t.TempDir()
+	cert1, key1, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	cert2, key2, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if string(cert1) != string(cert2) || string(key1) != string(key2) {
+		t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick")
+	}
+
+	// The private key must not be world-readable on disk.
+	info, err := os.Stat(filepath.Join(dir, "tls.key"))
+	if err != nil {
+		t.Fatalf("stat key: %v", err)
+	}
+	if info.Mode().Perm()&0o077 != 0 {
+		t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm())
+	}
+}
@@ -35,6 +35,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -115,6 +116,33 @@ Each phase waits for all nodes to ACK before proceeding to the next.`,
 	Run: runClusterShutdown,
 }

+var nodeCmd = &cobra.Command{
+	Use:   "node",
+	Short: "Node-local operations",
+	Long:  `Node-local administrative operations such as a graceful local guest drain.`,
+}
+
+var nodeDrainCmd = &cobra.Command{
+	Use:   "drain",
+	Short: "Gracefully drain guests on the local node",
+	Long: `Run the GATE and DRAIN shutdown phases against the local node only: power down
+its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
+every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
+ordered unit teardown. This is the ExecStop of spinifex-shutdown.service, so a
+systemctl stop or host reboot drains guests before any storage service stops.`,
+	Run: runNodeDrainLocal,
+}
+
+var clusterDrainDHCPCmd = &cobra.Command{
+	Use:   "drain-dhcp",
+	Short: "Release all upstream DHCP leases held by vpcd",
+	Long: `Ask each vpcd to DHCPRELEASE every external-pool DHCP lease it currently
+holds, returning them to the upstream DHCP server. Run this on teardown before
+stopping services: an env reset otherwise strands held leases on the upstream
+server until their TTL expires, eventually exhausting the upstream scope.`,
+	Run: runClusterDrainDHCP,
+}
+
 var adminInitCmd = &cobra.Command{
 	Use:   "init",
 	Short: "Initialize Spinifex platform configuration",
@@ -245,6 +273,14 @@ func init() {
 	clusterShutdownCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
 	clusterShutdownCmd.Flags().Bool("dry-run", false, "Print phase plan without executing")

+	clusterCmd.AddCommand(clusterDrainDHCPCmd)
+	clusterDrainDHCPCmd.Flags().Duration("timeout", 30*time.Second, "Reply-collection window for vpcd drain responders")
+
+	adminCmd.AddCommand(nodeCmd)
+	nodeCmd.AddCommand(nodeDrainCmd)
+	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
+	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
+
 	adminCmd.AddCommand(imagesCmd)
 	imagesCmd.AddCommand(imagesImportCmd)
 	imagesCmd.AddCommand(imagesListCmd)
@@ -285,6 +321,7 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("token-ttl", "30m", "Join token validity duration (e.g. 30m, 1h, 2h)")
+	adminInitCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
 	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
@@ -318,6 +355,7 @@ func init() {
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
+	adminJoinCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 	adminJoinCmd.MarkFlagRequired("token")
@@ -628,6 +666,13 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		EncryptionEnabled: mkey != nil,
 	}

+	// Bake the deployment CA into the image trust store so a stock cloud image
+	// trusts the gateway from first boot. Best-effort: never fails the import.
+	// The CA path derives from the --spinifex-dir data root (its config/ symlink
+	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
+	// command, which would collapse the path to a relative config/ca.pem.
+	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
+
 	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath)

 	if err != nil {
@@ -640,6 +685,64 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }

+// caBakeRunCommand installs the uploaded CA into the guest trust store across
+// the distro families, then removes the staged copy. Each branch is gated on its
+// updater existing so only the matching distro's branch writes anything (install
+// -D creates the anchor dir on minimal images). Debian/Ubuntu use
+// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
+// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
+// appending the PEM to the static bundle. The install chain's exit code is
+// preserved past the cleanup rm so a failure surfaces instead of a false success.
+// The RHEL branch restorecon's only the trust-store paths it touched so the cert
+// is correctly labelled without the image-wide --no-selinux-relabel (below)
+// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
+const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
+	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
+	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
+	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
+	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
+
+// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
+// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
+const caBakeTimeout = 5 * time.Minute
+
+// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
+// into the disk image at imagePath and installs it into the guest trust store.
+// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
+// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
+// reboot, so the run-command relabels only the touched trust paths instead.
+func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
+	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
+		"--no-selinux-relabel",
+		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
+		"--run-command", caBakeRunCommand)
+}
+
+// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
+var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
+	if _, err := exec.LookPath("virt-customize"); err != nil {
+		return nil, fmt.Errorf("virt-customize not found: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
+	defer cancel()
+	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
+}
+
+// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
+// virt-customize so an imported stock image trusts the gateway from first boot.
+// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
+// cannot inspect logs and continues — image import never fails on the CA.
+func bakeCACertIntoImage(imagePath, caCertPath string) {
+	if _, err := os.Stat(caCertPath); err != nil {
+		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+		return
+	}
+	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
+		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
+		return
+	}
+}
+
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 	imageID, _ := cmd.Flags().GetString("image-id")
 	force, _ := cmd.Flags().GetBool("force")
@@ -778,17 +881,8 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
 		{"NAME", "DISTRO", "VERSION", "ARCH", "BOOT"},
 	}

-	// Sort A .. Z
-	// 1. Collect keys
-	keys := make([]string, 0, len(utils.AvailableImages))
-	for k := range utils.AvailableImages {
-		keys = append(keys, k)
-	}
-
-	// 2. Sort keys alphabetically (A→Z)
-	sort.Strings(keys)
-
-	// 3. Iterate in sorted order
+	// Sort A→Z then iterate.
+	keys := slices.Sorted(maps.Keys(utils.AvailableImages))
 	for _, k := range keys {
 		img := utils.AvailableImages[k]

@@ -833,6 +927,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
 	formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout")
 	tokenTTLStr, _ := cmd.Flags().GetString("token-ttl")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")

@@ -1114,6 +1209,17 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
 	fmt.Printf("   Key: %s\n", predastoreKeyPath)

+	// Viperblock at-rest encryption key is cluster-wide; on a single-node init
+	// there are no joiners, so generate it locally and enable encryption by
+	// default for all volumes created on this install.
+	viperblockKeyPath, err := writeViperblockEncryptionKey(configDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating viperblock encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated viperblock at-rest encryption key")
+	fmt.Printf("   Key: %s\n", viperblockKeyPath)
+
 	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
 	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
 	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
@@ -1121,7 +1227,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   AWS Profile: spinifex\n")

 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)

 	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
 	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
@@ -1232,7 +1338,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}

 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1273,8 +1379,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,

 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",
@@ -1301,6 +1408,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {

 		GPUPassthrough: gpuPassthrough,
 		IPSecEnabled:   ipsecEnabled,
+
+		EncryptionKeyFile: viperblockKeyPath,
 	}

 	// Print external networking summary
@@ -1377,6 +1486,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
+
 	joinToken, err := formation.GenerateJoinToken()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error generating join token: %v\n", err)
@@ -1409,6 +1520,22 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
 	fmt.Printf("   Key: %s\n", predastoreKeyPath)

+	// Viperblock at-rest encryption key is cluster-wide and shared: the leader
+	// generates it once and distributes it to joiners via the formation server
+	// so a volume sealed on any node can be opened on any other.
+	viperblockKey, err := handlers_iam.GenerateMasterKey()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error generating viperblock encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	viperblockKeyPath, err := saveViperblockEncryptionKey(configDir, viperblockKey)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error saving viperblock encryption key: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println("\n🔐 Generated viperblock at-rest encryption key (cluster-wide, shared with joiners)")
+	fmt.Printf("   Key: %s\n", viperblockKeyPath)
+
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
 	if err != nil {
@@ -1437,6 +1564,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// Include master key in formation server for distribution to joining nodes
 	fs.SetMasterKey(base64.StdEncoding.EncodeToString(masterKey))

+	// Distribute the cluster-wide viperblock encryption key to joiners
+	fs.SetViperblockKey(base64.StdEncoding.EncodeToString(viperblockKey))
+
 	// Register self (init node) as the first node
 	selfNode := formation.NodeInfo{
 		Name:        node,
@@ -1501,7 +1631,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	var predastoreNodeID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1538,12 +1668,15 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(allNodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(allNodes, node),

 		OperatorEmail: email,

+		EncryptionKeyFile: viperblockKeyPath,
+
 		// Init node runs ovn-central locally
 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",
@@ -1585,7 +1718,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	}
 	fmt.Println("\n📋 Next steps:")
 	fmt.Println("   1. Start services on ALL nodes:")
-	fmt.Println("      ./scripts/start-dev.sh")
+	fmt.Println("      sudo systemctl start spinifex.target")
 	fmt.Println()
 }

@@ -1608,6 +1741,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	configDir, _ := cmd.Flags().GetString("config-dir")
 	clusterBind, _ := cmd.Flags().GetString("cluster-bind")
 	services, _ := cmd.Flags().GetStringSlice("services")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")

 	email, _ := cmd.Flags().GetString("email")
 	email = strings.TrimSpace(email)
@@ -1889,8 +2023,30 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ Predastore encryption key generated: %s\n", predastoreKeyPath)

+	// Viperblock at-rest encryption key is cluster-wide: receive it from the
+	// leader rather than generating one, so this node can open volumes sealed
+	// elsewhere. Lenient on absence — an older leader that predates key
+	// distribution returns nothing; fall back to cleartext rather than fail
+	// the join.
+	var viperblockKeyPath string
+	if statusResp.ViperblockKey == "" {
+		fmt.Println("⚠️  Leader did not provide a viperblock encryption key; at-rest encryption disabled on this node")
+	} else {
+		viperblockKeyBytes, err := base64.StdEncoding.DecodeString(statusResp.ViperblockKey)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error decoding viperblock encryption key: %v\n", err)
+			os.Exit(1)
+		}
+		viperblockKeyPath, err = saveViperblockEncryptionKey(configDir, viperblockKeyBytes)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error saving viperblock encryption key: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Viperblock encryption key received from leader: %s\n", viperblockKeyPath)
+	}
+
 	// Generate server cert signed by CA with this node's bind IP
-	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
+	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSInternalSuffix); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -1933,7 +2089,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	hasPredastoreConfig := len(predastoreNodes) >= 2

 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1974,12 +2130,15 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(statusResp.Nodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node),

 		OperatorEmail: email,

+		EncryptionKeyFile: viperblockKeyPath,
+
 		// Joining nodes connect to the init node's OVN NB/SB DB
 		OVNNBAddr: fmt.Sprintf("tcp:%s:6641", leaderIP),
 		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
@@ -2265,6 +2424,7 @@ func runCertRenew(cmd *cobra.Command, _ []string) {
 type configDirs struct {
 	AWSGW      string
 	Predastore string
+	Viperblock string
 	NATS       string
 	Spinifex   string
 }
@@ -2274,10 +2434,11 @@ func createConfigSubdirs(configDir string) (configDirs, error) {
 	dirs := configDirs{
 		AWSGW:      filepath.Join(configDir, "awsgw"),
 		Predastore: filepath.Join(configDir, "predastore"),
+		Viperblock: filepath.Join(configDir, "viperblock"),
 		NATS:       filepath.Join(configDir, "nats"),
 		Spinifex:   filepath.Join(configDir, "spinifex"),
 	}
-	for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.NATS, dirs.Spinifex} {
+	for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.Viperblock, dirs.NATS, dirs.Spinifex} {
 		if err := os.MkdirAll(dir, 0700); err != nil {
 			return configDirs{}, fmt.Errorf("create directory %s: %w", dir, err)
 		}
@@ -2399,6 +2560,43 @@ func writePredastoreEncryptionKey(configDir string) (string, error) {
 	return keyPath, nil
 }

+// writeViperblockEncryptionKey generates a fresh 32-byte AES-256 master key for
+// viperblock at-rest encryption and writes it to
+// <configDir>/viperblock/encryption.key with mode 0600. Unlike the predastore
+// key, this key is cluster-wide and shared: the leader generates it once at init
+// and distributes it to joiners via the formation server, so a volume sealed on
+// any node can be opened on any other. Loaded at startup via masterkey.LoadShared.
+func writeViperblockEncryptionKey(configDir string) (string, error) {
+	viperblockDir := filepath.Join(configDir, "viperblock")
+	if err := os.MkdirAll(viperblockDir, 0750); err != nil {
+		return "", fmt.Errorf("create viperblock config dir: %w", err)
+	}
+	key, err := handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return "", fmt.Errorf("generate viperblock encryption key: %w", err)
+	}
+	keyPath := filepath.Join(viperblockDir, "encryption.key")
+	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
+		return "", fmt.Errorf("save viperblock encryption key: %w", err)
+	}
+	return keyPath, nil
+}
+
+// saveViperblockEncryptionKey writes an already-generated 32-byte viperblock
+// master key (received from the formation leader) to
+// <configDir>/viperblock/encryption.key with mode 0600.
+func saveViperblockEncryptionKey(configDir string, key []byte) (string, error) {
+	viperblockDir := filepath.Join(configDir, "viperblock")
+	if err := os.MkdirAll(viperblockDir, 0750); err != nil {
+		return "", fmt.Errorf("create viperblock config dir: %w", err)
+	}
+	keyPath := filepath.Join(viperblockDir, "encryption.key")
+	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
+		return "", fmt.Errorf("save viperblock encryption key: %w", err)
+	}
+	return keyPath, nil
+}
+
 // writeSystemCredentials writes the system access key to a plaintext JSON file.
 // The daemon reads this at startup to inject credentials into ALB VM cloud-init
 // for SigV4-authenticated communication with the AWS gateway.
@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCABakeCmd_Args(t *testing.T) {
+	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
+	args := cmd.Args
+
+	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
+	assert.Contains(t, args, "-a")
+	assert.Contains(t, args, "/img/disk.raw")
+	assert.Contains(t, args, "--upload")
+	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
+	assert.Contains(t, args, "--run-command")
+
+	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
+	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
+	assert.Contains(t, args, "--no-selinux-relabel")
+
+	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
+	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
+	// ca-certificates bundle when no updater is present. The RHEL branch relabels
+	// only the touched trust paths so the cert is labelled without a full relabel.
+	joined := strings.Join(args, " ")
+	assert.Contains(t, joined, "update-ca-certificates")
+	assert.Contains(t, joined, "update-ca-trust")
+	assert.Contains(t, joined, "restorecon")
+	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
+}
+
+func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
+	defer func() { caBakeRunner = orig }()
+
+	// A missing CA must not invoke virt-customize and must not panic.
+	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
+	assert.False(t, called, "runner must not be called when the CA is absent")
+}
+
+func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
+	caPath := filepath.Join(t.TempDir(), "ca.pem")
+	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
+
+	called := false
+	orig := caBakeRunner
+	caBakeRunner = func(string, string) ([]byte, error) {
+		called = true
+		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
+	}
+	defer func() { caBakeRunner = orig }()
+
+	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
+	bakeCACertIntoImage("/img/disk.raw", caPath)
+	assert.True(t, called, "runner must be invoked when the CA is present")
+}
@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/utils"
+	toml "github.com/pelletier/go-toml/v2"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -206,7 +209,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 		{ID: 3, Host: "10.0.0.3"},
 	}
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content, `nats_url = "nats://localhost:4222"`)
@@ -214,7 +217,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {

 	// Specific bind IP → stays as-is.
 	content2, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
@@ -252,6 +255,74 @@ func TestWritePredastoreEncryptionKey(t *testing.T) {
 	assert.NotEqual(t, key1, key2, "per-node keys must differ")
 }

+// The viperblock key gates at-rest encryption for every volume on a fresh
+// install. masterkey.LoadShared rejects anything that isn't exactly 32 bytes at
+// 0640-or-stricter, so a regression in size/mode would silently fall back to
+// cleartext (empty path) or fail service startup.
+func TestWriteViperblockEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writeViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
+}
+
+// Joiners persist the leader's shared key via saveViperblockEncryptionKey
+// against a configDir with no viperblock/ subdir yet. The bytes round-trip
+// verbatim — the whole cluster shares one key, so any mutation would orphan
+// volumes sealed on other nodes.
+func TestSaveViperblockEncryptionKey_RoundTrip(t *testing.T) {
+	configDir := t.TempDir()
+	key := make([]byte, 32)
+	for i := range key {
+		key[i] = byte(i)
+	}
+
+	keyPath, err := saveViperblockEncryptionKey(configDir, key)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
+
+	got, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, key, got, "shared key must be written verbatim")
+}
+
+// A fresh install must render encryption_key_file so viperblockd enables
+// at-rest encryption; an empty field (upgrade / legacy) must omit the line
+// entirely so the volume stays in cleartext legacy mode.
+func TestSpinifexTomlTemplate_EncryptionKeyFile(t *testing.T) {
+	base := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432", Region: "ap-southeast-2",
+		BindIP: "0.0.0.0", AccessKey: "AKIATEST", SecretKey: "SECRET",
+		AccountID: "123456789012", NatsToken: "token",
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+	}
+
+	dir := t.TempDir()
+	withKey := base
+	withKey.ConfigDir = dir
+	withKey.EncryptionKeyFile = "/etc/spinifex/viperblock/encryption.key"
+	path := filepath.Join(dir, "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, withKey))
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Contains(t, string(data), `encryption_key_file = "/etc/spinifex/viperblock/encryption.key"`)
+
+	dir2 := t.TempDir()
+	noKey := base
+	noKey.ConfigDir = dir2
+	path2 := filepath.Join(dir2, "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path2, spinifexTomlTemplate, noKey))
+	data2, err := os.ReadFile(path2)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data2), "encryption_key_file", "blank key must omit the field")
+}
+
 // Joiners run `spx admin join` against a configDir that doesn't yet contain
 // a predastore/ subdir. The helper must create it (predastore subdir is
 // owned by spinifex-storage in prod, but the helper just needs the dir to
@@ -386,3 +457,90 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 	assert.Equal(t, "dhcp", settings.PoolSource)
 	assert.Equal(t, "br-wan", settings.PoolBindBridge)
 }
+
+func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "predastore.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, predastoreTomlTemplate, settings))
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
+	return []admin.PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"},
+	}
+}
+
+// Unset knob must leave production config byte-identical: no [compaction] block
+// and the original trailing bytes unchanged (single-node has no trailing
+// newline, multinode keeps one).
+func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok",
+	})
+	assert.NotContains(t, single, "[compaction]")
+	assert.True(t, strings.HasSuffix(single, `access_keys_bucket = "spinifex-iam-access-keys"`),
+		"unset single-node tail changed: %q", single[len(single)-40:])
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0,
+	)
+	require.NoError(t, err)
+	assert.NotContains(t, multi, "[compaction]")
+	assert.True(t, strings.HasSuffix(multi, "access_keys_bucket = \"spinifex-iam-access-keys\"\n"),
+		"unset multinode tail changed: %q", multi[len(multi)-40:])
+}
+
+func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
+	const interval = 30
+
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok", CompactionIntervalSeconds: interval,
+	})
+	assert.Contains(t, single, "[compaction]")
+	var singleCfg compactionConfig
+	require.NoError(t, toml.Unmarshal([]byte(single), &singleCfg))
+	assert.Equal(t, interval, singleCfg.Compaction.IntervalSeconds)
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", interval,
+	)
+	require.NoError(t, err)
+	var multiCfg compactionConfig
+	require.NoError(t, toml.Unmarshal([]byte(multi), &multiCfg))
+	assert.Equal(t, interval, multiCfg.Compaction.IntervalSeconds)
+}
+
+// compactionConfig parses just the rendered [compaction] block. Predastore's
+// s3.Config dropped its Compaction field, so this asserts the spinifex template
+// still emits a valid, parseable block carrying the interval.
+type compactionConfig struct {
+	Compaction struct {
+		IntervalSeconds int `toml:"interval_seconds"`
+	} `toml:"compaction"`
+}
+
+// The init/join flag must register with an unset (0) default and flow into the
+// rendered config via ConfigSettings.
+func TestAdminInitFlag_CompactionIntervalReachesRenderedConfig(t *testing.T) {
+	for _, c := range []*cobra.Command{adminInitCmd, adminJoinCmd} {
+		f := c.Flags().Lookup("predastore-compaction-interval")
+		require.NotNil(t, f, "%s missing --predastore-compaction-interval", c.Name())
+		assert.Equal(t, "0", f.DefValue)
+	}
+
+	require.NoError(t, adminInitCmd.Flags().Set("predastore-compaction-interval", "45"))
+	defer func() { _ = adminInitCmd.Flags().Set("predastore-compaction-interval", "0") }()
+
+	v, _ := adminInitCmd.Flags().GetInt("predastore-compaction-interval")
+	require.Equal(t, 45, v)
+
+	out := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", CompactionIntervalSeconds: v,
+	})
+	assert.Contains(t, out, "interval_seconds = 45")
+}
@@ -8,6 +8,7 @@ import (
 	"time"

 	"github.com/mulgadc/spinifex/spinifex/daemon"
+	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
 	"github.com/nats-io/nats.go"
 	"github.com/spf13/cobra"
 )
@@ -160,6 +161,166 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 	fmt.Printf("Cluster shutdown complete (%s)\n", time.Since(start).Round(time.Millisecond))
 }

+// runNodeDrainLocal runs the GATE and DRAIN shutdown phases against the local
+// node only, leaving STORAGE/PERSIST/INFRA to systemd's ordered unit teardown.
+// It is the ExecStop of spinifex-shutdown.service: guests are powered down and
+// their volumes unmounted while every service is still up, so the subsequent
+// service stops never tear storage out from under a running guest.
+func runNodeDrainLocal(cmd *cobra.Command, args []string) {
+	local, _ := cmd.Flags().GetBool("local")
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	if !local {
+		fmt.Fprintln(os.Stderr, "Error: node drain currently supports only --local")
+		os.Exit(1)
+	}
+
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	node := cfg.Node
+	fmt.Printf("Draining local node %q (gate -> drain), timeout %s/phase\n", node, timeout)
+
+	for _, phase := range []string{"gate", "drain"} {
+		topic := "spinifex.cluster.shutdown." + phase
+		req := daemon.ShutdownRequest{Phase: phase, Timeout: int(timeout.Seconds())}
+		reqData, err := json.Marshal(req)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error marshaling request: %v\n", err)
+			os.Exit(1)
+		}
+
+		ack, err := collectLocalShutdownACK(nc, topic, reqData, node, timeout)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[%s] %v\n", strings.ToUpper(phase), err)
+			os.Exit(1)
+		}
+		if ack.Error != "" {
+			fmt.Fprintf(os.Stderr, "[%s] %s: ERROR - %s\n", strings.ToUpper(phase), ack.Node, ack.Error)
+			os.Exit(1)
+		}
+		if len(ack.Stopped) > 0 {
+			fmt.Printf("[%s] %s: stopped %s\n", strings.ToUpper(phase), ack.Node, strings.Join(ack.Stopped, ", "))
+		} else {
+			fmt.Printf("[%s] %s: OK\n", strings.ToUpper(phase), ack.Node)
+		}
+	}
+
+	fmt.Printf("Local node %q drained; systemd may now stop storage services\n", node)
+}
+
+// collectLocalShutdownACK publishes a shutdown-phase request and returns the ACK
+// from the local node, ignoring ACKs from any other node. Returns an error if
+// the local node does not respond within timeout.
+func collectLocalShutdownACK(nc *nats.Conn, topic string, reqData []byte, node string, timeout time.Duration) (daemon.ShutdownACK, error) {
+	inbox := nats.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return daemon.ShutdownACK{}, fmt.Errorf("failed to subscribe to inbox: %w", err)
+	}
+	defer sub.Unsubscribe()
+
+	if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
+		return daemon.ShutdownACK{}, fmt.Errorf("failed to publish request: %w", err)
+	}
+	nc.Flush()
+
+	deadline := time.Now().Add(timeout)
+	for {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
+		}
+		var ack daemon.ShutdownACK
+		if err := json.Unmarshal(msg.Data, &ack); err != nil {
+			continue
+		}
+		if ack.Node == node {
+			return ack, nil
+		}
+	}
+}
+
+// runClusterDrainDHCP asks every vpcd to DHCPRELEASE all external-pool leases
+// it currently holds, returning them to the upstream DHCP server. Intended for
+// the teardown path: an env reset otherwise strands held leases until TTL
+// because vpcd's Stop() preserves them for adopt-on-reboot. Best-effort — if
+// the cluster is already partly down it warns and exits 0 so teardown proceeds.
+func runClusterDrainDHCP(cmd *cobra.Command, args []string) {
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	if timeout <= 0 {
+		timeout = 30 * time.Second
+	}
+
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: drain-dhcp could not connect (cluster may be down): %v\n", err)
+		return
+	}
+	defer nc.Close()
+
+	total, responders := drainDHCPLeases(nc, timeout)
+	fmt.Printf("DHCP drain complete: released %d lease(s) from %d vpcd responder(s)\n", total, responders)
+}
+
+// drainDHCPLeases publishes a drain request to vpc.dhcp.drain and collects
+// replies until timeout — one responder per AZ (vpcd uses a per-AZ queue
+// group). Returns the summed released count and the number of responders.
+func drainDHCPLeases(nc *nats.Conn, timeout time.Duration) (released, responders int) {
+	reqData, err := json.Marshal(struct{}{})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		return 0, 0
+	}
+
+	inbox := nats.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: drain-dhcp subscribe failed: %v\n", err)
+		return 0, 0
+	}
+	defer sub.Unsubscribe()
+
+	if err := nc.PublishRequest(dhcp.TopicDrain, inbox, reqData); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: drain-dhcp publish failed: %v\n", err)
+		return 0, 0
+	}
+	nc.Flush()
+
+	deadline := time.Now().Add(timeout)
+	for {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		msg, err := sub.NextMsg(remaining)
+		if err != nil {
+			break
+		}
+		var reply struct {
+			Released int    `json:"released"`
+			Error    string `json:"error,omitempty"`
+		}
+		if err := json.Unmarshal(msg.Data, &reply); err != nil {
+			continue
+		}
+		responders++
+		if reply.Error != "" {
+			fmt.Fprintf(os.Stderr, "  vpcd drain error: %s\n", reply.Error)
+			continue
+		}
+		released += reply.Released
+	}
+	return released, responders
+}
+
 // collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
 func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
 	inbox := nats.NewInbox()
@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeDrainResponder subscribes a stand-in vpcd to vpc.dhcp.drain that
+// replies with the given released count (or error), mirroring
+// Manager.handleDrainMsg without a real lease store.
+func fakeDrainResponder(t *testing.T, nc *nats.Conn, released int, errMsg string) {
+	t.Helper()
+	sub, err := nc.Subscribe(dhcp.TopicDrain, func(msg *nats.Msg) {
+		body, _ := json.Marshal(struct {
+			Released int    `json:"released"`
+			Error    string `json:"error,omitempty"`
+		}{Released: released, Error: errMsg})
+		_ = msg.Respond(body)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func TestDrainDHCPLeasesSumsReleased(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	fakeDrainResponder(t, nc, 7, "")
+
+	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	assert.Equal(t, 7, released)
+	assert.Equal(t, 1, responders)
+}
+
+func TestDrainDHCPLeasesReportsResponderError(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	fakeDrainResponder(t, nc, 0, "list leases for drain: boom")
+
+	// An erroring responder still counts as a responder but contributes
+	// zero released leases.
+	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	assert.Equal(t, 0, released)
+	assert.Equal(t, 1, responders)
+}
+
+func TestDrainDHCPLeasesNoResponders(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	// No vpcd subscribed: the collection window elapses with zero replies.
+	released, responders := drainDHCPLeases(nc, 200*time.Millisecond)
+	assert.Equal(t, 0, released)
+	assert.Equal(t, 0, responders)
+}
@@ -19,8 +19,10 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -206,11 +208,7 @@ func runGetNodes(cmd *cobra.Command, args []string) {
 	for name := range respondedNodes {
 		nodeSet[name] = struct{}{}
 	}
-	nodeNames := make([]string, 0, len(nodeSet))
-	for name := range nodeSet {
-		nodeNames = append(nodeNames, name)
-	}
-	sort.Strings(nodeNames)
+	nodeNames := slices.Sorted(maps.Keys(nodeSet))

 	for _, name := range nodeNames {
 		nodeCfg := cfg.Nodes[name]
@@ -291,7 +289,7 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 	})

 	tableData := pterm.TableData{
-		{"INSTANCE", "STATUS", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
+		{"INSTANCE", "STATUS", "HEALTH", "CRASHES", "TYPE", "VCPU", "MEM", "NODE", "IP", "AGE"},
 	}

 	for _, v := range allVMs {
@@ -299,9 +297,15 @@ func runGetVMs(cmd *cobra.Command, args []string) {
 		if v.LaunchTime > 0 {
 			age = formatUptime(time.Now().Unix() - v.LaunchTime)
 		}
+		health := v.Health
+		if health == "" {
+			health = "-"
+		}
 		tableData = append(tableData, []string{
 			v.InstanceID,
 			v.Status,
+			health,
+			strconv.Itoa(v.CrashCount),
 			v.InstanceType,
 			strconv.Itoa(v.VCPU),
 			formatMemGB(v.MemoryGB),
@@ -1,5 +1,5 @@
 # AWS Gateway configuration file in TOML format.
-version = "2"
+version = "3"
 region = "{{.Region}}"

 # Enable verbose debugging
@@ -8,8 +8,8 @@ debug = false
 # API request throttling (per-account, per-action token bucket)
 [ratelimit]
 enabled = false
-rate = 20           # Default sustained requests per second per account per action
-burst = 100         # Default maximum burst capacity
+rate = 20 # Default sustained requests per second per account per action
+burst = 100 # Default maximum burst capacity

 [ratelimit.action.RunInstances]
 rate = 2
@@ -26,3 +26,12 @@ burst = 40
 [ratelimit.action.StopInstances]
 rate = 2
 burst = 40
+
+# Per-account service quotas (default off)
+[quota]
+enabled = false
+vcpus = 8
+vpcs = 8
+subnets = 16
+eips = 4
+volumes_gib = 100
@@ -75,3 +75,8 @@ nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -103,4 +103,9 @@ nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}
 nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
-access_keys_bucket = "spinifex-iam-access-keys"
+access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -8,6 +8,10 @@ node = "{{.Node}}"
 email = "{{.OperatorEmail}}"
 {{- end}}

+[aws]
+region = "{{.Region}}"
+internal_suffix = "spinifex.internal"
+
 [network]
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
@@ -93,6 +97,9 @@ token = "{{.NatsToken}}"

 [nodes.{{.Node}}.viperblock]
 shardwal = false
+{{- if .EncryptionKeyFile}}
+encryption_key_file = "{{.EncryptionKeyFile}}"
+{{- end}}

 [nodes.{{.Node}}.vpcd]
 ovn_nb_addr = "{{.OVNNBAddr}}"
@@ -19,8 +19,9 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"os"
-	"sort"
+	"slices"
 	"strconv"
 	"time"

@@ -81,11 +82,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {
 	for name := range respondedNodes {
 		nodeSet[name] = struct{}{}
 	}
-	nodeNames := make([]string, 0, len(nodeSet))
-	for name := range nodeSet {
-		nodeNames = append(nodeNames, name)
-	}
-	sort.Strings(nodeNames)
+	nodeNames := slices.Sorted(maps.Keys(nodeSet))

 	nodeTable := pterm.TableData{
 		{"NAME", "CPU (used/total)", "MEM (used/total)", "GPU (used/total)", "VMs"},
@@ -141,11 +138,7 @@ func runTopNodes(cmd *cobra.Command, args []string) {

 	fmt.Println()

-	capNames := make([]string, 0, len(capacityMap))
-	for name := range capacityMap {
-		capNames = append(capNames, name)
-	}
-	sort.Strings(capNames)
+	capNames := slices.Sorted(maps.Keys(capacityMap))

 	capTable := pterm.TableData{
 		{"INSTANCE TYPE", "AVAILABLE", "VCPU", "MEMORY"},
@@ -1,183 +0,0 @@
-# Spinifex Systemd Service
-
-This directory contains the systemd service configuration for running Spinifex services on Ubuntu and Debian systems.
-
-## Files
-
- `spinifex.service` - The systemd service file that manages all Spinifex services
- `install-spinifex-service.sh` - Installation script to set up the service
- `README.md` - This documentation file
-
-## Installation
-
-### Prerequisites
-
-1. Ensure Spinifex is installed in `/opt/spinifex/`
-2. Ensure the `spx` binary is available at `/opt/spinifex/bin/spx`
-3. Run the installation script as root
-
-### Setup Configuration
-
-```
-/opt/spinifex/bin/spx admin init
-```
-
-This will init the Spinifex environment and create the default configuration files in the home directory, e.g `~/spinifex/`.
-
-```
-/home/username/spinifex:
-amis  config  images  logs  nats  predastore  state  viperblock  volumes
-
-/home/username/spinifex/amis:
-
-/home/username/spinifex/config:
-awsgw  spinifex.toml  nats  predastore  server.key  server.pem
-
-/home/username/spinifex/config/awsgw:
-awsgw.toml
-
-/home/username/spinifex/config/nats:
-nats.conf
-
-/home/username/spinifex/config/predastore:
-predastore.toml
-
-/home/username/spinifex/images:
-
-/home/username/spinifex/logs:
-awsgw.log  awsgw.pid  spinifex.log  spinifex.pid  nats.log  nats.pid  predastore.log  predastore.pid  viperblock.log  viperblock.pid
-
-/home/username/spinifex/nats:
-
-/home/username/spinifex/predastore:
-data
-
-/home/username/spinifex/predastore/data:
-predastore
-
-/home/ben/spinifex/state:
-
-/home/ben/spinifex/viperblock:
-
-/home/ben/spinifex/volumes:
-```
-
-### Quick Installation
-
-```bash
-sudo ./config/install-spinifex-service.sh
-```
-
-### Manual Installation
-
-If you prefer to install manually:
-
-1. Copy the service file:
-   ```bash
-   sudo cp config/spinifex.service /etc/systemd/system/
-   ```
-
-2. Reload systemd:
-   ```bash
-   sudo systemctl daemon-reload
-   ```
-
-3. Enable the service:
-   ```bash
-   sudo systemctl enable spinifex.service
-   ```
-
-## Service Management
-
-### Start the service
-```bash
-sudo systemctl start spinifex
-```
-
-### Stop the service
-```bash
-sudo systemctl stop spinifex
-```
-
-### Check service status
-```bash
-sudo systemctl status spinifex
-```
-
-### Restart the service
-```bash
-sudo systemctl restart spinifex
-```
-
-### View service logs
-```bash
-sudo journalctl -u spinifex -f
-```
-
-### Disable auto-start
-```bash
-sudo systemctl disable spinifex
-```
-
-## Service Details
-
-The `spinifex.service` file:
-
- Starts services in the correct order: NATS → Predastore → Viperblock
- Includes proper delays between service starts to ensure dependencies are ready
- Stops services in reverse order during shutdown
- Runs as root (required for system services)
- Includes security hardening settings
- Provides proper logging to systemd journal
- Automatically restarts on failure
-
-## Service Order
-
-The service starts the following components in order:
-
-1. **NATS** - Message broker service
-2. **Predastore** - Data storage service (waits 3 seconds after NATS)
-3. **Viperblock** - Block storage service (waits 2 seconds after Predastore)
-
-## Troubleshooting
-
-### Check if the service is running
-```bash
-sudo systemctl is-active spinifex
-```
-
-### View recent logs
-```bash
-sudo journalctl -u spinifex --since "1 hour ago"
-```
-
-### Check service configuration
-```bash
-sudo systemctl cat spinifex
-```
-
-### Verify binary exists
-```bash
-ls -la /opt/spinifex/bin/spx
-```
-
-### Test service commands manually
-```bash
-/opt/spinifex/bin/spx service nats start
-/opt/spinifex/bin/spx service predastore start
-/opt/spinifex/bin/spx service viperblock start
-```
-
-## Customization
-
-If you need to modify the service configuration:
-
-1. Edit the service file: `sudo nano /etc/systemd/system/spinifex.service`
-2. Reload systemd: `sudo systemctl daemon-reload`
-3. Restart the service: `sudo systemctl restart spinifex`
-
-## Security Notes
-
- The service runs as root, which is necessary for system-level services
- Security hardening is enabled with `ProtectSystem=strict` and `NoNewPrivileges=true`
- Only specific paths are writable: `/opt/spinifex`, `/var/log`, `/var/run` 
@@ -1,74 +0,0 @@
-#!/bin/bash
-
-# Spinifex Service Installation Script
-# This script installs the spinifex systemd service on Ubuntu/Debian systems
-
-set -e
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Function to print colored output
-print_status() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-print_warning() {
-    echo -e "${YELLOW}[WARNING]${NC} $1"
-}
-
-print_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-# Check if running as root
-if [[ $EUID -ne 0 ]]; then
-   print_error "This script must be run as root (use sudo)"
-   exit 1
-fi
-
-# Check if systemd is available
-if ! command -v systemctl &> /dev/null; then
-    print_error "systemd is not available on this system"
-    exit 1
-fi
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SERVICE_FILE="$SCRIPT_DIR/spinifex.service"
-
-# Check if service file exists
-if [[ ! -f "$SERVICE_FILE" ]]; then
-    print_error "Service file not found at $SERVICE_FILE"
-    exit 1
-fi
-
-print_status "Installing Spinifex systemd service..."
-
-# Copy service file to systemd directory
-cp "$SERVICE_FILE" /etc/systemd/system/
-
-# Reload systemd to recognize the new service
-systemctl daemon-reload
-
-# Enable the service to start on boot
-systemctl enable spinifex.service
-
-print_status "Service installed successfully!"
-print_status "You can now use the following commands:"
-echo "  systemctl start spinifex    # Start the service"
-echo "  systemctl stop spinifex     # Stop the service"
-echo "  systemctl status spinifex   # Check service status"
-echo "  systemctl restart spinifex  # Restart the service"
-echo "  journalctl -u spinifex -f   # View service logs"
-
-# Check if spinifex binary exists
-if [[ ! -f "/opt/spinifex/bin/spinifex" ]]; then
-    print_warning "Spinifex binary not found at /opt/spinifex/bin/spinifex"
-    print_warning "Please ensure spinifex is installed in /opt/spinifex before starting the service"
-fi
-
-print_status "Installation complete!" 
@@ -1,80 +0,0 @@
-# Predastore S3-compatible storage configuration file in TOML format.
-# MULTI NODE used for testing
-version = "1.0"
-region = "ap-southeast-2"
-
-# Optionally configure port and host, or specify via daemon CLI options
-host = "0.0.0.0"
-port = 8443
-
-# Enable verbose debugging
-debug = false
-
-# Optionally configure base-path if full directory path not defined for "fs" buckets
-# base_path = "/home/predastore/data/"
-
-# Specify reed solomon encoding
-[rs]
-data = 2
-parity = 1
-
-[[db]]
-id = 1
-host = "127.0.0.1"
-port = 6660
-#raft_port = 7660       # Optional: Raft consensus port (default: port + 1000)
-#advertise_host = ""    # Optional: Address to advertise to other Raft nodes
-path = "distributed/db/node-1/"
-access_key_id = "X"
-secret_access_key = "X"
-leader = true      # Bootstrap leader hint
-
-[[db]]
-id = 2
-host = "127.0.0.1"
-port = 6661
-path = "distributed/db/node-2/"
-access_key_id = "X"
-secret_access_key = "X"
-
-[[db]]
-id = 3
-host = "127.0.0.1"
-port = 6662
-path = "distributed/db/node-3/"
-access_key_id = "X"
-secret_access_key = "X"
-
-[[nodes]]
-id = 1
-host = "0.0.0.0"
-port = 9991
-path = "distributed/nodes/node-1/"
-
-[[nodes]]
-id = 2
-host = "0.0.0.0"
-port = 9992
-path = "distributed/nodes/node-2/"
-
-[[nodes]]
-id = 3
-host = "0.0.0.0"
-port = 9993
-path = "distributed/nodes/node-3/"
-
-[[buckets]]
-name = "predastore"
-region = "ap-southeast-2"
-type = "distributed"
-public = false
-encryption = ""
-
-[[auth]]
-access_key_id = "X"
-secret_access_key = "X"
-policy = [
-  { bucket = "predastore", actions = ["s3:ListBucket",  "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
-]
-
-
@@ -1,49 +0,0 @@
-[Unit]
-Description=Spinifex Services
-Documentation=https://github.com/mulgadc/spinifex
-After=network.target
-Wants=network.target
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-User=root
-Group=root
-WorkingDirectory=/opt/spinifex
-Environment=PATH=/opt/spinifex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-# Start NATS service
-ExecStart=/opt/spinifex/bin/spinifex service nats start
-ExecStartPost=/bin/sleep 3
-
-# Start Predastore service
-ExecStart=/opt/spinifex/bin/spinifex service predastore start
-ExecStartPost=/bin/sleep 2
-
-# Start Viperblock service
-ExecStart=/opt/spinifex/bin/spinifex service viperblock start
-
-# Stop services in reverse order
-ExecStop=/opt/spinifex/bin/spinifex service viperblock stop
-ExecStopPost=/bin/sleep 2
-ExecStop=/opt/spinifex/bin/spinifex service predastore stop
-ExecStopPost=/bin/sleep 2
-ExecStop=/opt/spinifex/bin/spinifex service nats stop
-
-# Restart policy
-Restart=on-failure
-RestartSec=10
-
-# Security settings
-NoNewPrivileges=true
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths=/opt/spinifex /var/log /var/run
-
-# Logging
-StandardOutput=journal
-StandardError=journal
-SyslogIdentifier=spinifex
-
-[Install]
-WantedBy=multi-user.target 
@@ -65,6 +65,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Flags | Description |
 |---------|-------|-------------|
 | `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
+| `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
+| `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |

 ### Certificate Management

@@ -89,7 +91,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. |
+| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. On every import a best-effort `virt-customize` step bakes the deployment CA (`<data-root>/config/ca.pem`) into the image's trust store before the block copy, so in-guest SDK-over-TLS calls to Spinifex endpoints trust the gateway from first boot; an image libguestfs cannot customize is imported as-is (CA-free) and logs a skip. Because the CA is fixed into the image, rotating the cluster CA requires re-importing affected images. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
 | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |

@@ -112,7 +114,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | **DONE** |
+| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only), `--metadata-options` (HttpPutResponseHopLimit; IMDSv2-only enforced — rejects `http-tokens=optional`) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--launch-template`, `--hibernate-options` | **DONE** |
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | **DONE** |
@@ -122,6 +124,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination` | `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest`, `--dry-run` | **DONE** |
 | `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, disableApiTermination, instanceInitiatedShutdownBehavior, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | **DONE** |
+| `modify-instance-metadata-options` | `--instance-id`, `--http-put-response-hop-limit` (1–64), `--http-tokens` (`required`), `--http-endpoint` (`enabled`), `--http-protocol-ipv6`/`--instance-metadata-tags` (`disabled`) — secure values are no-ops, downgrades return `UnsupportedOperation` | `--dry-run` | **DONE** |
 | `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | **DONE** (stub — always returns `standard`) |
 | `describe-instance-status` | `--instance-ids`, `--include-all-instances`, `--filters` (availability-zone, instance-state-code, instance-state-name, tag:*) | `--max-results`, `--next-token`, `--dry-run`, event/instance-status/system-status filters | **DONE** (static health) |
 | `monitor-instances` | — | `--instance-ids` | **NOT STARTED** |
@@ -131,6 +134,18 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 role ARN and rejects cross-account profile ARNs. Placement strategies: `spread`
 (1 per node, atomic CAS) and `cluster` (pin all to one node).

+`run-instances --capacity-reservation-specification` consumes a reserved slot on
+the node owning the targeted reservation (targeted-by-id only; `open`
+auto-matching is not supported). The combination with `--placement` (group) is
+rejected. `describe-instances` then reports the instance's `CapacityReservationId`.
+
+`modify-instance-metadata-options` is the only metadata-options mutator — AWS has
+no `get-instance-metadata-options` API, so the per-instance block is read through
+`describe-instances`. Only the hop limit is mutable; `--http-tokens optional` and
+the other posture-weakening values are refused with `UnsupportedOperation`, since
+the platform enforces IMDSv2 permanently. `run-instances --metadata-options` runs
+the same validation at launch.
+
 ### EC2 — IAM Instance Profile Associations

 Stored as `vm.VM.IamInstanceProfileArn` + `IamInstanceProfileAssociationId`
@@ -183,7 +198,7 @@ input but treat it as `uefi` at launch.
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (auto-assigns `/dev/sd[f-p]`) | `--dry-run` | **DONE** |
 | `detach-volume` | `--volume-id`, `--instance-id` (optional), `--device`, `--force` | `--dry-run` | **DONE** |
 | `describe-volume-status` | `--volume-ids`, `--filters` (volume-id, volume-status.status, availability-zone) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
-| `describe-volumes-modifications` | — | `--volume-ids`, `--filters`, `--max-results` | **NOT STARTED** |
+| `describe-volumes-modifications` | `--volume-ids`, `--filters` (modification-state, original-iops, original-size, original-volume-type, start-time, target-iops, target-size, target-volume-type, volume-id) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |

 ### EC2 — Snapshots

@@ -233,6 +248,11 @@ by downstream services.
 | `modify-instance-metadata-defaults` | — | `--http-tokens`, `--http-put-response-hop-limit`, `--http-endpoint`, `--instance-metadata-tags` | **NOT STARTED** |
 | `get-instance-metadata-defaults` | — | `--dry-run` | **NOT STARTED** |

+The account-level `modify-/get-instance-metadata-defaults` are blocked on an
+`aws-sdk-go` bump: the `InstanceMetadataDefaultsResponse` type is absent from the
+pinned v1.44.266 (these APIs landed ~March 2024), and the gateway's generic
+handler needs the typed input struct to route them.
+
 ### EC2 — VPC Core

 | Command | Implemented Flags | Missing Flags | Status |
@@ -432,9 +452,9 @@ EIP commands are only registered when an external IPAM pool is configured.
 | `delete-dhcp-options` | — | `--dhcp-options-id`, `--dry-run` | **NOT STARTED** |
 | `describe-dhcp-options` | — | `--dhcp-options-ids`, `--filters`, `--max-results` | **NOT STARTED** |
 | `associate-dhcp-options` | — | `--dhcp-options-id`, `--vpc-id`, `--dry-run` | **NOT STARTED** |
-| `create-capacity-reservation` | — | `--instance-type`, `--instance-platform`, `--availability-zone`, `--instance-count`, `--end-date`, `--end-date-type`, `--instance-match-criteria`, `--tag-specifications` | **NOT STARTED** |
-| `cancel-capacity-reservation` | — | `--capacity-reservation-id`, `--dry-run` | **NOT STARTED** |
-| `describe-capacity-reservations` | — | `--capacity-reservation-ids`, `--filters`, `--max-results` | **NOT STARTED** |
+| `create-capacity-reservation` | `--instance-type`, `--instance-count`, `--availability-zone`, `--instance-platform`, `--instance-match-criteria`, `--tenancy` (default only), `--dry-run` | `--end-date`, `--end-date-type` (unlimited only), `--availability-zone-id`, `--tag-specifications` | **DONE** |
+| `cancel-capacity-reservation` | `--capacity-reservation-id`, `--dry-run` | — | **DONE** |
+| `describe-capacity-reservations` | `--capacity-reservation-ids`, `--filters` | `--max-results`, `--next-token` | **DONE** |
 | `modify-capacity-reservation` | — | `--capacity-reservation-id`, `--instance-count`, `--end-date`, `--end-date-type` | **NOT STARTED** |

 ### EC2 — Misc
@@ -488,6 +508,7 @@ bypasses policy evaluation entirely.
 | `create-policy` | `--policy-name`, `--policy-document`, `--path`, `--description` | `--tags` | **DONE** |
 | `get-policy` | `--policy-arn` | — | **DONE** |
 | `get-policy-version` | `--policy-arn`, `--version-id` | — | **DONE** |
+| `list-policy-versions` | `--policy-arn` | `--max-items`, `--marker` | **DONE** |
 | `list-policies` | — | `--scope`, `--only-attached`, `--path-prefix`, `--max-items`, `--marker` | **DONE** |
 | `delete-policy` | `--policy-arn` | — | **DONE** |
 | `attach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
@@ -507,6 +528,10 @@ bypasses policy evaluation entirely.
 | `attach-role-policy` | `--role-name`, `--policy-arn` | — | **DONE** |
 | `detach-role-policy` | `--role-name`, `--policy-arn` | — | **DONE** |
 | `list-attached-role-policies` | `--role-name`, `--path-prefix` | `--max-items`, `--marker` | **DONE** |
+| `list-role-policies` | `--role-name` | `--max-items`, `--marker` | **DONE** |
+| `put-role-policy` | `--role-name`, `--policy-name`, `--policy-document` | — | **DONE** |
+| `get-role-policy` | `--role-name`, `--policy-name` | — | **DONE** (document returned as raw JSON, not URL-encoded) |
+| `delete-role-policy` | `--role-name`, `--policy-name` | — | **DONE** |

 ### IAM — Instance Profiles

@@ -520,6 +545,19 @@ bypasses policy evaluation entirely.
 | `add-role-to-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |
 | `remove-role-from-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |

+### IAM — OIDC Providers
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `create-open-id-connect-provider` | `--url`, `--client-id-list`, `--thumbprint-list`, `--tags` | — | **DONE** |
+| `get-open-id-connect-provider` | `--open-id-connect-provider-arn` | — | **DONE** |
+| `list-open-id-connect-providers` | — | — | **DONE** |
+| `delete-open-id-connect-provider` | `--open-id-connect-provider-arn` | — | **DONE** |
+| `add-client-id-to-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
+| `remove-client-id-from-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
+| `update-open-id-connect-provider-thumbprint` | — | `--open-id-connect-provider-arn`, `--thumbprint-list` | **NOT STARTED** |
+| `tag-open-id-connect-provider` / `untag-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--tags`/`--tag-keys` | **NOT STARTED** |
+
 ### IAM — Groups

 | Command | Implemented Flags | Missing Flags | Status |
@@ -544,16 +582,13 @@ bypasses policy evaluation entirely.
 | `get-caller-identity` | — | — | **DONE** |
 | `assume-role` | `--role-arn`, `--role-session-name`, `--duration-seconds` (900–min(role MaxSessionDuration, 43200)) | `--policy`, `--policy-arns` (→ `PackedPolicyTooLarge`); `--tags`, `--transitive-tag-keys` (→ `InvalidParameterValue`); `--serial-number`, `--token-code` (→ `InvalidParameterValue`); `--external-id`, `--source-identity` (accepted and logged, **not enforced** — no Condition evaluator in v1) | **DONE** |
 | `get-session-token` | `--duration-seconds` (900–129600, default 43200 = 12h; clamped, not rejected) | `--serial-number`, `--token-code` (MFA → `InvalidParameterValue`) | **DONE** |
-| `assume-role-with-web-identity` | — | `--role-arn`, `--role-session-name`, `--web-identity-token`, `--provider-id`, `--policy`, `--policy-arns`, `--duration-seconds` | **NOT STARTED** |
+| `assume-role-with-web-identity` | `--role-arn`, `--role-session-name`, `--web-identity-token`, `--duration-seconds` (900–43200, default 3600) | `--provider-id`; `--policy`, `--policy-arns` (→ `PackedPolicyTooLarge`) | **DONE** |
 | `assume-role-with-saml` | — | `--role-arn`, `--principal-arn`, `--saml-assertion`, `--policy`, `--policy-arns`, `--duration-seconds` | **NOT STARTED** |
 | `get-access-key-info` | — | `--access-key-id` | **NOT STARTED** |
 | `get-federation-token` | — | `--name`, `--policy`, `--policy-arns`, `--duration-seconds`, `--tags` | **NOT STARTED** |
 | `decode-authorization-message` | — | `--encoded-message` | **NOT STARTED** |

-Trust policies stored on roles (`AssumeRolePolicyDocument`) reject `Condition`,
-`NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty
-`Principal` blocks at write time (`MalformedPolicyDocument`) — v1 has no
-Condition evaluator so accepting them would silently allow.
+Trust policies stored on roles (`AssumeRolePolicyDocument`) reject `Condition`, `NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty `Principal` blocks at write time (`MalformedPolicyDocument`) — v1 has no Condition evaluator so accepting them would silently allow.

 ---

@@ -563,6 +598,8 @@ Available at `169.254.169.254` from inside every running guest VM, matching AWS.

 **IMDSv2-only.** Every read requires a session token. A tokenless (v1-style) `GET` returns `401 Unauthorized` with an empty body. Obtain a token with a `PUT /latest/api/token` carrying `X-aws-ec2-metadata-token-ttl-seconds` (1–21600), then send it back in `X-aws-ec2-metadata-token` on every read.

+The EC2 control plane reports this posture faithfully: `describe-instances` returns `MetadataOptions.HttpTokens=required`, and `run-instances`/`modify-instance-metadata-options` reject `--http-tokens optional` (and `--http-endpoint disabled`) with `UnsupportedOperation` — exactly as AWS does under account-level IMDSv2 enforcement.
+
 ```bash
 # Inside the guest VM:
 TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
@@ -579,24 +616,42 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | Path | Method | Source | Status |
 |------|--------|--------|--------|
 | `/latest/api/token` | PUT | Issues an ENI-bound IMDSv2 token; `X-aws-ec2-metadata-token-ttl-seconds` ∈ [1, 21600] required | **DONE** |
+| `/` | GET | Supported API-version list (`2021-07-15`, `latest`); token-gated | **DONE** |
+| `/latest` | GET | Top-level tree listing (`dynamic`, `meta-data`, `user-data`) | **DONE** |
+| `/<date>/...` | GET/PUT | Any dated API version aliases to `/latest` (cloud-init parity) | **DONE** |
 | `/latest/meta-data/` | GET | Directory listing of supported children | **DONE** |
 | `/latest/meta-data/instance-id` | GET | `vm.ID` | **DONE** |
 | `/latest/meta-data/instance-type` | GET | `vm.InstanceType` | **DONE** |
 | `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
+| `/latest/meta-data/ami-launch-index` | GET | Per-instance launch index (`0..n-1`), contiguous on partial failure | **DONE** |
+| `/latest/meta-data/reservation-id` | GET | `DescribeInstances` `Reservation.ReservationId` | **DONE** |
+| `/latest/meta-data/instance-life-cycle` | GET | `on-demand` (Spot not modelled) | **DONE** |
 | `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
 | `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
+| `/latest/meta-data/public-hostname` | GET | Mirrors `public-ipv4`; 404 when no public IP | **DONE** |
 | `/latest/meta-data/mac` | GET | `ENIRecord.MacAddress` | **DONE** |
 | `/latest/meta-data/security-groups` | GET | `ENIRecord.SecurityGroupIds`, newline-separated | **DONE** |
 | `/latest/meta-data/hostname`, `/local-hostname` | GET | Synthesised `ip-<dashed-ip>.<region>.compute.internal` | **DONE** |
 | `/latest/meta-data/placement/availability-zone` | GET | `ENIRecord.AvailabilityZone` | **DONE** |
 | `/latest/meta-data/placement/region` | GET | Derived from AZ (trailing letter stripped) | **DONE** |
+| `/latest/meta-data/services/{domain,partition}` | GET | Static: `amazonaws.com` / `aws` | **DONE** |
 | `/latest/meta-data/iam/info` | GET | `{InstanceProfileArn, InstanceProfileId}`; 404 if no profile | **DONE** |
 | `/latest/meta-data/iam/security-credentials/` | GET | Role name(s) under the profile, one per line; empty body if none | **DONE** |
 | `/latest/meta-data/iam/security-credentials/<role>` | GET | STS `AssumeRoleForInstance` → ASIA-prefixed temporary credential JSON | **DONE** |
+| `/latest/meta-data/public-keys/` | GET | `0=<keyName>` from the launch key pair; 404 if none | **DONE** |
+| `/latest/meta-data/public-keys/0/` | GET | `openssh-key` (format list for index 0) | **DONE** |
+| `/latest/meta-data/public-keys/0/openssh-key` | GET | Launch SSH public key, live-fetched from the key store; 404 if the key was deleted, 500 on backend fault | **DONE** |
 | `/latest/user-data` | GET | `vm.UserData`; 404 if none | **DONE** |
-| `/latest/dynamic/instance-identity/document` | GET | Needs a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
-| `/latest/meta-data/network/interfaces/...` | GET | Multi-ENI rendering | **NOT STARTED** (404) |
-| `/latest/meta-data/tags/...` | GET | Instance-tag metadata | **NOT STARTED** (404) |
+| `/latest/dynamic` | GET | Lists `instance-identity/` | **DONE** |
+| `/latest/dynamic/instance-identity` | GET | Lists `document` (signed forms listed when the signing key lands) | **DONE** |
+| `/latest/dynamic/instance-identity/document` | GET | Unsigned identity document from resolved ENI + instance facts | **DONE** |
+| `/latest/dynamic/instance-identity/{signature,pkcs7,rsa2048}` | GET | Signed forms; need a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
+| `/latest/meta-data/network/interfaces/macs/<mac>/...` | GET | Primary ENI subtree: `mac`, `device-number`, `interface-id`, `owner-id`, `subnet-id`, `vpc-id`, `local-ipv4s`, `local-hostname`, `security-group-ids`, `security-groups`, `subnet-ipv4-cidr-block`, `vpc-ipv4-cidr-block(s)`, and `public-ipv4s`/`public-hostname` when an EIP is attached | **DONE** (single-NIC; multi-ENI deferred) |
+| `/latest/meta-data/tags/instance/<key>` | GET | Instance-tag metadata; gated on `InstanceMetadataTags` enablement | **NOT STARTED** (404) |
+| `/latest/meta-data/block-device-mapping/...` | GET | `ami`/`root`/`ebsN`/`ephemeralN` device map | **NOT STARTED** (404) |
+| `/latest/meta-data/placement/{group-name,partition-number,availability-zone-id,host-id}` | GET | Placement extras beyond `availability-zone`/`region` | **NOT STARTED** (404) |
+| `/latest/meta-data/instance-action` | GET | `none` unless interruptible instances ship | **NOT STARTED** (404) |
+| `/latest/meta-data/spot/{instance-action,termination-time}` | GET | Only meaningful once Spot is modelled | **NOT STARTED** (404) |

 ---

@@ -632,10 +687,54 @@ The data plane uses a system-managed LB VM, launched automatically during `creat

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-listener` | `--load-balancer-arn`, `--default-actions` (Type=forward, TargetGroupArn), `--protocol` (HTTP), `--port` | `--ssl-policy`, `--certificates`, `--alpn-policy`, `--mutual-authentication`, `--dry-run` | **DONE** |
+| `create-listener` | `--load-balancer-arn`, `--default-actions` (Type=forward, TargetGroupArn), `--protocol` (HTTP/HTTPS/TLS), `--port`, `--certificates` (HTTPS/TLS), `--ssl-policy` | `--alpn-policy`, `--mutual-authentication`, `--dry-run` | **DONE** |
+| `modify-listener` | `--listener-arn`, `--protocol`, `--port`, `--default-actions`, `--certificates`, `--ssl-policy` | `--alpn-policy`, `--mutual-authentication`, `--dry-run` | **DONE** |
 | `delete-listener` | `--listener-arn` | `--dry-run` | **DONE** |
 | `describe-listeners` | `--load-balancer-arn`, `--listener-arns` | `--page-size`, `--marker`, `--dry-run` | **DONE** |

+### ELBv2 — Listener Rules
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `create-rule` | `--listener-arn`, `--priority` (1–50000), `--conditions` (host-header, path-pattern, http-header, http-request-method, query-string, source-ip), `--actions` (forward, redirect, fixed-response), `--tags` | `--dry-run` | **DONE** |
+| `modify-rule` | `--rule-arn`, `--conditions`, `--actions` | `--dry-run` | **DONE** |
+| `delete-rule` | `--rule-arn` | `--dry-run` | **DONE** |
+| `describe-rules` | `--listener-arn`, `--rule-arns` | `--page-size`, `--marker` (parsed, not enforced) | **DONE** |
+| `set-rule-priorities` | `--rule-priorities` (RuleArn, Priority) | — | **DONE** |
+
+A synthetic `default` rule is derived from the listener's `DefaultActions`.
+
+### ELBv2 — Listener Certificates & SSL Policies
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `add-listener-certificates` | `--listener-arn`, `--certificates` | `--dry-run` | **DONE** |
+| `remove-listener-certificates` | `--listener-arn`, `--certificates` | `--dry-run` | **DONE** |
+| `describe-listener-certificates` | `--listener-arn` | `--page-size`, `--marker` | **DONE** |
+| `describe-ssl-policies` | `--names` | `--load-balancer-type`, `--page-size`, `--marker` | **DONE** (static catalog: `ELBSecurityPolicy-FS-1-2-Res-2019-08`, `ELBSecurityPolicy-TLS13-1-2-2021-06` — metadata only, no in-platform TLS termination) |
+
+The default certificate cannot be added/removed via these calls — set it on the listener.
+
+### ELBv2 — Attributes & Modify
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `describe-load-balancer-attributes` | `--load-balancer-arn` | — | **DONE** (stored values over per-type defaults) |
+| `modify-load-balancer-attributes` | `--load-balancer-arn`, `--attributes` | — | **DONE** (unknown keys → `ValidationError`) |
+| `describe-target-group-attributes` | `--target-group-arn` | — | **DONE** |
+| `modify-target-group-attributes` | `--target-group-arn`, `--attributes` (incl. `deregistration_delay.timeout_seconds`, `stickiness.*`) | — | **DONE** |
+| `modify-target-group` | `--target-group-arn`, `--health-check-*`, `--matcher` | `--target-type`/`--protocol`/`--vpc-id` (immutable) | **DONE** |
+| `describe-listener-attributes` | `--listener-arn` | — | **DONE** (stub — returns empty; not persisted) |
+| `modify-listener-attributes` | `--listener-arn`, `--attributes` | — | **DONE** (stub — echoes input; not persisted) |
+
+### ELBv2 — Network & Security
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `set-security-groups` | `--load-balancer-arn`, `--security-groups` | `--enforce-security-group-inbound-rules-on-private-link-traffic` | **DONE** (ALB only — NLB → `InvalidConfigurationRequest`) |
+| `set-subnets` | `--load-balancer-arn`, `--subnets`, `--subnet-mappings` | `--ip-address-type` | **DONE** (live ENI add/remove with rollback) |
+| `set-ip-address-type` | `--load-balancer-arn`, `--ip-address-type` (ipv4) | dualstack/IPv6 (rejected) | **DONE** |
+
 ### ELBv2 — Tags

 | Command | Implemented Flags | Missing Flags | Status |
@@ -646,18 +745,32 @@ The data plane uses a system-managed LB VM, launched automatically during `creat

 | Feature | Priority | Status |
 |---------|----------|--------|
-| Listener rules (CreateRule, DeleteRule, DescribeRules, ModifyRule) | High | **NOT STARTED** |
-| HTTPS/TLS termination (SSL certificates, policies, ALPN) | High | **NOT STARTED** |
-| Modify operations (`ModifyLoadBalancerAttributes`, `ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `ModifyListener`) | Medium | **NOT STARTED** |
-| Connection draining (deregistration delay) | Medium | **NOT STARTED** |
-| Stickiness / session affinity | Medium | **NOT STARTED** |
-| Active health checking (API-driven, vs. HAProxy-only today) | Medium | **NOT STARTED** |
+| In-platform HTTPS/TLS termination (cert + SSL-policy APIs exist; data-plane TLS not terminated) | High | **NOT STARTED** |
+| ALPN policy, mutual TLS (mTLS) | Medium | **NOT STARTED** |
+| Listener attribute persistence (`Describe/ModifyListenerAttributes` are stubs) | Medium | **NOT STARTED** |
+| Active health checking (API-driven, vs. HAProxy/nginx-only today) | Medium | **NOT STARTED** |
 | IP and Lambda target types | Low | **NOT STARTED** |
 | S3 access log delivery | Low | **NOT STARTED** |
 | WAF integration | Low | **NOT STARTED** |

 ---

+## ACM (AWS Certificate Manager)
+
+Import-only — Spinifex stores externally-issued certificates for ELBv2 listener references; it does not issue certificates or validate domains (`RequestCertificate`). Certs are account-scoped; `describe`/`delete` enforce ownership.
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `import-certificate` | `--certificate`, `--private-key`, `--certificate-chain`, `--certificate-arn` (re-import) | `--tags` | **DONE** |
+| `describe-certificate` | `--certificate-arn` | — | **DONE** |
+| `list-certificates` | — | `--certificate-statuses`, `--includes`, `--max-items`, `--next-token` | **DONE** |
+| `delete-certificate` | `--certificate-arn` | — | **DONE** |
+| `request-certificate` | — | `--domain-name`, `--validation-method`, `--subject-alternative-names`, `--tags` | **NOT STARTED** |
+| `add-tags-to-certificate` / `list-tags-for-certificate` / `remove-tags-from-certificate` | — | `--certificate-arn`, `--tags`/`--tag-keys` | **NOT STARTED** |
+| `export-certificate` | — | `--certificate-arn`, `--passphrase` | **NOT STARTED** |
+
+---
+
 ## CloudWatch (Basic Monitoring)

 | Command | Implemented Flags | Missing Flags | Status |
@@ -123,7 +123,7 @@ Daemons (`spinifex/daemon/daemon.go`) subscribe to NATS topics and handle reques
 When a daemon handles `RunInstances`:

 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot, cloud-init, and EFI volumes via Viperblock
+2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
 3. **Volume Mount**: Sends `ebs.mount` request to Viperblock, receives NBD URI
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
@@ -217,7 +217,6 @@ type EBSRequest struct {
    VolType   string  // gp2, io1, etc.
    Boot      bool    // Boot volume flag
    EFI       bool    // EFI boot volume
-    CloudInit bool    // Cloud-init volume
    NBDURI    string  // NBD URI returned from mount
 }
 ```
@@ -95,13 +95,6 @@ checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster
 and `eks create-nodegroup` resolve the boot AMI by that tag, so no further
 configuration is needed.

-Operators publishing a freshly built node image to the mirror run the
-**Publish EKS node AMI** GitHub Actions workflow (manual dispatch; uses the
-`CF_R2_*` repo secrets) or, locally, `make publish-eks-node-image` (see
-`scripts/publish-system-image.sh`; requires `R2_ENDPOINT` + R2-scoped
-`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`).
-
-
 ## Cluster Shutdown

 Coordinated, phased shutdown of the entire cluster (API/UI → VMs → viperblock → predastore → NATS/daemon):
@@ -137,7 +137,7 @@ In development mode, find the QEMU port forward and connect via localhost:

 ```bash
 ps auxw | grep $INSTANCE_ID
-ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p <port>
+ssh -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 -p <port>
 ```

 For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
@@ -0,0 +1,188 @@
+---
+title: "ECR (Container Registry)"
+description: "Store and serve container images from Spinifex's AWS-compatible Elastic Container Registry — create a repository, authenticate Docker, push and pull images, and let your EKS workers pull from it, using the AWS CLI, the Spinifex console, or Terraform."
+category: "Containers"
+tags:
+  - ecr
+  - containers
+  - docker
+  - registry
+  - oci
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Terraform AWS Provider — ECR"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository"
+  - title: "OCI Distribution Spec"
+    url: "https://github.com/opencontainers/distribution-spec"
+  - title: "Docker CLI Reference"
+    url: "https://docs.docker.com/reference/cli/docker/"
+---
+
+# ECR (Container Registry)
+
+> Spinifex's ECR service is an AWS-compatible, OCI-compliant container registry. You create repositories with `aws ecr`, authenticate Docker with a registry token, and `docker push` / `docker pull` exactly as you would on AWS — and your EKS clusters can pull from it directly.
+
+## Overview
+
+ECR on Spinifex serves a private container registry on the Spinifex gateway endpoint — the same `host:9999` you already use for the AWS API. Authentication is account-scoped by token, so you push and pull at the gateway host and never need extra DNS:
+
+```
+<spinifex-gateway-host>:9999/<repository>
+```
+
+`aws ecr describe-repositories` and `aws ecr get-login-password` return the exact host to use — always prefer that value. Where real DNS is configured, the AWS-parity per-account host `<account-id>.dkr.ecr.<region>.<your-spinifex-domain>:9999` also works.
+
+Each repository holds image manifests and their layers. Images are stored in object storage ([Predastore](https://github.com/mulgadc/predastore)) in a per-account bucket, with blobs content-addressed and de-duplicated across repositories. Authentication is token-based: `aws ecr get-login-password` mints a short-lived bearer token that `docker login` uses against the registry's `/v2/` endpoint.
+
+**What works today**
+
+- Repository lifecycle — `CreateRepository`, `DeleteRepository`, `DescribeRepositories`.
+- OCI push and pull — blobs, manifests, and tags, including cross-repository blob mounts.
+- Token auth — `GetAuthorizationToken` plus bearer/basic auth on `/v2/`.
+- Repository policies — set, get, and delete a repository policy document.
+
+**Current limitations**
+
+- **Image scanning is not supported** — scan APIs return an explicit "operation not supported" error.
+- **No automatic garbage collection yet** — deleting images does not yet reclaim underlying blobs.
+- Lifecycle-policy enforcement and tag-immutability enforcement are not active.
+
+## Prerequisites
+
+- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)).
+- **Docker** installed locally to build, push, and pull images.
+- **Docker trusting the Spinifex CA.** The registry serves TLS signed by the Spinifex local CA. Install it into the host trust store once (Docker uses the host root pool for registry TLS, so this covers every account endpoint without per-host config):
+
+```bash
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-local-ca.crt
+sudo update-ca-certificates
+sudo systemctl restart docker
+```
+
+- **The registry endpoint** — the Spinifex gateway `host:9999`. `aws ecr get-login-password` and `aws ecr describe-repositories` return the exact `host:port`; copy it from there rather than constructing it by hand.
+- **For EKS pulls:** worker nodes need the `AmazonEC2ContainerRegistryReadOnly` policy on their node IAM role (the EKS prerequisites already include this) and network egress to the registry endpoint.
+
+## Instructions
+
+Create a repository and push an image using your preferred tool.
+
+:::tabs
+@tab AWS CLI
+
+### 1. Create a repository
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws ecr create-repository --repository-name my-app
+aws ecr describe-repositories --repository-names my-app \
+  --query 'repositories[0].repositoryUri'
+```
+
+The `repositoryUri` is the value you tag and push to — the gateway `host:9999`
+with the repository path, port included (for example `192.0.2.10:9999/my-app`).
+Use it verbatim; don't hand-build a hostname.
+
+### 2. Authenticate Docker
+
+Take the registry host straight from the API so the `:9999` port is correct:
+
+```bash
+REGISTRY=$(aws ecr describe-repositories --repository-names my-app \
+  --query 'repositories[0].repositoryUri' --output text | cut -d/ -f1)
+
+aws ecr get-login-password --region <region> \
+  | docker login --username AWS --password-stdin "$REGISTRY"
+```
+
+### 3. Build, tag, and push
+
+```bash
+docker build -t my-app:latest .
+docker tag my-app:latest "$REGISTRY/my-app:latest"
+docker push "$REGISTRY/my-app:latest"
+```
+
+### 4. Verify and pull
+
+```bash
+aws ecr describe-images --repository-name my-app
+docker pull "$REGISTRY/my-app:latest"
+```
+
+@tab Spinifex UI
+
+From the left navigation open **ECR → Repositories**.
+
+<img src="ecr-list-repositories.png" alt="The ECR Repositories list in the Spinifex console, showing a repository with its registry URI, tag mutability, and creation date">
+
+### 1. Create a repository
+
+1. Click **Create Repository**.
+2. Enter the **repository name** and, optionally, set **tag mutability** (mutable or immutable).
+3. Submit — the repository appears in the list with its full registry URI.
+
+### 2. Copy the push commands
+
+Open the repository to see its detail page. The **Push commands** panel lists the exact `aws ecr get-login-password | docker login`, `docker build`, `docker tag`, and `docker push` commands, pre-filled with your registry host and repository URI — use the **Copy** button and run them locally.
+
+<img src="ecr-push-commands.png" alt="A repository detail page in the Spinifex console, with the Push commands panel showing the docker login, build, tag, and push commands">
+
+### 3. Manage the repository
+
+- The **Permissions** tab edits the repository's access policy.
+- The **Lifecycle** tab manages retention rules, and **Tag immutability** can be toggled from the detail page.
+- The **Scan** tab reflects that image scanning is not supported on Spinifex.
+- Delete a repository from the **Delete** action in the repositories list.
+
+@tab Terraform
+
+Repositories can be managed as code with the standard `aws_ecr_repository` resource, with the AWS provider pointed at Spinifex's `ecr` endpoint:
+
+```hcl
+provider "aws" {
+  region                      = "ap-southeast-2"
+  skip_credentials_validation = true
+  skip_requesting_account_id  = true
+
+  endpoints {
+    ecr = "https://<your-spinifex-gateway>"
+  }
+}
+
+resource "aws_ecr_repository" "my_app" {
+  name = "my-app"
+}
+
+output "repository_url" {
+  value = aws_ecr_repository.my_app.repository_url
+}
+```
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+Then authenticate Docker and push using the `repository_url` output, following steps 2–4 in the AWS CLI tab.
+
+:::
+
+## Troubleshooting
+
+**`docker login` fails with an authorization error.** The token is account-scoped and short-lived. Re-run `aws ecr get-login-password` to mint a fresh token, and confirm the registry host in `docker login` matches your account's endpoint (`aws ecr describe-repositories` returns it).
+
+**`docker push` cannot reach the registry.** Use the exact host from `aws ecr describe-repositories` — the Spinifex gateway `host:9999`. Docker dials it directly, so the `:9999` port must be present (without it docker tries `:443` and fails). If you are using the AWS-parity `<account-id>.dkr.ecr…` hostname instead, that path needs DNS resolving it to the gateway — prefer the gateway host the API returns.
+
+**EKS workers cannot pull an image.** Confirm the node IAM role has `AmazonEC2ContainerRegistryReadOnly` and that the workers have egress to the registry endpoint (an Internet Gateway or NAT Gateway route). See the [EKS prerequisites](/docs/eks).
+
+**Image-scanning commands return an error.** Image scanning is not supported on Spinifex; the scan APIs intentionally reject these calls. Remove scan-on-push configuration from your tooling.
+
+**A pushed image still appears after deletion frees no space.** Garbage collection of unreferenced blobs is not yet automatic — deleting an image removes its manifest and tags but does not immediately reclaim the underlying layers.
@@ -0,0 +1,270 @@
+---
+title: "ECS (Elastic Container Service)"
+description: "Run AWS-compatible ECS workloads on Spinifex — create a cluster, register a task definition, boot container instances from the ECS node image, run tasks, and front a long-running service with an Application Load Balancer, all via the AWS CLI, the Spinifex console, or Terraform."
+category: "Containers"
+tags:
+  - ecs
+  - containers
+  - tasks
+  - alb
+  - iam
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
+resources:
+  - title: "ECS Quickstart Workbook"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/ecs-quickstart"
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "AWS ECS Task IAM Roles"
+    url: "https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html"
+---
+
+# ECS (Elastic Container Service)
+
+> Spinifex's ECS service gives you an AWS-compatible container orchestrator: you register task definitions and run them — one-off as tasks, or kept-running as a service behind a load balancer — on container instances you launch from the Spinifex ECS node image. Standard tools — `aws ecs`, the Terraform AWS provider — work unchanged.
+
+## Overview
+
+ECS on Spinifex follows the AWS **EC2 launch type**: you supply the compute. A cluster is a logical grouping; the capacity behind it is **container instances** — ordinary EC2 instances booted from Spinifex's `spinifex-ecs-node` image, each running the Spinifex ECS agent. The agent registers the instance with the cluster, reports its CPU/memory, and runs the containers the scheduler places on it.
+
+There is **no ECS-specific launch API** — this is AWS-faithful. You add capacity by launching EC2 instances from the ECS node image (with the `ecsInstanceRole` instance profile), exactly as on AWS. The Spinifex console wraps this in a one-click **Provision capacity** action, but underneath it is just `RunInstances`.
+
+A **task definition** describes one or more containers (image, CPU/memory, ports, environment, and an optional task IAM role). A **task** is a running instantiation of a task definition; the scheduler bin-packs tasks onto instances with free capacity. A **service** keeps a desired number of tasks running, replaces failed ones, and — when configured — registers each task's IP with an Application Load Balancer target group.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/ecs-topology.svg" alt="ECS on Spinifex — an Application Load Balancer fronts awsvpc tasks running on container instances booted from the spinifex-ecs-node AMI; the ECS agent registers with the Spinifex gateway over TLS and SigV4 and serves task-role credentials to containers" width="900">
+</p>
+
+**What you'll create:**
+
+| Resource | Purpose |
+|---|---|
+| VPC + subnets | The network the cluster and tasks run in |
+| Internet Gateway | Egress so instances can pull container images |
+| `ecsInstanceRole` | Instance profile letting the agent reach the control plane |
+| ECS cluster | The logical grouping tasks and instances join |
+| Task definition | The container spec (image, CPU/memory, ports, task role) |
+| Container instance(s) | EC2 VMs from the ECS node image that run tasks |
+| Service + ALB target group | Keeps N tasks running and load-balances them |
+
+**Spinifex specifics**
+
+- **EC2 launch type only.** There is no Fargate-equivalent serverless capacity; you run and pay for the container instances. `RequiresCompatibilities` of `FARGATE` is not honoured.
+- **`awsvpc` network mode.** Each task gets its own ENI and private IP in your subnet. Target groups must use `target_type = "ip"`; the service's `network_configuration` selects the subnets and security groups.
+- **The container-instance image is fixed.** Instances must boot Spinifex's `spinifex-ecs-node` image (resolve it by the `spinifex:managed-by=ecs` tag) and carry the `ecsInstanceRole` instance profile.
+- **Task IAM roles via the credential endpoint.** A task with a `taskRoleArn` gets `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` injected; the agent serves short-lived credentials for that role at `169.254.170.2`, AWS-faithful — your container's SDK picks them up with no static keys.
+
+### Limitations
+
+ECS v1 is deliberately minimal. Keep these in mind:
+
+- **No service discovery.** `serviceRegistries` / Cloud Map integration is not implemented — reach a service through its load balancer, not a DNS name.
+- **`awslogs` log driver is discarded.** Container stdout/stderr is not shipped to CloudWatch Logs; the log configuration is accepted but dropped.
+- **No rolling deployments.** The reconciler keeps the desired count running but ignores `deploymentConfiguration` (`minimumHealthyPercent` / `maximumPercent`); an `UpdateService` to a new revision replaces tasks without a health-gated rollout or circuit-breaker.
+- **No capacity providers or managed scaling.** Capacity is the static total of your registered instances; there is no scale-out/in or ASG binding — you manage the instance count.
+- **No distinct execution role.** ECR pulls use the instance role directly; `executionRoleArn` is not honoured separately.
+- **`secrets[]` are silently dropped**, and tags set via `TagResource` are not persisted. Health-check settings beyond the target group default are not forwarded.
+
+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — `spinifex-ecs-node` image required.**
+>
+> Container instances **must** boot Spinifex's **`spinifex-ecs-node`** image — it carries the ECS agent. Import it before provisioning capacity (during `spx admin init` or via the image catalogue); the console's provision-capacity action and the Terraform workbook both resolve it by tag.
+>
+> **Verify before continuing:**
+>
+> ```bash
+> aws ec2 describe-images \
+>   --filters 'Name=tag:spinifex:managed-by,Values=ecs' \
+>   --query 'Images[].[ImageId,Name]' --output text
+> ```
+>
+> No rows means the image is not imported — register it before continuing.
+
+The Terraform workbook builds all of this for you. For the CLI or console paths, have it in place first.
+
+- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)).
+- **The `spinifex-ecs-node` image** imported. Confirm it resolves:
+
+  ```bash
+  aws ec2 describe-images \
+    --filters 'Name=tag:spinifex:managed-by,Values=ecs' \
+    --query 'Images[].[ImageId,Name]' --output text
+  ```
+
+- **A VPC with at least one subnet** and an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`), so instances can pull container images.
+- **The `ecsInstanceRole` instance profile.** It is account-global: a role trusted by `ec2.amazonaws.com` with an `ecs:*` policy, exposed through an instance profile of the same name. The console's provision-capacity action creates it on first use; the Terraform workbook creates it unless you opt out.
+- **An optional task IAM role** trusted by `ecs-tasks.amazonaws.com`, if your containers call AWS APIs.
+
+## Instructions
+
+The same workload can be created three ways. Pick your tool — each path reaches the same running service.
+
+:::tabs
+@tab AWS CLI
+
+### 1. Create the cluster
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws ecs create-cluster --cluster-name demo
+```
+
+### 2. Register a task definition
+
+`awsvpc` network mode, EC2 launch type, one nginx container on port 80:
+
+```bash
+aws ecs register-task-definition \
+  --family web \
+  --network-mode awsvpc \
+  --requires-compatibilities EC2 \
+  --cpu 256 --memory 512 \
+  --container-definitions '[{
+    "name":"web",
+    "image":"docker.io/library/nginx:1.27-alpine",
+    "portMappings":[{"containerPort":80,"protocol":"tcp"}],
+    "essential":true
+  }]'
+```
+
+### 3. Add capacity
+
+Launch one or more EC2 instances from the ECS node image with the `ecsInstanceRole`
+instance profile and a cloud-init that points the agent at the cluster. The
+[ECS Quickstart workbook](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/ecs-quickstart)
+produces this user-data for you; the console **Provision capacity** action is the
+one-click equivalent. Once an instance boots and its agent registers, it appears here:
+
+```bash
+aws ecs list-container-instances --cluster demo
+```
+
+### 4. Run a task or create a service
+
+Run a one-off task:
+
+```bash
+aws ecs run-task \
+  --cluster demo --task-definition web --count 1 \
+  --network-configuration 'awsvpcConfiguration={subnets=[subnet-aaaa]}'
+```
+
+Or keep it running behind an ALB target group (`target_type = ip`):
+
+```bash
+aws ecs create-service \
+  --cluster demo --service-name web --task-definition web \
+  --desired-count 2 \
+  --network-configuration 'awsvpcConfiguration={subnets=[subnet-aaaa]}' \
+  --load-balancers 'targetGroupArn=<tg-arn>,containerName=web,containerPort=80'
+
+aws ecs describe-services --cluster demo --services web \
+  --query 'services[0].[runningCount,desiredCount]'
+```
+
+@tab Spinifex UI
+
+The Spinifex console drives the same workflow. From the left navigation open **ECS → Clusters**.
+
+### 1. Create the cluster
+
+Click **Create Cluster**, give it a name, and submit. It appears in the **Clusters** list.
+
+### 2. Provision capacity
+
+Open the cluster and use **Provision capacity** on the **Infrastructure** (container instances) tab. The console launches instances from the `spinifex-ecs-node` image, attaches the `ecsInstanceRole` instance profile (creating it if needed), and injects the gateway URL and CA into the instance's cloud-init — so the agent registers without any manual key handling. The instances appear under **Infrastructure** as they register.
+
+### 3. Register a task definition and run it
+
+- The **Task definitions** view lists families and revisions; register a new revision with the container image, CPU/memory, and ports.
+- From a task definition, **Run task** for a one-off, or create a **Service** to keep a desired count running.
+- A cluster's **Services** and **Tasks** tabs show what is running; open a task or service for its detail page (containers, networking, and public/private endpoints).
+
+### 4. Attach a load balancer
+
+When creating a service, select an ALB target group (`target_type = ip`) so each task's ENI is registered and traffic is balanced across tasks.
+
+@tab Terraform
+
+The [ECS Quickstart workbook](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/ecs-quickstart) builds the whole stack in one `apply`: VPC, IAM, cluster, task definition, container instances, and an ALB-fronted service. It is the Terraform-native equivalent of the console's provision-capacity action.
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/ecs-quickstart
+```
+
+The core resources point the AWS provider at Spinifex's `ecs`, `ec2`, `iam`, `sts`, and `elasticloadbalancingv2` endpoints, then create the cluster, an `awsvpc` task definition with a task role, and a service wired to a target group:
+
+```hcl
+resource "aws_ecs_cluster" "main" {
+  name = "ecs-quickstart"
+}
+
+resource "aws_ecs_task_definition" "web" {
+  family                   = "ecs-quickstart-web"
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["EC2"]
+  cpu                      = "256"
+  memory                   = "512"
+  task_role_arn            = aws_iam_role.task.arn
+
+  container_definitions = jsonencode([{
+    name         = "web"
+    image        = "docker.io/library/nginx:1.27-alpine"
+    essential    = true
+    portMappings = [{ containerPort = 80, protocol = "tcp" }]
+  }])
+}
+
+resource "aws_ecs_service" "web" {
+  name            = "ecs-quickstart-web"
+  cluster         = aws_ecs_cluster.main.id
+  task_definition = aws_ecs_task_definition.web.arn
+  desired_count   = 1
+
+  network_configuration {
+    subnets         = [aws_subnet.a.id, aws_subnet.b.id]
+    security_groups = [aws_security_group.node.id]
+  }
+
+  load_balancer {
+    target_group_arn = aws_lb_target_group.web.arn
+    container_name   = "web"
+    container_port   = 80
+  }
+}
+```
+
+Because a container instance reaches the control plane over the gateway (TLS + SigV4) rather than a managed AWS endpoint, the workbook makes the two things the console injects explicit: a **LAN-reachable** `gateway_url` (the host's bridge IP, not `127.0.0.1`) and the gateway CA, both baked into the instance's cloud-init. Apply it:
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply -var 'gateway_url=https://<host-lan-ip>:9999'
+```
+
+`ecsInstanceRole` is account-global — if it already exists (from the console), pass `-var 'create_instance_role=false'`. See the workbook README for the full variable list and teardown.
+
+:::
+
+## Troubleshooting
+
+**No container instances appear after launching them.** The agent registers over the gateway, not a managed endpoint, so check its cloud-init injected a correct **LAN-reachable** gateway URL (not `127.0.0.1` — a guest VM cannot reach the host loopback) and the gateway CA. `cloud-init write_files` runs once per instance, so fixing the user-data needs an instance **replacement**, not an in-place modify. Confirm the instance carries the `ecsInstanceRole` instance profile.
+
+**Tasks stay `PENDING`.** No instance has free capacity for the task's CPU/memory reservation. Add capacity, or lower the task definition's reservations. `aws ecs list-container-instances` should show at least one `ACTIVE` instance.
+
+**Service `runningCount` below `desiredCount`.** Either there is not enough capacity (see above), or tasks are failing to start — check the task's containers can pull their image (instances need an egress route) and that the image reference is valid.
+
+**Load balancer target unhealthy / app unreachable.** The target group must be `target_type = ip` (tasks register their ENI IP, not an instance ID). The ALB DNS name ends in `.elb.spinifex.local` and does not resolve from outside; fetch its public IP with `aws elbv2 describe-load-balancers` and curl that. Note health-check settings beyond the target group default are not forwarded.
+
+**Container cannot assume its task role.** Confirm the task definition sets `taskRoleArn` and the role is trusted by `ecs-tasks.amazonaws.com`. The agent injects `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`; a container that overrides this variable, or an SDK too old to honour it, will not pick up the credentials.
+
+**Expected a feature that is missing.** ECS v1 omits service discovery, `awslogs` shipping, rolling deployments, capacity providers, and a distinct execution role. See the Limitations in the Overview.
@@ -0,0 +1,267 @@
+---
+title: "EKS (Managed Kubernetes)"
+description: "Run a managed, AWS-compatible Kubernetes cluster on Spinifex — provision the control plane and worker node group, wire up the VPC, IAM, and security groups it needs, and deploy your first workload via the AWS CLI, the Spinifex console, or Terraform."
+category: "Containers"
+tags:
+  - eks
+  - kubernetes
+  - containers
+  - vpc
+  - iam
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
+resources:
+  - title: "EKS Quickstart Workbook"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-quickstart"
+  - title: "EKS HTTPS Ingress Workbook (LBC + ACM)"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-https-ingress"
+  - title: "EKS GitOps Workbook (Argo CD + EBS-CSI)"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-gitops-argocd"
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "kubectl Documentation"
+    url: "https://kubernetes.io/docs/reference/kubectl/"
+---
+
+# EKS (Managed Kubernetes)
+
+> Spinifex's EKS service gives you a managed, AWS-compatible Kubernetes API: you create a cluster and a node group, and Spinifex provisions and runs the control plane and workers for you. Standard tools — `aws eks`, `kubectl`, the Terraform AWS provider — work unchanged against your cluster.
+
+## Overview
+
+An EKS cluster on Spinifex has two parts: a **control plane** (the managed Kubernetes API server, provisioned on a Spinifex-managed VM and fronted by a network load balancer) and one or more **managed node groups** of worker VMs that run your pods. You provide the VPC, subnets, and IAM roles; Spinifex creates and reconciles the cluster, its load balancer, and its security groups.
+
+Authentication uses **EKS access entries** (the API authentication mode) — IAM principals are mapped to Kubernetes RBAC groups, and `kubectl` authenticates with `aws eks get-token` exactly as it does on AWS.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/eks-topology.svg" alt="EKS on Spinifex — a managed control plane (public API NLB plus apiserver VM) fronts a managed node group of eks-node worker VMs running pods across two subnets, pulling images from ECR" width="900">
+</p>
+
+**What you'll create:**
+
+| Resource | Purpose |
+|---|---|
+| VPC + subnets | The network the cluster and workers run in |
+| Internet Gateway / NAT Gateway | Egress so workers can pull container images |
+| Cluster IAM role | Lets the control plane manage cluster resources |
+| Node IAM role | Lets workers join the cluster and pull images |
+| EKS cluster | The managed Kubernetes API endpoint |
+| Managed node group | The worker VMs that run your pods |
+| Security-group rule | Opens your app's port to reach the workers |
+
+**Spinifex specifics**
+
+- **Auth mode is API-only.** `authentication_mode` must be `API`; the legacy `CONFIG_MAP` and `API_AND_CONFIG_MAP` modes are rejected. Grant access with access entries, not an `aws-auth` ConfigMap.
+- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
+- **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
+- **Default Kubernetes version is `1.32`.**
+
+## Prerequisites
+
+> [!IMPORTANT]
+> **Prerequisite — `eks-node` image required.**
+>
+> Node groups always boot Spinifex's **`eks-node`** image, and it **must** be registered before you create a cluster — the console blocks cluster creation until it is present. Import it during `spx admin init` (or via the image catalogue) ahead of time.
+>
+> **Verify before continuing:**
+>
+> ```bash
+> aws ec2 describe-images \
+>   --filters 'Name=tag:spinifex:managed-by,Values=eks' \
+>   --query 'Images[].[ImageId,Name]' --output text
+> ```
+>
+> No rows means the image is not imported — register it before continuing.
+
+Before creating a cluster you need the supporting infrastructure in place. The Terraform workbooks build all of this for you; if you are using the CLI or console, create it first.
+
+- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and `kubectl` installed.
+- **The `eks-node` image** registered on the cluster. The console blocks cluster creation until it is present.
+- **A VPC with at least one subnet.** The control plane launches in your first subnet; workers run in the subnets you pass to the node group.
+- **Egress for the workers.** Workers must reach the internet to pull container images:
+  - **Public workers** — an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
+  - **Private workers** — a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
+- **Two IAM roles:**
+  - A **cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
+  - A **node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
+
+## Instructions
+
+The same cluster can be created three ways. Pick your tool — each path reaches the same working cluster.
+
+:::tabs
+@tab AWS CLI
+
+### 1. Create the cluster
+
+Point the cluster at your VPC subnets and the cluster role, with API authentication and a public endpoint:
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws eks create-cluster \
+  --name demo \
+  --role-arn arn:aws:iam::000000000000:role/eks-cluster-role \
+  --resources-vpc-config subnetIds=subnet-aaaa,subnet-bbbb,endpointPublicAccess=true \
+  --access-config authenticationMode=API \
+  --kubernetes-version 1.32
+```
+
+Wait for it to become active:
+
+```bash
+aws eks wait cluster-active --name demo
+aws eks describe-cluster --name demo --query 'cluster.status'
+```
+
+### 2. Add a node group
+
+```bash
+aws eks create-nodegroup \
+  --cluster-name demo \
+  --nodegroup-name default \
+  --node-role-arn arn:aws:iam::000000000000:role/eks-node-role \
+  --subnets subnet-aaaa subnet-bbbb \
+  --scaling-config minSize=1,maxSize=2,desiredSize=1 \
+  --instance-types t3.medium
+
+aws eks wait nodegroup-active --cluster-name demo --nodegroup-name default
+```
+
+### 3. Connect kubectl
+
+```bash
+aws eks update-kubeconfig --name demo
+kubectl get nodes
+```
+
+### 4. Expose a workload
+
+Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
+
+```bash
+SG=$(aws ec2 describe-security-groups \
+  --filters Name=group-name,Values=eks-cluster-demo-nodegroup-sg \
+  --query 'SecurityGroups[0].GroupId' --output text)
+
+aws ec2 authorize-security-group-ingress \
+  --group-id "$SG" --protocol tcp --port 30080 --cidr 0.0.0.0/0
+```
+
+Then deploy and publish your app:
+
+```bash
+kubectl create deployment hello --image=nginxdemos/hello --replicas=2
+kubectl expose deployment hello --type=NodePort --port=80 \
+  --overrides='{"spec":{"ports":[{"port":80,"nodePort":30080}]}}'
+```
+
+@tab Spinifex UI
+
+The Spinifex console drives the same workflow through a guided form. From the left navigation open **EKS → Clusters**.
+
+<img src="eks-list-clusters.png" alt="The EKS Clusters list in the Spinifex console, with a Create Cluster button">
+
+### 1. Create the cluster
+
+1. Click **Create Cluster**.
+2. Enter a **name** and choose the **Kubernetes version** (default `1.32`).
+3. Choose the **cluster IAM role**, or create one inline from the dialog.
+4. Select the **VPC and subnets** the cluster will run in.
+5. Set **endpoint access** (public, private, or both) and submit.
+
+<img src="eks-create-cluster.png" alt="The Create Cluster form in the Spinifex console, showing name, Kubernetes version, IAM role, VPC, subnets, and endpoint access fields">
+
+The form notes that the authentication mode is API and that the legacy aws-auth ConfigMap is not supported. The cluster appears in the **Clusters** list as `CREATING` and flips to `ACTIVE` once the control plane is healthy.
+
+### 2. Add a node group
+
+1. Open the cluster and switch to the **Node groups** tab.
+2. Click **Add node group**, choose the **node IAM role**, **subnets**, **instance type**, and **scaling** (desired / min / max).
+3. Submit and wait for the node group to reach `ACTIVE`.
+
+### 3. Manage access and addons
+
+- The **Access** tab lists IAM principals and their cluster-access policies — add an access entry to let another user or role reach the cluster.
+- The **Addons** tab installs and shows the health of managed addons.
+- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed security groups.
+
+To run `kubectl` against the cluster, follow the AWS CLI tab's `update-kubeconfig` step.
+
+@tab Terraform
+
+The Terraform workbooks are the fastest way to a working cluster — they build the VPC, IAM roles, cluster, node group, and a demo workload in one `apply`. Start with **eks-quickstart**:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/eks-quickstart
+```
+
+The cluster config points the AWS provider at Spinifex's `eks`, `ec2`, `iam`, and `sts` endpoints, then creates the cluster and node group:
+
+```hcl
+resource "aws_eks_cluster" "this" {
+  name     = "eks-quickstart"
+  role_arn = aws_iam_role.cluster.arn
+  version  = "1.32"
+
+  access_config {
+    authentication_mode = "API"
+  }
+
+  vpc_config {
+    subnet_ids              = [aws_subnet.a.id, aws_subnet.b.id]
+    endpoint_public_access  = true
+  }
+}
+
+resource "aws_eks_node_group" "default" {
+  cluster_name    = aws_eks_cluster.this.name
+  node_group_name = "default"
+  node_role_arn   = aws_iam_role.node.arn
+  subnet_ids      = [aws_subnet.a.id, aws_subnet.b.id]
+
+  scaling_config {
+    desired_size = 1
+    min_size     = 1
+    max_size     = 2
+  }
+
+  instance_types = ["t3.medium"]
+}
+```
+
+Apply it, then connect:
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+aws eks update-kubeconfig --name eks-quickstart
+kubectl get nodes
+```
+
+The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-ingress** adds private subnets, a NAT gateway, and HTTPS via the AWS Load Balancer Controller + ACM; **eks-gitops-argocd** delivers the app from a git repo with the Argo CD addon and persists state on an EBS-CSI (Viperblock-backed) volume. See the full templates linked in Resources.
+
+:::
+
+## Troubleshooting
+
+**Cluster stuck in `CREATING`.** The control plane is still booting. Check `aws eks describe-cluster --name <name> --query 'cluster.health'` for reported issues. The control plane needs egress to bootstrap, so confirm its subnet has an Internet Gateway route.
+
+**`kubectl` cannot connect.** Re-run `aws eks update-kubeconfig --name <name>` to refresh the endpoint and CA, and confirm the cluster is `ACTIVE`. `aws sts get-caller-identity` verifies your credentials reach the Spinifex endpoint.
+
+**Workers stay `NotReady`.** Almost always an egress or IAM problem:
+- Confirm the workers can pull images — a default route to an Internet Gateway (public) or NAT Gateway (private).
+- Confirm the node role has `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly`.
+
+**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
+
+**`create-cluster` rejected with an authentication-mode error.** `authentication_mode` must be `API`. Remove any `CONFIG_MAP` / `API_AND_CONFIG_MAP` setting and grant access with access entries instead.
+
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
@@ -243,7 +243,7 @@ SSH directly to the instance's public IP:
 PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID \
  --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)

-ssh -i ~/.ssh/spinifex-key ec2-user@$PUBLIC_IP
+ssh -i ~/.ssh/spinifex-key ubuntu@$PUBLIC_IP
 ```

 > **Note:** cloud-init takes 30-60 seconds to configure the instance. If SSH is refused, wait and retry.
--- a/Show More
+++ b/Show More