release: update go.mod for v1.9.0

feat: predastore compaction E2E (#407 )
* feat: add predastore compaction E2E Inject a short compaction interval into test-cluster predastore config via a new --predastore-compaction-interval flag on admin init/join, gated so production output stays byte-identical when unset. Add a harness helper that drives write-then-delete churn against predastore S3, brackets it with du-over-SSH usage, and asserts end-of-churn usage settles below 70% of the churn peak; wire it into the multinode suite as TestPredastoreCompaction. * fix: compaction E2E gate writes churn to its own bucket The reclaim gate signed PutObject against the config-defined system 'predastore' bucket, which belongs to account 000000000000 with only a static [[auth]] grant. The harness admin identity is a different IAM account, so predastore returned 403 AccessDenied cross-account. Create a per-run bucket the admin identity owns (AdministratorAccess) and churn into it. Objects still erasure-code into distributed/nodes/ node-* so the du-based reclaim measurement is unchanged. The gateway (:9999) does not proxy S3, so the client stays predastore-direct. * test: run predastore compaction gate before destructive node tests The compaction reclaim gate was declared last in the multinode suite, so it measured shard-store usage after NodeFailure/NodeRecovery had torn down and restarted node2. NodeRecovery only gates on NATS/gateway/EC2 readiness, not predastore, so node2's shard store was still settling and the du baseline came back empty. Move the test before the destructive node tests so it runs against a pristine, fully-converged cluster primed by the volume tests. * fix: race-free SSH output capture + empty-tolerant du probe The compaction reclaim gate crashed with ParseInt("") when node1's du probe returned empty combined output. Harden the probe and the SSH layer: - ssh.Run used one bytes.Buffer for both session.Stdout and session.Stderr, which x/crypto/ssh copies in two concurrent goroutines — a data race on a non-concurrent-safe type. Switch to session.CombinedOutput, which guards the combined stream with a mutex-wrapped singleWriter. - shardStoreUsageBytes now terminates the remote command with a sentinel echo. A missing sentinel is reported as output lost in SSH retrieval; a present sentinel with no number is treated as a 0 measurement (anomalous, logged) instead of crashing the gate. The sentinel lets the next run distinguish a transport-layer drop from a genuinely empty remote measurement. * test(e2e): replace predastore du-reclaim gate with object lifecycle contract The multinode compaction gate measured shard-store disk usage over SSH (du) and asserted reclaim after churn. It could never pass: segments only seal at the 4 GiB roll threshold, so test-scale churn produces no compactable segment, and the du probe read 0 bytes before and after. Reclaim correctness is already covered deterministically by predastore's store-level unit tests. Delete AssertPredastoreCompaction and its du/churn/poll/sentinel machinery, the harness unit tests that exercised them, and the multinode TestPredastore Compaction entry. Add a single-node TestPredastoreObjectLifecycle that checks the user-visible contract: an object is readable after PutObject, gone after DeleteObject (GET returns NoSuchKey and ListObjectsV2 is empty), and the store keeps serving fresh writes. newPredastoreS3 is retained, refactored to take a host so it works outside the multinode cluster type. * test(e2e): drop orphaned multinode TestPredastoreCompaction Complete the du-reclaim -> object-lifecycle refactor: remove the multinode TestPredastoreCompaction caller (and its now-unused context and harness imports) that still referenced the deleted harness.AssertPredastoreCompaction, which broke e2e-build. Wire the single-node TestPredastoreObjectLifecycle entry that replaces it.
2026-10-24 13:14:23 +00:00 · 2026-06-23 16:22:08 +10:00 · 2026-06-23 16:19:10 +10:00 · 2026-06-23 15:29:14 +10:00 · 2026-06-23 09:59:54 +10:00 · 2026-06-23 09:18:59 +10:00
489 changed files with 60330 additions and 4528 deletions
@@ -0,0 +1,7 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Changes
+
+<!-- The notable changes, as bullets. -->
@@ -35,7 +35,7 @@ jobs:
          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          ref: ${{ inputs.ref || github.ref }}

@@ -46,7 +46,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -35,12 +35,12 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Spinifex
        with:
          path: spinifex

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        name: Checkout Mulga (tofu-cluster)
        with:
          repository: mulgadc/mulga
@@ -74,7 +74,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          repository: mulgadc/mulga
@@ -82,7 +82,7 @@ jobs:
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          path: spinifex
@@ -3,7 +3,7 @@
 # Restructures the looped e2e.yml into env-grouped job blocks so each suite
 # surfaces as its own node in the Actions graph (mulga-siv-283):
 #
-#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks)
+#   single_provision ─┬─ single_suite (matrix: single, cert, lb, eks, iam)
 #                     └─ single_teardown (if: always)
 #   multi_provision  ─┬─ multi_suite  (matrix: multinode)
 #                     └─ multi_teardown (if: always)
@@ -27,7 +27,7 @@ on:
      single_suites:
        description: "Single-node suites to run as matrix legs"
        required: false
-        default: "single cert lb eks"
+        default: "single cert lb eks iam"
      run_multinode:
        description: "Also run the multinode env block"
        type: boolean
@@ -92,14 +92,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -114,7 +114,7 @@ jobs:
      - name: Compute suite matrix
        id: suites
        run: |
-          LIST="${{ inputs.single_suites || 'single cert lb eks' }}"
+          LIST="${{ inputs.single_suites || 'single cert lb eks iam' }}"
          echo "list=${LIST}" >> "$GITHUB_OUTPUT"
          JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
          echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
@@ -277,7 +277,7 @@ jobs:
      SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
      ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -386,7 +386,7 @@ jobs:
      # Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
      # Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
      # again would wipe the local tofu state and orphan the VM.
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -505,14 +505,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -627,7 +627,7 @@ jobs:
      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -732,7 +732,7 @@ jobs:
      SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
      NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -14,13 +14,10 @@ on:
      - .github/workflows/e2e.yml
  workflow_dispatch:
    inputs:
-      sha:
-        description: "PR head SHA to report status on"
-        required: false
      suites:
-        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
+        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
        required: false
-        default: "e2e-single e2e-multinode e2e-cert e2e-lb"
+        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
      test_run:
        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
        required: false
@@ -32,7 +29,7 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}

 jobs:
@@ -44,7 +41,7 @@ jobs:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v6
@@ -78,9 +75,9 @@ jobs:
    # suites='e2e-multinode' only). Push triggers and the default SUITES list
    # both include single/cert/lb so the inputs.suites null-check keeps the
    # job running on plain PR pushes.
-    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-iam')
    runs-on: [self-hosted, ci-single]
-    timeout-minutes: 45
+    timeout-minutes: 50
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -110,6 +107,10 @@ jobs:
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -118,14 +119,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -466,14 +467,14 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

-  e2e_real_multi:
-    name: Real Multi-Node E2E
+  e2e_multi:
+    name: Multi-Node E2E
    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
@@ -506,6 +507,10 @@ jobs:
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
+          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
+          # stale suite-status.txt / analysis dirs can't leak into this attempt.
+          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
@@ -514,14 +519,14 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          path: spinifex

@@ -593,12 +598,15 @@ jobs:
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # single + iam are single-node-only suites — their fixtures skip
+          # unless SPINIFEX_MODE=single, so building/running them on multinode
+          # is wasted work.
          # eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
          # stream and create-cluster needs an elected leader, but multinode
          # JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
          # validated single-node only for now; re-enable here once clustered
          # JetStream + multinode EKS are a deliberate target.
-          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
+          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
@@ -681,7 +689,7 @@ jobs:
        id: e2e
        run: |
          set +e
-          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
+          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -876,7 +884,7 @@ jobs:
      - name: Fail job if any suite failed
        if: always()
        run: |
-          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
@@ -885,28 +893,24 @@ jobs:
  e2e_status:
    name: Report E2E Status
    if: always()
-    needs: [e2e_single, e2e_real_multi]
+    needs: [e2e_single, e2e_multi]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
        env:
          GH_TOKEN: ${{ github.token }}
-          REPORT_SHA: ${{ inputs.sha || github.sha }}
+          REPORT_SHA: ${{ github.sha }}
        run: |
-          if [[ -z "${REPORT_SHA}" ]]; then
-            echo "::error::no SHA available (inputs.sha and github.sha both empty)"
-            exit 1
-          fi
          single_ok=true; multi_ok=true
          case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
-          case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
+          case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
          if $single_ok && $multi_ok; then
            STATE="success"
            DESC="E2E suites passed"
          else
            STATE="failure"
-            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
            -f state="$STATE" \
@@ -30,19 +30,19 @@ jobs:

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -9,21 +9,21 @@ permissions:

 jobs:
  distro-amd64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -54,21 +54,21 @@ jobs:
            spinifex/scripts/setup.sh

  distro-arm64:
-    runs-on: ubuntu-24.04-arm
+    runs-on: ubuntu-26.04-arm
    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock
@@ -91,7 +91,7 @@ jobs:

  release:
    needs: [ distro-amd64, distro-arm64 ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    steps:
      - name: Download amd64 Artifacts
        uses: actions/download-artifact@v8
@@ -143,7 +143,7 @@ jobs:
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -190,7 +190,7 @@ jobs:

  promote:
    needs: [ verify, baremetal ]
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -211,7 +211,7 @@ jobs:
  mark-failed:
    needs: [ verify, baremetal ]
    if: failure()
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    env:
      TAG: ${{ github.ref_name }}
      REPO: ${{ github.repository }}
@@ -251,25 +251,25 @@ jobs:
          repositories: mulga

      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          ref: ${{ github.ref }}

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          path: predastore

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          path: viperblock

      - name: Checkout Mulga iso-builder
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
@@ -23,24 +23,24 @@ permissions:
 jobs:
  build_and_test:
    name: Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex
          fetch-depth: 2

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -72,23 +72,23 @@ jobs:

  race_detection:
    name: Race Detection
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -108,23 +108,23 @@ jobs:

  lint_and_security:
    name: Lint and Security
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04

    steps:
      - name: Checkout Spinifex
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          path: spinifex

      - name: Checkout Viperblock
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/viperblock
          ref: dev
          path: viperblock

      - name: Checkout Predastore
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          repository: mulgadc/predastore
          ref: dev
@@ -23,14 +23,14 @@ permissions:
 jobs:
  ui_build_and_test:
    name: UI Build and Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-26.04
    defaults:
      run:
        working-directory: spinifex/services/spinifexui/frontend

    steps:
      - name: Checkout Code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
        with:
          sparse-checkout: spinifex/services/spinifexui/frontend

@@ -37,13 +37,3 @@ Thumbs.db

 # Test reports
 *-report.txt
-
-# Internal docs (moved to private mulga repo)
-docs/INTERNAL.md
-docs/PRIVATE.md
-docs/DEVNOTES.md
-docs/PLAN.md
-docs/TODO.md
-docs/development/
-docs/internal/
-scripts/iac/aws/spinifex-test.pem
@@ -0,0 +1,368 @@
+// ecr-credential-provider is a kubelet exec credential provider that lets a
+// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
+//
+// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
+// reading a CredentialProviderResponse on STDOUT. The provider fetches the
+// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
+// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
+// for the requested registry host.
+//
+// Static node config (gateway URL, CA, region) is read from the cloud-init env
+// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/mulgadc/spinifex/internal/tlsconfig"
+
+	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+)
+
+const (
+	credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
+	defaultCacheDuration   = "10h"
+	// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
+	// before the JWT actually expires mid-run.
+	expiryMargin = 30 * time.Minute
+
+	defaultEnvFile   = "/etc/spinifex-eks/agent.env"
+	defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
+	defaultIMDSBase  = "http://169.254.169.254/latest"
+	imdsTokenTTL     = "21600"
+)
+
+// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
+type CredentialProviderRequest struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+	Image      string `json:"image"`
+}
+
+// AuthConfig is a single registry credential entry.
+type AuthConfig struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
+type CredentialProviderResponse struct {
+	APIVersion    string                `json:"apiVersion"`
+	Kind          string                `json:"kind"`
+	CacheKeyType  string                `json:"cacheKeyType"`
+	CacheDuration string                `json:"cacheDuration"`
+	Auth          map[string]AuthConfig `json:"auth"`
+}
+
+// config holds the static node settings the provider needs.
+type config struct {
+	GatewayURL string
+	GatewayCA  string
+	Region     string
+	IMDSBase   string
+}
+
+func main() {
+	cfg := loadConfig(defaultEnvFile)
+
+	httpClient, err := gatewayHTTPClient(cfg.GatewayCA)
+	if err != nil {
+		slog.Error("ecr-credential-provider: build gateway client", "err", err)
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+
+	if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
+		slog.Error("ecr-credential-provider: run failed", "err", err)
+		// Still emit a valid empty-auth response so kubelet has a parseable reply.
+		emitEmpty(os.Stdout)
+		os.Exit(1)
+	}
+}
+
+// run is the testable core: it reads a CredentialProviderRequest from in,
+// resolves credentials via IMDS + the gateway, and writes a
+// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
+func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
+	var req CredentialProviderRequest
+	if err := json.NewDecoder(in).Decode(&req); err != nil {
+		return fmt.Errorf("decode request: %w", err)
+	}
+	if req.Image == "" {
+		return fmt.Errorf("request image is empty")
+	}
+	imageHost := hostFromImage(req.Image)
+
+	creds, err := fetchIMDSCredentials(httpClient, cfg.IMDSBase)
+	if err != nil {
+		return fmt.Errorf("fetch IMDS credentials: %w", err)
+	}
+
+	username, password, proxyHost, expiresAt, err := getAuthToken(cfg, httpClient, creds)
+	if err != nil {
+		return fmt.Errorf("get authorization token: %w", err)
+	}
+
+	auth := AuthConfig{Username: username, Password: password}
+	authMap := map[string]AuthConfig{imageHost: auth}
+	if proxyHost != "" && proxyHost != imageHost {
+		authMap[proxyHost] = auth
+	}
+
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: cacheDurationFrom(expiresAt),
+		Auth:          authMap,
+	}
+	return json.NewEncoder(out).Encode(&resp)
+}
+
+// imdsCreds mirrors the IMDS security-credentials JSON shape.
+type imdsCreds struct {
+	Code            string `json:"Code"`
+	AccessKeyID     string `json:"AccessKeyId"`
+	SecretAccessKey string `json:"SecretAccessKey"`
+	Token           string `json:"Token"`
+	Expiration      string `json:"Expiration"`
+}
+
+// fetchIMDSCredentials resolves the node role name then its credentials via
+// IMDSv2 (PUT token, then GET with the token header).
+func fetchIMDSCredentials(client *http.Client, base string) (imdsCreds, error) {
+	base = strings.TrimRight(base, "/")
+	token := imdsV2Token(client, base)
+
+	roleName, err := imdsGet(client, base+"/meta-data/iam/security-credentials/", token)
+	if err != nil {
+		return imdsCreds{}, fmt.Errorf("fetch role name: %w", err)
+	}
+	roleName = strings.TrimSpace(roleName)
+	if roleName == "" {
+		return imdsCreds{}, fmt.Errorf("IMDS returned empty role name")
+	}
+
+	body, err := imdsGet(client, base+"/meta-data/iam/security-credentials/"+roleName, token)
+	if err != nil {
+		return imdsCreds{}, fmt.Errorf("fetch role credentials: %w", err)
+	}
+	var creds imdsCreds
+	if err := json.Unmarshal([]byte(body), &creds); err != nil {
+		return imdsCreds{}, fmt.Errorf("decode role credentials: %w", err)
+	}
+	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
+		return imdsCreds{}, fmt.Errorf("IMDS credentials missing access/secret key")
+	}
+	return creds, nil
+}
+
+// imdsV2Token requests an IMDSv2 session token; returns "" if the service does
+// not enforce v2 (tokenless v1 then applies).
+func imdsV2Token(client *http.Client, base string) string {
+	req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
+	if err != nil {
+		return ""
+	}
+	req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", imdsTokenTTL)
+	resp, err := client.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ""
+	}
+	body, _ := io.ReadAll(resp.Body)
+	return strings.TrimSpace(string(body))
+}
+
+func imdsGet(client *http.Client, url, token string) (string, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("IMDS %s returned %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return string(body), nil
+}
+
+// getAuthToken builds an aws-sdk ECR client pointed at the gateway and calls
+// GetAuthorizationToken, decoding the AWS:<jwt> token into Basic-auth parts. It
+// returns the username, password, the ProxyEndpoint host, and the token expiry.
+func getAuthToken(cfg config, httpClient *http.Client, creds imdsCreds) (username, password, proxyHost string, expiresAt time.Time, err error) {
+	region := cfg.Region
+	if region == "" {
+		region = "us-east-1"
+	}
+	sess, serr := session.NewSession(&aws.Config{
+		Region:      aws.String(region),
+		Endpoint:    aws.String(cfg.GatewayURL),
+		Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, creds.Token),
+		HTTPClient:  httpClient,
+		DisableSSL:  aws.Bool(false),
+	})
+	if serr != nil {
+		return "", "", "", time.Time{}, fmt.Errorf("new session: %w", serr)
+	}
+
+	out, gerr := ecr.New(sess).GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
+	if gerr != nil {
+		return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken: %w", gerr)
+	}
+	if len(out.AuthorizationData) == 0 || out.AuthorizationData[0].AuthorizationToken == nil {
+		return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken returned no authorization data")
+	}
+	data := out.AuthorizationData[0]
+
+	raw, derr := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken))
+	if derr != nil {
+		return "", "", "", time.Time{}, fmt.Errorf("decode authorization token: %w", derr)
+	}
+	user, pass, ok := strings.Cut(string(raw), ":")
+	if !ok {
+		return "", "", "", time.Time{}, fmt.Errorf("authorization token not in user:password form")
+	}
+
+	if data.ExpiresAt != nil {
+		expiresAt = aws.TimeValue(data.ExpiresAt)
+	}
+	return user, pass, hostFromImage(aws.StringValue(data.ProxyEndpoint)), expiresAt, nil
+}
+
+// cacheDurationFrom derives a Go-duration string from the token expiry minus a
+// safety margin, falling back to the default when expiry is unknown or too near.
+func cacheDurationFrom(expiresAt time.Time) string {
+	if expiresAt.IsZero() {
+		return defaultCacheDuration
+	}
+	d := time.Until(expiresAt) - expiryMargin
+	if d <= 0 {
+		return defaultCacheDuration
+	}
+	return d.Round(time.Second).String()
+}
+
+// gatewayHTTPClient builds an HTTP client trusting the gateway CA at caPath.
+// When caPath is empty it relies on the system trust store.
+func gatewayHTTPClient(caPath string) (*http.Client, error) {
+	tlsCfg := &tls.Config{
+		MinVersion:       tls.VersionTLS13,
+		CurvePreferences: tlsconfig.Curves,
+	}
+	if caPath != "" {
+		pem, err := os.ReadFile(caPath)
+		if err != nil {
+			return nil, fmt.Errorf("read gateway CA %q: %w", caPath, err)
+		}
+		pool := x509.NewCertPool()
+		if !pool.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("gateway CA %q has no usable certificates", caPath)
+		}
+		tlsCfg.RootCAs = pool
+	}
+	return &http.Client{
+		Timeout:   15 * time.Second,
+		Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
+	}, nil
+}
+
+// loadConfig reads the cloud-init env file then lets real env vars override. The
+// IMDS base is fixed (not in the env file) but overridable for tests.
+func loadConfig(envFile string) config {
+	env := parseEnvFile(envFile)
+	get := func(key string) string {
+		if v := os.Getenv(key); v != "" {
+			return v
+		}
+		return env[key]
+	}
+
+	cfg := config{
+		GatewayURL: get("EKS_GATEWAY_URL"),
+		GatewayCA:  get("EKS_GATEWAY_CA"),
+		Region:     get("EKS_REGION"),
+		IMDSBase:   get("EKS_IMDS_BASE"),
+	}
+	if cfg.GatewayCA == "" {
+		cfg.GatewayCA = defaultGatewayCA
+	}
+	if cfg.IMDSBase == "" {
+		cfg.IMDSBase = defaultIMDSBase
+	}
+	return cfg
+}
+
+// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
+func parseEnvFile(path string) map[string]string {
+	out := map[string]string{}
+	f, err := os.Open(path)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, val, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		out[strings.TrimSpace(key)] = strings.TrimSpace(val)
+	}
+	return out
+}
+
+// hostFromImage returns the host[:port] portion of an image ref or URL. It strips
+// any scheme and trims at the first path separator.
+func hostFromImage(image string) string {
+	if i := strings.Index(image, "://"); i >= 0 {
+		image = image[i+3:]
+	}
+	if i := strings.IndexByte(image, '/'); i >= 0 {
+		image = image[:i]
+	}
+	return image
+}
+
+// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
+func emitEmpty(out io.Writer) {
+	resp := CredentialProviderResponse{
+		APIVersion:    credProviderAPIVersion,
+		Kind:          "CredentialProviderResponse",
+		CacheKeyType:  "Registry",
+		CacheDuration: defaultCacheDuration,
+		Auth:          map[string]AuthConfig{},
+	}
+	if err := json.NewEncoder(out).Encode(&resp); err != nil {
+		slog.Error("ecr-credential-provider: encode empty response", "err", err)
+	}
+}
@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
+// ECR GetAuthorizationToken on one TLS server, routed by request path.
+func fakeBackend(t *testing.T) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+
+	// IMDSv2 token.
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPut {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		_, _ = w.Write([]byte("imds-v2-token"))
+	})
+	// IMDS role name.
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		// Role credentials.
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"Code":            "Success",
+			"Type":            "AWS-HMAC",
+			"AccessKeyId":     "AKIATEST",
+			"SecretAccessKey": "secrettest",
+			"Token":           "sessiontoken",
+			"AccountId":       "123456789012",
+		})
+	})
+	// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"authorizationData": []map[string]any{{
+				"authorizationToken": authToken,
+				"proxyEndpoint":      "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
+			}},
+		})
+	})
+
+	srv := httptest.NewTLSServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func clientTrusting(srv *httptest.Server) *http.Client {
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+}
+
+func TestRun_ReturnsBasicAuth(t *testing.T) {
+	srv := fakeBackend(t)
+	client := clientTrusting(srv)
+
+	cfg := config{
+		GatewayURL: srv.URL,
+		Region:     "us-east-1",
+		IMDSBase:   srv.URL + "/latest",
+	}
+
+	const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
+	reqBody, err := json.Marshal(CredentialProviderRequest{
+		APIVersion: credProviderAPIVersion,
+		Kind:       "CredentialProviderRequest",
+		Image:      image,
+	})
+	if err != nil {
+		t.Fatalf("marshal request: %v", err)
+	}
+
+	var out bytes.Buffer
+	if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+
+	var resp CredentialProviderResponse
+	if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
+		t.Fatalf("unmarshal response: %v\n%s", err, out.String())
+	}
+
+	if resp.Kind != "CredentialProviderResponse" {
+		t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
+	}
+	if resp.CacheKeyType != "Registry" {
+		t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
+	}
+
+	host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
+	auth, ok := resp.Auth[host]
+	if !ok {
+		t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
+	}
+	if auth.Username != "AWS" {
+		t.Errorf("username = %q, want AWS", auth.Username)
+	}
+	if auth.Password != "testjwt" {
+		t.Errorf("password = %q, want testjwt", auth.Password)
+	}
+}
+
+func TestHostFromImage(t *testing.T) {
+	cases := map[string]string{
+		"acct.dkr.ecr.us-east-1.spinifex.internal/demo:latest": "acct.dkr.ecr.us-east-1.spinifex.internal",
+		"https://acct.dkr.ecr.us-east-1.spinifex.internal":     "acct.dkr.ecr.us-east-1.spinifex.internal",
+		"host:9999/repo:tag": "host:9999",
+		"plainhost":          "plainhost",
+	}
+	for in, want := range cases {
+		if got := hostFromImage(in); got != want {
+			t.Errorf("hostFromImage(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
@@ -0,0 +1,147 @@
+// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving
+// certificate for an in-cluster admission webhook and prints its base64 PEM so
+// the addon renderer can embed it as the webhook Secret + each caBundle.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent of the
+// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster,
+// cached on disk under -dir, stable across addon-sync ticks. The single cert is
+// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf
+// and the caBundle the apiserver trusts.
+//
+// Usage:
+//
+//	eks-webhook-cert -dir /var/lib/spinifex-eks/webhook-certs/<addon> \
+//	    -cn <svc>.<ns>.svc -dns <svc>.<ns>.svc,<svc>.<ns>.svc.cluster.local
+//
+// Output (one line): <ca_b64>\t<tls_crt_b64>\t<tls_key_b64>
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	cn := flag.String("cn", "", "certificate common name")
+	dnsCSV := flag.String("dns", "", "comma-separated DNS SANs")
+	flag.Parse()
+
+	if *dir == "" || *cn == "" || *dnsCSV == "" {
+		fatal("eks-webhook-cert: -dir, -cn and -dns are required")
+	}
+	dnsNames := splitCSV(*dnsCSV)
+	if len(dnsNames) == 0 {
+		fatal("eks-webhook-cert: -dns lists no names")
+	}
+
+	certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames)
+	if err != nil {
+		fatal("eks-webhook-cert: " + err.Error())
+	}
+	b := base64.StdEncoding.EncodeToString
+	fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM))
+}
+
+// ensureCert returns the cached cert/key under dir, minting and persisting a
+// fresh self-signed pair (for cn + dnsNames) when none is present or readable.
+func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if c, cerr := os.ReadFile(certPath); cerr == nil {
+		if k, kerr := os.ReadFile(keyPath); kerr == nil {
+			if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
+				return c, k, nil
+			}
+		}
+	}
+
+	certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return nil, nil, fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPEM, keyPEM, nil
+}
+
+// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
+// a server cert, with the given common name and DNS SANs.
+func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		DNSNames:              dnsNames,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func splitCSV(s string) []string {
+	var out []string
+	for p := range strings.SplitSeq(s, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/x509"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const (
+	testCN  = "aws-load-balancer-webhook-service.kube-system.svc"
+	testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local"
+)
+
+func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
+	t.Helper()
+	leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
+	if err != nil {
+		t.Fatalf("parse cert: %v", err)
+	}
+	return leaf
+}
+
+func TestEnsureCertMintsServingCert(t *testing.T) {
+	dir := t.TempDir()
+	certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS})
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if len(keyPEM) == 0 {
+		t.Fatal("empty key PEM")
+	}
+	leaf := parseLeaf(t, certPEM)
+
+	// Both webhook service DNS names must be present, or the apiserver's TLS
+	// dial to the webhook fails SAN verification.
+	for _, want := range []string{testCN, testDNS} {
+		if err := leaf.VerifyHostname(want); err != nil {
+			t.Errorf("SAN %q missing: %v", want, err)
+		}
+	}
+	if !leaf.IsCA {
+		t.Error("cert must be a CA so the same PEM serves as the webhook caBundle")
+	}
+
+	// The single self-signed cert must verify against itself as a root — this is
+	// exactly what the apiserver does with caBundle == the serving cert.
+	roots := x509.NewCertPool()
+	roots.AddCert(leaf)
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   testCN,
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("self-signed cert does not verify as its own CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesCachedPair(t *testing.T) {
+	dir := t.TempDir()
+	cert1, key1, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	cert2, key2, err := ensureCert(dir, testCN, []string{testCN})
+	if err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if string(cert1) != string(cert2) || string(key1) != string(key2) {
+		t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick")
+	}
+
+	// The private key must not be world-readable on disk.
+	info, err := os.Stat(filepath.Join(dir, "tls.key"))
+	if err != nil {
+		t.Fatalf("stat key: %v", err)
+	}
+	if info.Mode().Perm()&0o077 != 0 {
+		t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm())
+	}
+}
@@ -298,6 +298,7 @@ func init() {
 	adminInitCmd.Flags().String("predastore-nodes", "", "Comma-separated IPs for multi-node Predastore cluster (e.g., 10.11.12.1,10.11.12.2,10.11.12.3). Requires >= 3 nodes.")
 	adminInitCmd.Flags().String("formation-timeout", "10m", "Timeout for cluster formation (e.g., 5m, 30s)")
 	adminInitCmd.Flags().String("token-ttl", "30m", "Join token validity duration (e.g. 30m, 1h, 2h)")
+	adminInitCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminInitCmd.Flags().String("cluster-name", "spinifex", "NATS cluster name")
 	adminInitCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during init (default: enabled)")
 	adminInitCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
@@ -331,6 +332,7 @@ func init() {
 	adminJoinCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all)")
 	adminJoinCmd.Flags().Bool("no-telemetry", false, "Disable telemetry metrics sent during join (default: enabled)")
 	adminJoinCmd.Flags().String("email", "", "Operator email address (used for update and security notifications)")
+	adminJoinCmd.Flags().Int("predastore-compaction-interval", 0, "Predastore compactor interval in seconds (0 = unset, uses built-in default). Test clusters set a short interval.")
 	adminJoinCmd.MarkFlagRequired("node")
 	adminJoinCmd.MarkFlagRequired("host")
 	adminJoinCmd.MarkFlagRequired("token")
@@ -846,6 +848,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	predastoreNodesStr, _ := cmd.Flags().GetString("predastore-nodes")
 	formationTimeoutStr, _ := cmd.Flags().GetString("formation-timeout")
 	tokenTTLStr, _ := cmd.Flags().GetString("token-ttl")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
 	clusterName, _ := cmd.Flags().GetString("cluster-name")
 	services, _ := cmd.Flags().GetStringSlice("services")

@@ -1145,7 +1148,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   AWS Profile: spinifex\n")

 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
-	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP)
+	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)

 	// Generate per-node IPsec peer cert when cluster-wide IPsec is enabled
 	// (default true). Reuses the cluster CA — no intermediate strongSwan PKI.
@@ -1256,7 +1259,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}

 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1297,8 +1300,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,

 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",
@@ -1403,6 +1407,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")
+
 	joinToken, err := formation.GenerateJoinToken()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "❌ Error generating join token: %v\n", err)
@@ -1546,7 +1552,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	var predastoreNodeID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1583,9 +1589,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(allNodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(allNodes, node),

 		OperatorEmail: email,

@@ -1655,6 +1662,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	configDir, _ := cmd.Flags().GetString("config-dir")
 	clusterBind, _ := cmd.Flags().GetString("cluster-bind")
 	services, _ := cmd.Flags().GetStringSlice("services")
+	compactionInterval, _ := cmd.Flags().GetInt("predastore-compaction-interval")

 	email, _ := cmd.Flags().GetString("email")
 	email = strings.TrimSpace(email)
@@ -1959,7 +1967,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	}

 	// Generate server cert signed by CA with this node's bind IP
-	if err := admin.GenerateServerCertOnly(configDir, bindIP); err != nil {
+	if err := admin.GenerateServerCertOnly(configDir, bindIP, region, config.DefaultAWSInternalSuffix); err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2002,7 +2010,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	hasPredastoreConfig := len(predastoreNodes) >= 2

 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP, compactionInterval)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -2043,9 +2051,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,

-		PredastoreNodeID: predastoreNodeID,
-		Services:         services,
-		RemoteNodes:      buildRemoteNodes(statusResp.Nodes, node),
+		PredastoreNodeID:          predastoreNodeID,
+		CompactionIntervalSeconds: compactionInterval,
+		Services:                  services,
+		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node),

 		OperatorEmail: email,

@@ -6,11 +6,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

+	s3 "github.com/mulgadc/predastore/s3"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/utils"
+	toml "github.com/pelletier/go-toml/v2"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -206,7 +210,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 		{ID: 3, Host: "10.0.0.3"},
 	}
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content, `nats_url = "nats://localhost:4222"`)
@@ -214,7 +218,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {

 	// Specific bind IP → stays as-is.
 	content2, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1",
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1", 0,
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
@@ -454,3 +458,81 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 	assert.Equal(t, "dhcp", settings.PoolSource)
 	assert.Equal(t, "br-wan", settings.PoolBindBridge)
 }
+
+func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "predastore.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, predastoreTomlTemplate, settings))
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
+	return []admin.PredastoreNodeConfig{
+		{ID: 1, Host: "10.0.0.1"}, {ID: 2, Host: "10.0.0.2"}, {ID: 3, Host: "10.0.0.3"},
+	}
+}
+
+// Unset knob must leave production config byte-identical: no [compaction] block
+// and the original trailing bytes unchanged (single-node has no trailing
+// newline, multinode keeps one).
+func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok",
+	})
+	assert.NotContains(t, single, "[compaction]")
+	assert.True(t, strings.HasSuffix(single, `access_keys_bucket = "spinifex-iam-access-keys"`),
+		"unset single-node tail changed: %q", single[len(single)-40:])
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0,
+	)
+	require.NoError(t, err)
+	assert.NotContains(t, multi, "[compaction]")
+	assert.True(t, strings.HasSuffix(multi, "access_keys_bucket = \"spinifex-iam-access-keys\"\n"),
+		"unset multinode tail changed: %q", multi[len(multi)-40:])
+}
+
+func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
+	const interval = 30
+
+	single := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
+		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok", CompactionIntervalSeconds: interval,
+	})
+	assert.Contains(t, single, "[compaction]")
+	var singleCfg s3.Config
+	require.NoError(t, toml.Unmarshal([]byte(single), &singleCfg))
+	assert.Equal(t, interval, singleCfg.Compaction.IntervalSeconds)
+
+	multi, err := admin.GenerateMultiNodePredastoreConfig(
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", interval,
+	)
+	require.NoError(t, err)
+	var multiCfg s3.Config
+	require.NoError(t, toml.Unmarshal([]byte(multi), &multiCfg))
+	assert.Equal(t, interval, multiCfg.Compaction.IntervalSeconds)
+}
+
+// The init/join flag must register with an unset (0) default and flow into the
+// rendered config via ConfigSettings.
+func TestAdminInitFlag_CompactionIntervalReachesRenderedConfig(t *testing.T) {
+	for _, c := range []*cobra.Command{adminInitCmd, adminJoinCmd} {
+		f := c.Flags().Lookup("predastore-compaction-interval")
+		require.NotNil(t, f, "%s missing --predastore-compaction-interval", c.Name())
+		assert.Equal(t, "0", f.DefValue)
+	}
+
+	require.NoError(t, adminInitCmd.Flags().Set("predastore-compaction-interval", "45"))
+	defer func() { _ = adminInitCmd.Flags().Set("predastore-compaction-interval", "0") }()
+
+	v, _ := adminInitCmd.Flags().GetInt("predastore-compaction-interval")
+	require.Equal(t, 45, v)
+
+	out := renderSingleNodePredastore(t, admin.ConfigSettings{
+		Region: "ap-southeast-2", BindIP: "0.0.0.0", CompactionIntervalSeconds: v,
+	})
+	assert.Contains(t, out, "interval_seconds = 45")
+}
@@ -75,3 +75,8 @@ nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -103,4 +103,9 @@ nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}
 nats_token = "{{.NatsToken}}"
 nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
-access_keys_bucket = "spinifex-iam-access-keys"
+access_keys_bucket = "spinifex-iam-access-keys"
+{{- if .CompactionIntervalSeconds}}
+
+[compaction]
+interval_seconds = {{.CompactionIntervalSeconds}}
+{{- end}}
@@ -8,6 +8,10 @@ node = "{{.Node}}"
 email = "{{.OperatorEmail}}"
 {{- end}}

+[aws]
+region = "{{.Region}}"
+internal_suffix = "spinifex.internal"
+
 [network]
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
@@ -113,7 +113,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn) | `--security-group-ids`, `--tag-specifications`, `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | **DONE** |
+| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--metadata-options`, `--launch-template`, `--hibernate-options` | **DONE** |
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | **DONE** |
@@ -132,6 +132,11 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 role ARN and rejects cross-account profile ARNs. Placement strategies: `spread`
 (1 per node, atomic CAS) and `cluster` (pin all to one node).

+`run-instances --capacity-reservation-specification` consumes a reserved slot on
+the node owning the targeted reservation (targeted-by-id only; `open`
+auto-matching is not supported). The combination with `--placement` (group) is
+rejected. `describe-instances` then reports the instance's `CapacityReservationId`.
+
 ### EC2 — IAM Instance Profile Associations

 Stored as `vm.VM.IamInstanceProfileArn` + `IamInstanceProfileAssociationId`
@@ -433,9 +438,9 @@ EIP commands are only registered when an external IPAM pool is configured.
 | `delete-dhcp-options` | — | `--dhcp-options-id`, `--dry-run` | **NOT STARTED** |
 | `describe-dhcp-options` | — | `--dhcp-options-ids`, `--filters`, `--max-results` | **NOT STARTED** |
 | `associate-dhcp-options` | — | `--dhcp-options-id`, `--vpc-id`, `--dry-run` | **NOT STARTED** |
-| `create-capacity-reservation` | — | `--instance-type`, `--instance-platform`, `--availability-zone`, `--instance-count`, `--end-date`, `--end-date-type`, `--instance-match-criteria`, `--tag-specifications` | **NOT STARTED** |
-| `cancel-capacity-reservation` | — | `--capacity-reservation-id`, `--dry-run` | **NOT STARTED** |
-| `describe-capacity-reservations` | — | `--capacity-reservation-ids`, `--filters`, `--max-results` | **NOT STARTED** |
+| `create-capacity-reservation` | `--instance-type`, `--instance-count`, `--availability-zone`, `--instance-platform`, `--instance-match-criteria`, `--tenancy` (default only), `--dry-run` | `--end-date`, `--end-date-type` (unlimited only), `--availability-zone-id`, `--tag-specifications` | **DONE** |
+| `cancel-capacity-reservation` | `--capacity-reservation-id`, `--dry-run` | — | **DONE** |
+| `describe-capacity-reservations` | `--capacity-reservation-ids`, `--filters` | `--max-results`, `--next-token` | **DONE** |
 | `modify-capacity-reservation` | — | `--capacity-reservation-id`, `--instance-count`, `--end-date`, `--end-date-type` | **NOT STARTED** |

 ### EC2 — Misc
@@ -593,17 +598,25 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | Path | Method | Source | Status |
 |------|--------|--------|--------|
 | `/latest/api/token` | PUT | Issues an ENI-bound IMDSv2 token; `X-aws-ec2-metadata-token-ttl-seconds` ∈ [1, 21600] required | **DONE** |
+| `/` | GET | Supported API-version list (`2021-07-15`, `latest`); token-gated | **DONE** |
+| `/latest` | GET | Top-level tree listing (`dynamic`, `meta-data`, `user-data`) | **DONE** |
+| `/<date>/...` | GET/PUT | Any dated API version aliases to `/latest` (cloud-init parity) | **DONE** |
 | `/latest/meta-data/` | GET | Directory listing of supported children | **DONE** |
 | `/latest/meta-data/instance-id` | GET | `vm.ID` | **DONE** |
 | `/latest/meta-data/instance-type` | GET | `vm.InstanceType` | **DONE** |
 | `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
+| `/latest/meta-data/ami-launch-index` | GET | Per-instance launch index (`0..n-1`), contiguous on partial failure | **DONE** |
+| `/latest/meta-data/reservation-id` | GET | `DescribeInstances` `Reservation.ReservationId` | **DONE** |
+| `/latest/meta-data/instance-life-cycle` | GET | `on-demand` (Spot not modelled) | **DONE** |
 | `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
 | `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
+| `/latest/meta-data/public-hostname` | GET | Mirrors `public-ipv4`; 404 when no public IP | **DONE** |
 | `/latest/meta-data/mac` | GET | `ENIRecord.MacAddress` | **DONE** |
 | `/latest/meta-data/security-groups` | GET | `ENIRecord.SecurityGroupIds`, newline-separated | **DONE** |
 | `/latest/meta-data/hostname`, `/local-hostname` | GET | Synthesised `ip-<dashed-ip>.<region>.compute.internal` | **DONE** |
 | `/latest/meta-data/placement/availability-zone` | GET | `ENIRecord.AvailabilityZone` | **DONE** |
 | `/latest/meta-data/placement/region` | GET | Derived from AZ (trailing letter stripped) | **DONE** |
+| `/latest/meta-data/services/{domain,partition}` | GET | Static: `amazonaws.com` / `aws` | **DONE** |
 | `/latest/meta-data/iam/info` | GET | `{InstanceProfileArn, InstanceProfileId}`; 404 if no profile | **DONE** |
 | `/latest/meta-data/iam/security-credentials/` | GET | Role name(s) under the profile, one per line; empty body if none | **DONE** |
 | `/latest/meta-data/iam/security-credentials/<role>` | GET | STS `AssumeRoleForInstance` → ASIA-prefixed temporary credential JSON | **DONE** |
@@ -611,16 +624,15 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | `/latest/meta-data/public-keys/0/` | GET | `openssh-key` (format list for index 0) | **DONE** |
 | `/latest/meta-data/public-keys/0/openssh-key` | GET | Launch SSH public key, live-fetched from the key store; 404 if the key was deleted, 500 on backend fault | **DONE** |
 | `/latest/user-data` | GET | `vm.UserData`; 404 if none | **DONE** |
-| `/latest/dynamic/instance-identity/document` (+ `signature`, `pkcs7`, `rsa2048`) | GET | Needs a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
+| `/latest/dynamic` | GET | Lists `instance-identity/` | **DONE** |
+| `/latest/dynamic/instance-identity` | GET | Lists `document` (signed forms listed when the signing key lands) | **DONE** |
+| `/latest/dynamic/instance-identity/document` | GET | Unsigned identity document from resolved ENI + instance facts | **DONE** |
+| `/latest/dynamic/instance-identity/{signature,pkcs7,rsa2048}` | GET | Signed forms; need a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
 | `/latest/meta-data/network/interfaces/macs/<mac>/...` | GET | Multi-ENI rendering (subnet-id, vpc-id, ipv4s, security-group-ids, …) | **NOT STARTED** (404) |
 | `/latest/meta-data/tags/instance/<key>` | GET | Instance-tag metadata; gated on `InstanceMetadataTags` enablement | **NOT STARTED** (404) |
-| `/latest/meta-data/public-hostname` | GET | No public DNS today; would mirror `public-ipv4` | **NOT STARTED** (404) |
-| `/latest/meta-data/reservation-id` | GET | Available on the stored `Reservation`; cheap, not yet surfaced | **NOT STARTED** (404) |
-| `/latest/meta-data/ami-launch-index` | GET | `0` for single-instance launches; trivial once multi-count is exercised | **NOT STARTED** (404) |
 | `/latest/meta-data/block-device-mapping/...` | GET | `ami`/`root`/`ebsN`/`ephemeralN` device map | **NOT STARTED** (404) |
 | `/latest/meta-data/placement/{group-name,partition-number,availability-zone-id,host-id}` | GET | Placement extras beyond `availability-zone`/`region` | **NOT STARTED** (404) |
-| `/latest/meta-data/instance-life-cycle`, `/instance-action` | GET | `on-demand`/`spot`; `none` unless interruptible instances ship | **NOT STARTED** (404) |
-| `/latest/meta-data/services/{domain,partition}` | GET | Static per-deployment values | **NOT STARTED** (404) |
+| `/latest/meta-data/instance-action` | GET | `none` unless interruptible instances ship | **NOT STARTED** (404) |
 | `/latest/meta-data/spot/{instance-action,termination-time}` | GET | Only meaningful once Spot is modelled | **NOT STARTED** (404) |

 ---
@@ -95,13 +95,6 @@ checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster
 and `eks create-nodegroup` resolve the boot AMI by that tag, so no further
 configuration is needed.

-Operators publishing a freshly built node image to the mirror run the
-**Publish EKS node AMI** GitHub Actions workflow (manual dispatch; uses the
-`CF_R2_*` repo secrets) or, locally, `make publish-eks-node-image` (see
-`scripts/publish-system-image.sh`; requires `R2_ENDPOINT` + R2-scoped
-`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`).
-
-
 ## Cluster Shutdown

 Coordinated, phased shutdown of the entire cluster (API/UI → VMs → viperblock → predastore → NATS/daemon):
@@ -0,0 +1,188 @@
+---
+title: "ECR (Container Registry)"
+description: "Store and serve container images from Spinifex's AWS-compatible Elastic Container Registry — create a repository, authenticate Docker, push and pull images, and let your EKS workers pull from it, using the AWS CLI, the Spinifex console, or Terraform."
+category: "Containers"
+tags:
+  - ecr
+  - containers
+  - docker
+  - registry
+  - oci
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Terraform AWS Provider — ECR"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository"
+  - title: "OCI Distribution Spec"
+    url: "https://github.com/opencontainers/distribution-spec"
+  - title: "Docker CLI Reference"
+    url: "https://docs.docker.com/reference/cli/docker/"
+---
+
+# ECR (Container Registry)
+
+> Spinifex's ECR service is an AWS-compatible, OCI-compliant container registry. You create repositories with `aws ecr`, authenticate Docker with a registry token, and `docker push` / `docker pull` exactly as you would on AWS — and your EKS clusters can pull from it directly.
+
+## Overview
+
+ECR on Spinifex serves a private container registry on the Spinifex gateway endpoint — the same `host:9999` you already use for the AWS API. Authentication is account-scoped by token, so you push and pull at the gateway host and never need extra DNS:
+
+```
+<spinifex-gateway-host>:9999/<repository>
+```
+
+`aws ecr describe-repositories` and `aws ecr get-login-password` return the exact host to use — always prefer that value. Where real DNS is configured, the AWS-parity per-account host `<account-id>.dkr.ecr.<region>.<your-spinifex-domain>:9999` also works.
+
+Each repository holds image manifests and their layers. Images are stored in object storage ([Predastore](https://github.com/mulgadc/predastore)) in a per-account bucket, with blobs content-addressed and de-duplicated across repositories. Authentication is token-based: `aws ecr get-login-password` mints a short-lived bearer token that `docker login` uses against the registry's `/v2/` endpoint.
+
+**What works today**
+
+- Repository lifecycle — `CreateRepository`, `DeleteRepository`, `DescribeRepositories`.
+- OCI push and pull — blobs, manifests, and tags, including cross-repository blob mounts.
+- Token auth — `GetAuthorizationToken` plus bearer/basic auth on `/v2/`.
+- Repository policies — set, get, and delete a repository policy document.
+
+**Current limitations**
+
+- **Image scanning is not supported** — scan APIs return an explicit "operation not supported" error.
+- **No automatic garbage collection yet** — deleting images does not yet reclaim underlying blobs.
+- Lifecycle-policy enforcement and tag-immutability enforcement are not active.
+
+## Prerequisites
+
+- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)).
+- **Docker** installed locally to build, push, and pull images.
+- **Docker trusting the Spinifex CA.** The registry serves TLS signed by the Spinifex local CA. Install it into the host trust store once (Docker uses the host root pool for registry TLS, so this covers every account endpoint without per-host config):
+
+```bash
+sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-local-ca.crt
+sudo update-ca-certificates
+sudo systemctl restart docker
+```
+
+- **The registry endpoint** — the Spinifex gateway `host:9999`. `aws ecr get-login-password` and `aws ecr describe-repositories` return the exact `host:port`; copy it from there rather than constructing it by hand.
+- **For EKS pulls:** worker nodes need the `AmazonEC2ContainerRegistryReadOnly` policy on their node IAM role (the EKS prerequisites already include this) and network egress to the registry endpoint.
+
+## Instructions
+
+Create a repository and push an image using your preferred tool.
+
+:::tabs
+@tab AWS CLI
+
+### 1. Create a repository
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws ecr create-repository --repository-name my-app
+aws ecr describe-repositories --repository-names my-app \
+  --query 'repositories[0].repositoryUri'
+```
+
+The `repositoryUri` is the value you tag and push to — the gateway `host:9999`
+with the repository path, port included (for example `192.0.2.10:9999/my-app`).
+Use it verbatim; don't hand-build a hostname.
+
+### 2. Authenticate Docker
+
+Take the registry host straight from the API so the `:9999` port is correct:
+
+```bash
+REGISTRY=$(aws ecr describe-repositories --repository-names my-app \
+  --query 'repositories[0].repositoryUri' --output text | cut -d/ -f1)
+
+aws ecr get-login-password --region <region> \
+  | docker login --username AWS --password-stdin "$REGISTRY"
+```
+
+### 3. Build, tag, and push
+
+```bash
+docker build -t my-app:latest .
+docker tag my-app:latest "$REGISTRY/my-app:latest"
+docker push "$REGISTRY/my-app:latest"
+```
+
+### 4. Verify and pull
+
+```bash
+aws ecr describe-images --repository-name my-app
+docker pull "$REGISTRY/my-app:latest"
+```
+
+@tab Spinifex UI
+
+From the left navigation open **ECR → Repositories**.
+
+<img src="ecr-list-repositories.png" alt="The ECR Repositories list in the Spinifex console, showing a repository with its registry URI, tag mutability, and creation date">
+
+### 1. Create a repository
+
+1. Click **Create Repository**.
+2. Enter the **repository name** and, optionally, set **tag mutability** (mutable or immutable).
+3. Submit — the repository appears in the list with its full registry URI.
+
+### 2. Copy the push commands
+
+Open the repository to see its detail page. The **Push commands** panel lists the exact `aws ecr get-login-password | docker login`, `docker build`, `docker tag`, and `docker push` commands, pre-filled with your registry host and repository URI — use the **Copy** button and run them locally.
+
+<img src="ecr-push-commands.png" alt="A repository detail page in the Spinifex console, with the Push commands panel showing the docker login, build, tag, and push commands">
+
+### 3. Manage the repository
+
+- The **Permissions** tab edits the repository's access policy.
+- The **Lifecycle** tab manages retention rules, and **Tag immutability** can be toggled from the detail page.
+- The **Scan** tab reflects that image scanning is not supported on Spinifex.
+- Delete a repository from the **Delete** action in the repositories list.
+
+@tab Terraform
+
+Repositories can be managed as code with the standard `aws_ecr_repository` resource, with the AWS provider pointed at Spinifex's `ecr` endpoint:
+
+```hcl
+provider "aws" {
+  region                      = "ap-southeast-2"
+  skip_credentials_validation = true
+  skip_requesting_account_id  = true
+
+  endpoints {
+    ecr = "https://<your-spinifex-gateway>"
+  }
+}
+
+resource "aws_ecr_repository" "my_app" {
+  name = "my-app"
+}
+
+output "repository_url" {
+  value = aws_ecr_repository.my_app.repository_url
+}
+```
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+```
+
+Then authenticate Docker and push using the `repository_url` output, following steps 2–4 in the AWS CLI tab.
+
+:::
+
+## Troubleshooting
+
+**`docker login` fails with an authorization error.** The token is account-scoped and short-lived. Re-run `aws ecr get-login-password` to mint a fresh token, and confirm the registry host in `docker login` matches your account's endpoint (`aws ecr describe-repositories` returns it).
+
+**`docker push` cannot reach the registry.** Use the exact host from `aws ecr describe-repositories` — the Spinifex gateway `host:9999`. Docker dials it directly, so the `:9999` port must be present (without it docker tries `:443` and fails). If you are using the AWS-parity `<account-id>.dkr.ecr…` hostname instead, that path needs DNS resolving it to the gateway — prefer the gateway host the API returns.
+
+**EKS workers cannot pull an image.** Confirm the node IAM role has `AmazonEC2ContainerRegistryReadOnly` and that the workers have egress to the registry endpoint (an Internet Gateway or NAT Gateway route). See the [EKS prerequisites](/docs/eks).
+
+**Image-scanning commands return an error.** Image scanning is not supported on Spinifex; the scan APIs intentionally reject these calls. Remove scan-on-push configuration from your tooling.
+
+**A pushed image still appears after deletion frees no space.** Garbage collection of unreferenced blobs is not yet automatic — deleting an image removes its manifest and tags but does not immediately reclaim the underlying layers.
@@ -0,0 +1,248 @@
+---
+title: "EKS (Managed Kubernetes)"
+description: "Run a managed, AWS-compatible Kubernetes cluster on Spinifex — provision the control plane and worker node group, wire up the VPC, IAM, and security groups it needs, and deploy your first workload via the AWS CLI, the Spinifex console, or Terraform."
+category: "Containers"
+tags:
+  - eks
+  - kubernetes
+  - containers
+  - vpc
+  - iam
+sections:
+  - overview
+  - prerequisites
+  - instructions
+  - troubleshooting
+resources:
+  - title: "EKS Quickstart Workbook"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-quickstart"
+  - title: "EKS HTTPS Ingress Workbook (LBC + ACM)"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-https-ingress"
+  - title: "EKS GitOps Workbook (Argo CD + EBS-CSI)"
+    url: "https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/eks-gitops-argocd"
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "kubectl Documentation"
+    url: "https://kubernetes.io/docs/reference/kubectl/"
+---
+
+# EKS (Managed Kubernetes)
+
+> Spinifex's EKS service gives you a managed, AWS-compatible Kubernetes API: you create a cluster and a node group, and Spinifex provisions and runs the control plane and workers for you. Standard tools — `aws eks`, `kubectl`, the Terraform AWS provider — work unchanged against your cluster.
+
+## Overview
+
+An EKS cluster on Spinifex has two parts: a **control plane** (the managed Kubernetes API server, provisioned on a Spinifex-managed VM and fronted by a network load balancer) and one or more **managed node groups** of worker VMs that run your pods. You provide the VPC, subnets, and IAM roles; Spinifex creates and reconciles the cluster, its load balancer, and its security groups.
+
+Authentication uses **EKS access entries** (the API authentication mode) — IAM principals are mapped to Kubernetes RBAC groups, and `kubectl` authenticates with `aws eks get-token` exactly as it does on AWS.
+
+**What you'll create:**
+
+| Resource | Purpose |
+|---|---|
+| VPC + subnets | The network the cluster and workers run in |
+| Internet Gateway / NAT Gateway | Egress so workers can pull container images |
+| Cluster IAM role | Lets the control plane manage cluster resources |
+| Node IAM role | Lets workers join the cluster and pull images |
+| EKS cluster | The managed Kubernetes API endpoint |
+| Managed node group | The worker VMs that run your pods |
+| Security-group rule | Opens your app's port to reach the workers |
+
+**Spinifex specifics**
+
+- **Auth mode is API-only.** `authentication_mode` must be `API`; the legacy `CONFIG_MAP` and `API_AND_CONFIG_MAP` modes are rejected. Grant access with access entries, not an `aws-auth` ConfigMap.
+- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
+- **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
+- **Default Kubernetes version is `1.32`.**
+
+## Prerequisites
+
+Before creating a cluster you need the supporting infrastructure in place. The Terraform workbooks build all of this for you; if you are using the CLI or console, create it first.
+
+- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and `kubectl` installed.
+- **The `eks-node` image** registered on the cluster. The console blocks cluster creation until it is present.
+- **A VPC with at least one subnet.** The control plane launches in your first subnet; workers run in the subnets you pass to the node group.
+- **Egress for the workers.** Workers must reach the internet to pull container images:
+  - **Public workers** — an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
+  - **Private workers** — a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
+- **Two IAM roles:**
+  - A **cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
+  - A **node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
+
+## Instructions
+
+The same cluster can be created three ways. Pick your tool — each path reaches the same working cluster.
+
+:::tabs
+@tab AWS CLI
+
+### 1. Create the cluster
+
+Point the cluster at your VPC subnets and the cluster role, with API authentication and a public endpoint:
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws eks create-cluster \
+  --name demo \
+  --role-arn arn:aws:iam::000000000000:role/eks-cluster-role \
+  --resources-vpc-config subnetIds=subnet-aaaa,subnet-bbbb,endpointPublicAccess=true \
+  --access-config authenticationMode=API \
+  --kubernetes-version 1.32
+```
+
+Wait for it to become active:
+
+```bash
+aws eks wait cluster-active --name demo
+aws eks describe-cluster --name demo --query 'cluster.status'
+```
+
+### 2. Add a node group
+
+```bash
+aws eks create-nodegroup \
+  --cluster-name demo \
+  --nodegroup-name default \
+  --node-role-arn arn:aws:iam::000000000000:role/eks-node-role \
+  --subnets subnet-aaaa subnet-bbbb \
+  --scaling-config minSize=1,maxSize=2,desiredSize=1 \
+  --instance-types t3.medium
+
+aws eks wait nodegroup-active --cluster-name demo --nodegroup-name default
+```
+
+### 3. Connect kubectl
+
+```bash
+aws eks update-kubeconfig --name demo
+kubectl get nodes
+```
+
+### 4. Expose a workload
+
+Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
+
+```bash
+SG=$(aws ec2 describe-security-groups \
+  --filters Name=group-name,Values=eks-cluster-demo-nodegroup-sg \
+  --query 'SecurityGroups[0].GroupId' --output text)
+
+aws ec2 authorize-security-group-ingress \
+  --group-id "$SG" --protocol tcp --port 30080 --cidr 0.0.0.0/0
+```
+
+Then deploy and publish your app:
+
+```bash
+kubectl create deployment hello --image=nginxdemos/hello --replicas=2
+kubectl expose deployment hello --type=NodePort --port=80 \
+  --overrides='{"spec":{"ports":[{"port":80,"nodePort":30080}]}}'
+```
+
+@tab Spinifex UI
+
+The Spinifex console drives the same workflow through a guided form. From the left navigation open **EKS → Clusters**.
+
+<img src="eks-list-clusters.png" alt="The EKS Clusters list in the Spinifex console, with a Create Cluster button">
+
+### 1. Create the cluster
+
+1. Click **Create Cluster**.
+2. Enter a **name** and choose the **Kubernetes version** (default `1.32`).
+3. Choose the **cluster IAM role**, or create one inline from the dialog.
+4. Select the **VPC and subnets** the cluster will run in.
+5. Set **endpoint access** (public, private, or both) and submit.
+
+<img src="eks-create-cluster.png" alt="The Create Cluster form in the Spinifex console, showing name, Kubernetes version, IAM role, VPC, subnets, and endpoint access fields">
+
+The form notes that the authentication mode is API and that the legacy aws-auth ConfigMap is not supported. The cluster appears in the **Clusters** list as `CREATING` and flips to `ACTIVE` once the control plane is healthy.
+
+### 2. Add a node group
+
+1. Open the cluster and switch to the **Node groups** tab.
+2. Click **Add node group**, choose the **node IAM role**, **subnets**, **instance type**, and **scaling** (desired / min / max).
+3. Submit and wait for the node group to reach `ACTIVE`.
+
+### 3. Manage access and addons
+
+- The **Access** tab lists IAM principals and their cluster-access policies — add an access entry to let another user or role reach the cluster.
+- The **Addons** tab installs and shows the health of managed addons.
+- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed security groups.
+
+To run `kubectl` against the cluster, follow the AWS CLI tab's `update-kubeconfig` step.
+
+@tab Terraform
+
+The Terraform workbooks are the fastest way to a working cluster — they build the VPC, IAM roles, cluster, node group, and a demo workload in one `apply`. Start with **eks-quickstart**:
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/eks-quickstart
+```
+
+The cluster config points the AWS provider at Spinifex's `eks`, `ec2`, `iam`, and `sts` endpoints, then creates the cluster and node group:
+
+```hcl
+resource "aws_eks_cluster" "this" {
+  name     = "eks-quickstart"
+  role_arn = aws_iam_role.cluster.arn
+  version  = "1.32"
+
+  access_config {
+    authentication_mode = "API"
+  }
+
+  vpc_config {
+    subnet_ids              = [aws_subnet.a.id, aws_subnet.b.id]
+    endpoint_public_access  = true
+  }
+}
+
+resource "aws_eks_node_group" "default" {
+  cluster_name    = aws_eks_cluster.this.name
+  node_group_name = "default"
+  node_role_arn   = aws_iam_role.node.arn
+  subnet_ids      = [aws_subnet.a.id, aws_subnet.b.id]
+
+  scaling_config {
+    desired_size = 1
+    min_size     = 1
+    max_size     = 2
+  }
+
+  instance_types = ["t3.medium"]
+}
+```
+
+Apply it, then connect:
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply
+aws eks update-kubeconfig --name eks-quickstart
+kubectl get nodes
+```
+
+The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-ingress** adds private subnets, a NAT gateway, and HTTPS via the AWS Load Balancer Controller + ACM; **eks-gitops-argocd** delivers the app from a git repo with the Argo CD addon and persists state on an EBS-CSI (Viperblock-backed) volume. See the full templates linked in Resources.
+
+:::
+
+## Troubleshooting
+
+**Cluster stuck in `CREATING`.** The control plane is still booting. Check `aws eks describe-cluster --name <name> --query 'cluster.health'` for reported issues. The control plane needs egress to bootstrap, so confirm its subnet has an Internet Gateway route.
+
+**`kubectl` cannot connect.** Re-run `aws eks update-kubeconfig --name <name>` to refresh the endpoint and CA, and confirm the cluster is `ACTIVE`. `aws sts get-caller-identity` verifies your credentials reach the Spinifex endpoint.
+
+**Workers stay `NotReady`.** Almost always an egress or IAM problem:
+- Confirm the workers can pull images — a default route to an Internet Gateway (public) or NAT Gateway (private).
+- Confirm the node role has `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly`.
+
+**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
+
+**`create-cluster` rejected with an authentication-mode error.** `authentication_mode` must be `API`. Remove any `CONFIG_MAP` / `API_AND_CONFIG_MAP` setting and grant access with access entries instead.
+
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
@@ -57,9 +57,9 @@ GPU passthrough requires a host kernel and OS that supports vfio-pci. Ubuntu 24.
 ## Prerequisites

 - Supermicro H14 with 8× AMD Instinct MI350X installed
- Host OS: **Ubuntu 24.04 LTS** (kernel 6.8+) or **Debian 13** (kernel 6.12+) — minimum for vfio-pci support
+- Host OS: **Ubuntu 26.04 LTS** (kernel 6.8+) or **Debian 13** (kernel 6.12+) — minimum for vfio-pci support
 - Spinifex installed and all services running (`systemctl status spinifex.target`)
- AMD GPU AMI registered (`ami-ubuntu-amd-gpu`) — Ubuntu 26.04 LTS with ROCm-compatible kernel
+- AMD GPU AMI registered (`ubuntu-26.04-amd-gpu-x86_64`) — Ubuntu 26.04 LTS with ROCm-compatible kernel
 - AWS CLI configured with `AWS_PROFILE=spinifex` pointing at the Spinifex endpoint
 - SSH key pair imported into Spinifex, VPC and security group created (see [Launching Instances](/docs/launching-instances))

@@ -0,0 +1,556 @@
+---
+title: "Mixed AI Workloads on a Single H200 Chassis: Guest-Managed MIG"
+description: "Install Spinifex from source, configure host-local networking, attach Predastore storage, and run four concurrent AI workloads across MIG partitions managed inside guest VMs."
+category: "Reference Architectures"
+tags:
+  - nvidia
+  - h200
+  - mig
+  - vllm
+  - yolo
+  - predastore
+  - bare-metal
+resources:
+  - title: "Install from Source"
+    url: "/docs/install-source"
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
+  - title: "NVIDIA MIG User Guide"
+    url: "https://docs.nvidia.com/datacenter/tesla/mig-user-guide/"
+sections:
+  - overview
+  - prerequisites
+  - instructions
+---
+
+## Overview
+
+Spinifex is an open-source infrastructure platform that brings core AWS services including EC2, EBS and S3 to bare-metal, edge, and on-prem environments. It exposes an AWS compatible API, so any tooling that works against AWS (the `aws` CLI, Terraform, SDKs) works against a Spinifex node unchanged, with a single profile swap.
+
+This guide documents a full bare-metal AI deployment on a single [Supermicro X13](https://www.supermicro.com/en/products/x13) chassis using NVIDIA H200 SXM GPUs. These GPUs include the [NVIDIA Multi-Instance GPU](https://www.nvidia.com/en-au/technologies/multi-instance-gpu/) (MIG) capability, which allows a single GPU to be "sliced" into up to seven independent GPU partitions that are hardware isolated. Each partition is capable of running its own workloads with reserved resources from the "host" GPU.
+
+<p align="center"><img src="../../../.github/assets/images/h200/X13.png" alt="Supermicro X13 8U GPU System"></p>
+
+In this setup, each EC2 instance receives an entire H200 via PCIe passthrough and manages its own MIG partitions. This gives each tenant full control over how they slice their GPU — including the ability to run heterogeneous workloads at different partition sizes on the same physical card.
+
+A key consideration for this example is that we do not have access to the router upstream of the X13 host, or knowledge of potentially available IP addresses for the VMs we provision on it. The first step in the Instructions section outlines how we attach our own local IP address range to the bridge on the host, effectively creating a pseudo-airgapped environment - although in this specific case outbound connectivity is still required for remote access to the host, downloading required packages etc, with this configuration it is *not* required for host/EC2 instance communication.
+
+### EC2 workload layout
+
+| EC2 Workload | IP | MIG config | Workload | Model |
+|---|---|---|---|---|
+| `vm-llama3b` | 192.168.10.X  | 7 × 1g.18gb | Chat inference × 7 | Llama-3.2-3B-Instruct |
+| `vm-qwen32b` | 192.168.10.X  | 2 × 3g.71gb | Chat inference × 2 | Qwen2.5-32B-Instruct |
+| `vm-llama70b` | 192.168.10.X | 1 × 7g.141gb | Chat inference × 1 | Llama-3.1-70B-FP8 |
+| `vm-yolo`    | 192.168.10.X | 2 × 3g.71gb | Object detection × 2 | YOLO11x + YOLO11s |
+
+Twelve concurrent inference endpoints in total: 7 fast 3B slots, 2 mid-tier 32B slots, 1 full-GPU 70B slot, and 2 real-time vision streams running side-by-side to compare detection models.
+
+
+### Platform
+
+| Component | Specification |
+|---|---|
+| **Chassis** | Supermicro X13 8U GPU System |
+| **CPUs** | 2× Intel Xeon Platinum 8568Y+ (48 cores each, 96 cores / 192 threads total) |
+| **RAM** | 2 TB DDR5-4800 (32× 64 GB SK Hynix DIMMs) |
+| **Storage** | 4× 7.68 TB KIOXIA CD6 NVMe SSDs (30.72 TB raw) |
+| **GPUs** | 8× NVIDIA H200 SXM5 (141 GiB HBM3e per GPU, ~1.13 TB total) — 4 used in this demo |
+| **Host OS** | Ubuntu 26.04 LTS |
+| **Orchestration** | Spinifex — EC2-compatible bare-metal API |
+| **Guest OS** | Ubuntu 26.04 LTS |
+| **GPU partitioning** | NVIDIA MIG — managed inside each guest VM |
+| **Block storage** | Predastore — S3-compatible, NVMe-backed |
+| **Container runtime** | Docker |
+| **Inference runtime** | vLLM (`vllm-openai` image from local registry) |
+| **Vision runtime** | Ultralytics YOLO11 |
+
+
+## Prerequisites
+
+### 1. Verify GPU visibility on host
+Before making any changes, verify that the host can see its GPUs:
+
+<img src="../../../.github/assets/images/h200/nvidia-smi.png" alt="NVIDIA SMI">
+
+
+## Instructions
+
+
+### 1. Configure host-local VPC networking
+
+Spinifex utilises bridged networking via OVN. In this example, the remote host exposes only one public IP address, attached to a single physical NIC. Before provisioning Spinifex, we must first ensure that `br-wan` exists and enslave the physical NIC to it.
+
+Since we have no access to an upstream router or knowledge of available IP address pools, we must also attach our own `192.168.10.0/24` address range to `br-wan` which will be used by the guest VMs to communicate with the host and each other.
+
+For example:
+
+```yaml
+# /etc/netplan/…
+bridges:
+  br-wan:
+    addresses:
+      - 192.168.10.1/24      # VM pool gateway — host-local
+      - 198.51.100.10/24     # existing WAN IP — unchanged
+    routes:
+      - to: default
+        via: 198.51.100.1
+```
+
+Then apply with `sudo netplan apply`. VMs, once provisioned, are reachable from the host at
+`192.168.10.x`. For internet access through the host's WAN interface:
+
+```bash
+sysctl -w net.ipv4.ip_forward=1
+iptables -t nat -A POSTROUTING -s 192.168.10.0/24 -o br-wan -j MASQUERADE
+```
+The exact process for this is described in the [VPC Networking](/docs/vpc-networking#host-local-subnet-no-upstream-router) guide.
+
+
+
+### 2. Install Spinifex
+
+Follow the [Single Node Install](/docs/install) guide. This process will install Spinifex and start Spinifex services, however `spinifex.toml` needs to be edited to finalise the changes made to the networking in the previous section, as described in the following section.
+
+### 3. Configure spinifex.toml and restart services
+
+Edit `/etc/spinifex/spinifex.toml` to point the external pool and VPCD at the bridge created in step 1:
+
+```toml
+[network]
+external_mode = "pool"
+
+[[network.external_pools]]
+name        = "wan"
+source      = "static"         # required — no upstream DHCP for this range
+range_start = "192.168.10.2"
+range_end   = "192.168.10.100"
+gateway     = "192.168.10.1"   # second address on br-wan
+prefix_len  = 24
+dns_servers = ["8.8.8.8"]
+```
+
+Then restart all services:
+
+```bash
+sudo systemctl start spinifex.target
+sudo systemctl status spinifex.target
+```
+
+### 4. Attach Predastore storage
+
+Spinifex distributes the Predastore object storage volume across multiple nodes using Reed–Solomon encoding for data redundancy. This chassis has four NVMe drives — one for the OS, three dedicated to data — so we back each Predastore storage node with its own physical drive, giving us fault-tolerant distributed storage on a single machine — conceptually similar to RAID 5, where data and parity are spread across drives so a single drive failure is recoverable.
+
+Confirm drive assignments with `lsblk` before proceeding, as device names vary between systems.
+
+```bash
+lsblk   # identify the OS drive and the three data drives
+
+# Stop services so Predastore isn't writing while we relocate its data directories
+sudo systemctl stop spinifex.target
+
+# --- Repeat the block below for each data drive (node-1/nvme-1, node-2/nvme-2, node-3/nvme-3) ---
+
+# Mount the physical drive at a stable path
+sudo mkdir -p /mnt/nvme-1
+sudo mount /dev/nvme1n1 /mnt/nvme-1
+sudo mkdir -p /mnt/nvme-1/nodes /mnt/nvme-1/db
+
+# Move Predastore's node data and metadata off the OS drive onto the physical NVMe
+sudo mv /var/lib/spinifex/predastore/distributed/nodes/node-1 /mnt/nvme-1/nodes/node-1
+sudo mv /var/lib/spinifex/predastore/distributed/db/node-1    /mnt/nvme-1/db/node-1
+
+# Symlink the original paths back so Predastore finds its data unchanged
+sudo ln -s /mnt/nvme-1/nodes/node-1 /var/lib/spinifex/predastore/distributed/nodes/node-1
+sudo ln -s /mnt/nvme-1/db/node-1    /var/lib/spinifex/predastore/distributed/db/node-1
+
+# Persist the mount across reboots
+echo "/dev/nvme1n1  /mnt/nvme-1  auto  defaults  0  2" | sudo tee -a /etc/fstab
+
+# --- End of per-drive block ---
+
+sudo systemctl start spinifex.target
+```
+
+Verify the nodes are healthy before proceeding:
+
+```bash
+export AWS_PROFILE=spinifex
+
+aws s3 ls --endpoint-url https://localhost:8443
+# Should return without error (empty bucket list is fine)
+```
+
+### 5. Enable GPU Passthrough
+Spinifex allows GPUs to be utilised by guest VMS via PCIe-passthrough. This can be enabled via `sudo spx admin gpu setup`. Then, after a reboot, run `sudo spx admin gpu enable`. The Spinifex banner should update to reflect GPU passthrough state after every step:
+
+<p align="center"><img src="../../../.github/assets/images/h200/spinifex-banner1.png" alt="GPU Enabled"></p>
+
+
+### 6. Import the GPU AMI
+
+The demo uses the standard Spinifex NVIDIA GPU AMI (`ubuntu-26.04-nvidia-gpu-x86_64`), which includes:
+
+- Ubuntu 26.04 LTS guest image
+- NVIDIA server driver (DKMS pre-built against the pinned kernel)
+- Docker CE + nvidia-container-toolkit (`--gpus` support enabled at boot)
+- Python 3 + venv, common utilities (tmux, curl, ffmpeg, etc.)
+
+```bash
+spx admin images import --name ubuntu-26.04-nvidia-gpu-x86_64
+```
+
+Confirm and set the AMI ID:
+```bash
+AMI_ID=$(aws ec2 describe-images \
+    --filters "Name=name,Values=ubuntu-26.04-nvidia-gpu-x86_64" \
+    --query 'Images[0].ImageId' --output text)
+```
+
+### 7. Create VPC resources
+
+```bash
+# Create VPC and subnet
+VPC_ID=$(aws ec2 create-vpc --cidr-block 192.168.10.0/24 \
+    --query 'Vpc.VpcId' --output text)
+SUBNET_ID=$(aws ec2 create-subnet --vpc-id $VPC_ID \
+    --cidr-block 192.168.10.0/24 \
+    --query 'Subnet.SubnetId' --output text)
+
+# Security group — allow SSH + all vLLM/YOLO ports
+SG_ID=$(aws ec2 create-security-group \
+    --group-name demo-sg --description "H200 MIG demo" \
+    --vpc-id $VPC_ID --query 'GroupId' --output text)
+
+aws ec2 authorize-security-group-ingress --group-id $SG_ID \
+    --protocol tcp --port 22 --cidr 0.0.0.0/0
+aws ec2 authorize-security-group-ingress --group-id $SG_ID \
+    --protocol tcp --port 8000-8011 --cidr 0.0.0.0/0
+```
+
+```bash
+# Create SSH key
+aws ec2 create-key-pair --key-name spinifex-key \
+  | jq -r '.KeyMaterial | rtrimstr("\n")' > ~/.ssh/spinifex-key
+
+chmod 600 ~/.ssh/spinifex-key
+ssh-keygen -y -f ~/.ssh/spinifex-key > ~/.ssh/spinifex-key.pub
+```
+
+Verify:
+
+```bash
+aws ec2 describe-key-pairs
+```
+
+### 8. Launch the four VMs
+
+Each VM gets a whole H200 via PCIe passthrough. The `p5e.4xlarge` instance type maps one H200 per VM:
+
+```bash
+aws ec2 run-instances \
+  --image-id $AMI_ID \
+  --instance-type p5e.4xlarge \
+  --key-name spinifex-key \
+  --subnet-id $SUBNET_ID \
+  --security-group-ids $SG_ID \
+  --count 4
+```
+
+Wait until all four reach running state, then take note of the public IPs assigned to each instance:
+
+```bash
+aws ec2 describe-instances \
+    --query 'Reservations[*].Instances[*].[InstanceId,PublicIpAddress,State.Name]' \
+    --output table
+```
+
+### 9. Enable MIG inside each VM
+
+MIG is enabled inside each guest VM, not on the host. SSH into each VM and enable it:
+
+```bash
+# Repeat on each VM IP
+ssh -i ~/.ssh/spinifex-key ec2-user@<vm-ip>
+
+sudo nvidia-smi -mig 1
+nvidia-smi | grep "MIG M."
+# Should show: MIG M.  Enabled
+```
+
+### 10. Create MIG partitions inside each VM
+
+With MIG enabled, we are now able to partition each VM's assigned H200 GPU into several separate GPU instances. This process assigns each GPU slice its own UUID, so each VM goes from seeing one whole GPU to seeing a number of "MIG devices":
+
+<p align="center"><img src="../../../.github/assets/images/h200/mig-activated.png" alt="NVIDIA SMI"></p>
+
+This allows us to run several separate workloads, each assigned to its own GPU instance, on the same physical GPU, and thus utilise more of the overall GPU's resources.
+
+Run `nvidia-smi mig -lgip` to see the available partition types. MIG profiles are named Xg.Ygb, where:
+
+* X is the number of GPU slices (GPU Instances, or GIs) allocated to the partition. On an H200 there are 7 allocatable GPU slices, so the largest profile is 7g.
+* Y is the amount of HBM memory allocated to that partition.
+
+Larger profiles also receive proportionally more SMs (Streaming Multiprocessors - analagous to CPU cores), cache, copy engines, encoders/decoders, and other GPU resources.
+
+Thus we partition the GPUs assigned to our VMs as follows:
+
+**vm-llama3b — 7 × 1g.18gb (Llama 3B):**
+```bash
+sudo nvidia-smi mig -cgi 1g.18gb,1g.18gb,1g.18gb,1g.18gb,1g.18gb,1g.18gb,1g.18gb -C
+nvidia-smi -L  # Verify 7 MIG devices
+```
+
+**vm-qwen32b — 2 × 3g.71gb (Qwen 32B):**
+```bash
+sudo nvidia-smi mig -cgi 3g.71gb,3g.71gb -C
+nvidia-smi -L  # Verify 2 MIG devices
+```
+
+**vm-llama70b — 1 × 7g.141gb (Llama 70B, full GPU):**
+```bash
+sudo nvidia-smi mig -cgi 7g.141gb -C
+nvidia-smi -L  # Verify 1 MIG device
+```
+
+**vm-yolo — 2 × 3g.71gb (YOLO):**
+```bash
+sudo nvidia-smi mig -cgi 3g.71gb,3g.71gb -C
+nvidia-smi -L  # Verify 2 MIG devices
+```
+
+Each MIG device gets a UUID of the form `MIG-xxxxxxxx-...`. These UUIDs are used to pin individual containers to their slice via `--gpus "device=<UUID>"` (for Docker) or `CUDA_VISIBLE_DEVICES=<UUID>` (for direct Python processes).
+
+### 11. Configure Docker to use the local image registry
+
+The vLLM image is served from the host's local Docker registry (`192.168.10.1:5000`).
+
+Setting up the local registry on the host:
+
+```bash
+# Configure Docker to trust the local registry address
+sudo tee /etc/docker/daemon.json > /dev/null <<'EOF'
+{
+  "insecure-registries": ["192.168.10.1:5000"]
+}
+EOF
+sudo systemctl restart docker
+
+# Start the registry container, bound to the bridge interface only
+docker run -d \
+    -p 192.168.10.1:5000:5000 \
+    --name registry \
+    --restart=always \
+    registry:2
+
+# Pull the vLLM image and push it into the local registry
+docker pull vllm/vllm-openai:latest
+docker tag vllm/vllm-openai:latest 192.168.10.1:5000/vllm-openai:latest
+docker push 192.168.10.1:5000/vllm-openai:latest
+
+# Verify the image is available
+curl http://192.168.10.1:5000/v2/_catalog
+# {"repositories":["vllm-openai"]}
+```
+
+Each VM then needs to trust it as an insecure registry:
+
+```bash
+# On each VM — vm-yolo uses direct Python, not Docker for YOLO
+sudo tee /etc/docker/daemon.json > /dev/null <<'EOF'
+{
+  "insecure-registries": ["192.168.10.1:5000"]
+}
+EOF
+sudo systemctl restart docker
+```
+
+### 12. Deploy the LLM workloads
+
+Each LLM is downloaded using the Huggingface CLI onto the respective VM. First install the CLI:
+
+```bash
+pip install -U "huggingface_hub[cli]"
+```
+
+Then download the model and determine its snapshot directory. This directory contains the model’s config.json, tokenizer files, and weight files, and will be used as MODEL_DIR when launching vLLM.
+
+```bash
+#On vm-llama3b
+hf download meta-llama/Llama-3.2-3B-Instruct
+
+MODEL_DIR=$(dirname "$(find ~/.cache/huggingface/hub -path '*Llama-3.2-3B-Instruct*' -name config.json | head -1)")
+echo "$MODEL_DIR"
+
+#On vm-qwen32b
+hf download Qwen/Qwen2.5-32B-Instruct
+
+MODEL_DIR=$(dirname "$(find ~/.cache/huggingface/hub -path '*Qwen2.5-32B-Instruct*' -name config.json | head -1)")
+echo "$MODEL_DIR"
+
+#On vm-llama70b
+hf download RedHatAI/Meta-Llama-3.1-70B-Instruct-FP8
+
+MODEL_DIR=$(dirname "$(find ~/.cache/huggingface/hub -path '*Meta-Llama-3.1-70B-Instruct-FP8*' -name config.json | head -1)")
+echo "$MODEL_DIR"
+```
+
+Then we run the docker containers as follows:
+
+**vm-llama3b — 7 × Llama-3.2-3B-Instruct (one container per MIG slice):**
+
+Enumerate the MIG UUIDs and start one vLLM container per slice:
+
+
+```bash
+mapfile -t UUIDS < <(nvidia-smi -L | grep -oP 'MIG-[0-9a-f-]+')
+for i in "${!UUIDS[@]}"; do
+    docker run -d --rm \
+        --name "vllm-$i" \
+        --gpus "device=${UUIDS[$i]}" \
+        --ipc host \
+        -p $((8000 + i)):8000 \
+        -v "${MODEL_DIR}:/models:ro" \
+        192.168.10.1:5000/vllm-openai:latest \
+        vllm serve /models \
+            --served-model-name llama-3b \
+            --dtype bfloat16 \
+            --max-model-len 4096 \
+            --gpu-memory-utilization 0.90 \
+            --port 8000
+done
+```
+
+**vm-qwen32b — 2 × Qwen2.5-32B-Instruct:**
+
+```bash
+mapfile -t UUIDS < <(nvidia-smi -L | grep -oP 'MIG-[0-9a-f-]+')
+for i in "${!UUIDS[@]}"; do
+    docker run -d --rm \
+        --name "qwen32b-$i" \
+        --gpus "device=${UUIDS[$i]}" \
+        --ipc host \
+        -p $((8000 + i)):8000 \
+        -v "${MODEL_DIR}:/models:ro" \
+        192.168.10.1:5000/vllm-openai:latest \
+        vllm serve /models \
+            --served-model-name qwen2.5-32b \
+            --dtype bfloat16 \
+            --max-model-len 4096 \
+            --gpu-memory-utilization 0.90 \
+            --port 8000
+done
+```
+
+**vm-llama70b — Llama-3.1-70B-Instruct-FP8 (full GPU slice):**
+
+```bash
+UUID=$(nvidia-smi -L | grep -oP 'MIG-[0-9a-f-]+' | head -1)
+docker run -d --rm \
+    --name vllm \
+    --gpus "device=${UUID}" \
+    --ipc host \
+    -p 8000:8000 \
+    -v "${MODEL_DIR}:/models:ro" \
+    192.168.10.1:5000/vllm-openai:latest \
+    vllm serve /models \
+        --served-model-name meta-llama-3.1-70b \
+        --dtype auto \
+        --max-model-len 8192 \
+        --gpu-memory-utilization 0.90 \
+        --port 8000
+```
+
+### 13. Deploy YOLO object detection (vm-yolo)
+
+For this demo we built a simple YOLO inference server that runs detection on a looping video file and serves the annotated output as an MJPEG stream. The same script runs twice on vm-yolo — once for YOLO11x (larger, more accurate) and once for YOLO11s (smaller, faster) — each pinned to its own MIG slice and listening on a different port.
+
+The MIG pinning works through `CUDA_VISIBLE_DEVICES`: setting it to a MIG UUID before launch scopes the process to that slice, and CUDA remaps it to device index 0 inside the process. From the script's perspective it always sees one GPU at index 0 — the MIG boundary is invisible to it. `YOLO_MODEL` and `PORT` are what actually differentiate the two instances:
+
+```bash
+python3 -m venv ~/yolo-venv
+~/yolo-venv/bin/pip install ultralytics fastapi "uvicorn[standard]" opencv-python-headless httpx
+
+# Get MIG UUIDs
+nvidia-smi -L
+# GPU 0: NVIDIA H200 (UUID: GPU-...)
+#   MIG 3g.71gb  Device 0: (UUID: MIG-<uuid0>)
+#   MIG 3g.71gb  Device 1: (UUID: MIG-<uuid1>)
+
+# YOLO11x on slice 0 — larger model, port 8010
+CUDA_VISIBLE_DEVICES=MIG-<uuid0> \
+  VIDEO_PATH=<path-to-video> \
+  YOLO_MODEL=yolo11x.pt \
+  YOLO_DEVICE=0 \
+  PORT=8010 \
+  ~/yolo-venv/bin/python ~/yolo_stream.py >> ~/yolo-x.log 2>&1 &
+
+# YOLO11s on slice 1 — smaller/faster model, port 8011
+CUDA_VISIBLE_DEVICES=MIG-<uuid1> \
+  VIDEO_PATH=<path-to-video> \
+  YOLO_MODEL=yolo11s.pt \
+  YOLO_DEVICE=0 \
+  PORT=8011 \
+  ~/yolo-venv/bin/python ~/yolo_stream.py >> ~/yolo-s.log 2>&1 &
+```
+
+Each instance exposes a `/video` endpoint serving a `multipart/x-mixed-replace` MJPEG stream, consumable directly by browsers and most HTTP clients. YOLO11x (~75 MB) and YOLO11s (~9 MB) weights download automatically on first run from the Ultralytics model hub.
+
+### 14. Dashboard
+
+We also built a simple host-side dashboard — another FastAPI application that proxies all the VM streams to the browser so only one port on the host needs to be exposed. Each VM endpoint is wired in by address at startup, mapping directly to the IPs assigned in step 8:
+
+```
+http://vm-yolo-IP:8010  →  vm-yolo  YOLO11x MJPEG stream
+http://vm-yolo-IP:8011  →  vm-yolo  YOLO11s MJPEG stream
+http://vm-llama3b-IP:8000   →  vm-llama3b  vLLM endpoint 0
+http://vm-llama3b-IP:8001   →  vm-llama3b  vLLM endpoint 1
+... (one entry per endpoint across all three LLM VMs)
+```
+
+The dashboard has two proxy patterns: MJPEG passthrough for the YOLO feeds (forwarding the raw boundary stream from vm-yolo to the browser), and SSE passthrough for the LLM token streams (subscribing to each vLLM endpoint and re-emitting tokens as server-sent events). Both use a reconnect loop so the browser connection stays open if a VM is temporarily unreachable.
+
+
+
+<p><video src="https://iso.mulgadc.com/h200-demo.mp4" controls width="100%" style="border-radius:6px"></video></p>
+
+The dashboard shows:
+- GPU allocation bars for all four EC2 instancess (proportional to MIG slice size)
+- Live streaming LLM responses per endpoint, colour-coded by tier
+- Side-by-side YOLO11x vs YOLO11s video feeds with FPS and detection counts
+
+The dashboard also displays the overall GPU utilisation, derived as a percentage of maximum power usage. Importantly, it demonstrates how Spinifex combined with NVIDIA's MIG capability enables the deployment of multiple heterogeneous workloads on owned hardware.
+
+### 15. Teardown
+
+```bash
+# On vm-yolo:
+# Stop YOLO processes
+pkill -f yolo_stream.py
+
+# On each LLM instance:
+# Stop docker containers
+
+docker stop $(docker ps -q)
+
+# Disable MIG inside each VM before terminating.
+sudo nvidia-smi mig -dci
+sudo nvidia-smi mig -dgi
+sudo nvidia-smi -mig 0
+
+# On the host:
+# Terminate all four instances — releases 4× H200 back to the Spinifex pool
+aws ec2 terminate-instances --instance-ids \
+    $(aws ec2 describe-instances \
+        --filters "Name=instance-state-name,Values=running" \
+        --query 'Reservations[*].Instances[*].InstanceId' \
+        --output text)
+```
+
+The four H200s are immediately returned to the host GPU pool once the instances terminate, ready for reallocation.
+
+### 16. Conclusion
+
+This document highlights how Spinifex can turn a single bare-metal chassis into a multi-tenant AI serving platform. Spinifex utilises the flexibility of PCIe passthrough combined with NVIDIA's MIG capability to allocate GPU resources in whichever configuration is required by the workload/s. This flexibility and fine-grain control ensures maximum GPU utilisation.
+
+Importantly, it does so with standard `aws ec2` CLI calls — `run-instances`, `describe-instances`, `terminate-instances` — against Spinifex's EC2-compatible endpoint. Teams already operating AWS infrastructure can point their existing tooling at a Spinifex node with a single profile change, against GPUs that sit in their own rack.
@@ -284,6 +284,22 @@ suites:
    # into per-test fixture-driven coverage so the selector can pick
    # subtests, not just whole suites.

+  e2e-iam:
+    path: tests/e2e/iam
+    covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
+    # "Everything IAM": the IAM/STS control plane plus the instance-identity
+    # tests (instance-profile association + IMDS), which boot guest VMs — OVN
+    # via vpcd, AMI via predastore, root volume via viperblockd, lifecycle via
+    # spinifex_daemon. Single-mode, VM-booting leg.
+
+  e2e-ecr:
+    path: tests/e2e/ecr
+    covers: [awsgw, spinifex_daemon, predastore]
+    # ECR control plane (handlers/ecr, gateway/ecr*) + OCI data plane: blobs and
+    # manifests in predastore, control + auth bridge in awsgw, KV via
+    # spinifex_daemon. docker/crane/skopeo data-plane subtests skip when the
+    # client binary or registry-host resolution is absent.
+
  e2e-ddil:
    path: tests/e2e/ddil
    covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
@@ -0,0 +1,4 @@
+README.md
+*.tfstate
+*.tfstate.backup
+.terraform
@@ -0,0 +1,17 @@
+# Spinifex EKS demo app — multi-stage build to a tiny distroless image.
+#
+# Stage 1 compiles a static binary; stage 2 ships only that binary on a
+# distroless base (no shell, no package manager) for a small, production-shaped
+# image suitable for pushing to ECR and pulling onto EKS workers.
+
+FROM golang:1.23-alpine AS build
+WORKDIR /src
+COPY go.mod ./
+COPY main.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/spinifex-demo .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/spinifex-demo /spinifex-demo
+EXPOSE 8080
+USER nonroot:nonroot
+ENTRYPOINT ["/spinifex-demo"]
@@ -0,0 +1,69 @@
+# Spinifex EKS demo app
+
+A small, Spinifex-themed web app shared by the EKS Terraform workbooks. It
+replaces `nginxdemos/hello` with something production-shaped that you build
+yourself and push to **ECR**, then pull onto your EKS workers.
+
+The page reports the pod, node, cluster, and region that answered the request,
+plus a hit counter — refresh and watch requests land on different pods across a
+multi-node cluster.
+
+## What it does
+
+- `GET /` — themed HTML page.
+- `GET /api/info` — JSON of the same facts.
+- `GET /healthz` — liveness/readiness probe (`200 ok`), used by the ALB target
+  group health check.
+
+All configuration is read from the environment, so one image serves every tier:
+
+| Variable | Purpose | Source |
+|---|---|---|
+| `POD_NAME`, `NODE_NAME`, `POD_NAMESPACE` | identity shown on the page | downward API |
+| `CLUSTER_NAME`, `AWS_REGION` | cluster/region labels | deployment env |
+| `APP_TITLE` | page heading | deployment env (default `Spinifex EKS`) |
+| `PORT` | listen port | default `8080` |
+| `DATA_DIR` | when set, the hit counter is persisted here and survives pod restarts | mounted **EBS-CSI PVC** in `eks-gitops-argocd` |
+
+When `DATA_DIR` points at a writable volume the page shows a **"persisted to EBS
+volume"** badge — the gitops workbook mounts a Viperblock-backed PersistentVolume
+there to demonstrate durable storage.
+
+## Build and push to ECR
+
+The image is named `spinifex-demo`. Build it once and push it to a repository in
+your Spinifex ECR; every workbook references it by its ECR URI.
+
+```bash
+export AWS_PROFILE=spinifex
+REGION=ap-southeast-2
+
+# 1. Create the repository (the workbooks also create it via Terraform; skip if
+#    it already exists).
+aws ecr create-repository --repository-name spinifex-demo --region "$REGION"
+
+# 2. Resolve the registry host and authenticate Docker.
+REGISTRY=$(aws ecr describe-repositories --repository-names spinifex-demo \
+  --region "$REGION" --query 'repositories[0].repositoryUri' --output text)
+REGISTRY_HOST=${REGISTRY%%/*}
+aws ecr get-login-password --region "$REGION" \
+  | docker login --username AWS --password-stdin "$REGISTRY_HOST"
+
+# 3. Build, tag, and push.
+docker build -t spinifex-demo:latest .
+docker tag spinifex-demo:latest "${REGISTRY}:latest"
+docker push "${REGISTRY}:latest"
+
+echo "Image: ${REGISTRY}:latest"
+```
+
+Pass the resulting `${REGISTRY}:latest` value to a workbook as the `demo_image`
+variable (or accept its default, which resolves the URI from the ECR repository
+the workbook creates).
+
+## Run locally
+
+```bash
+go run .            # http://localhost:8080
+DATA_DIR=/tmp/demo go run .   # exercise the persisted-counter path
+```
@@ -0,0 +1,3 @@
+module github.com/mulgadc/spinifex-demo
+
+go 1.23
@@ -0,0 +1,214 @@
+// Command spinifex-demo is the demo web app served by the EKS Terraform
+// workbooks. It renders a Spinifex-themed page reporting the pod, node, cluster,
+// and region that answered the request, plus a hit counter.
+//
+// The counter is held in memory by default. When DATA_DIR points at a writable
+// directory (an EBS-CSI PersistentVolume in the gitops workbook), the counter is
+// persisted there and survives pod restarts — demonstrating Viperblock-backed
+// durable storage. All configuration comes from the environment so a single
+// image serves every workbook tier.
+package main
+
+import (
+	"encoding/json"
+	"html/template"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// instance captures the per-pod facts surfaced on the page, populated from the
+// downward API and the deployment environment.
+type instance struct {
+	Pod       string
+	Node      string
+	Namespace string
+	Cluster   string
+	Region    string
+	Title     string
+	Persisted bool
+}
+
+func env(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// counter tracks page hits, optionally persisting to dir/hits so the count
+// survives a pod restart when backed by a PersistentVolume.
+type counter struct {
+	mu   sync.Mutex
+	n    int64
+	path string
+}
+
+func newCounter(dir string) *counter {
+	c := &counter{}
+	if dir == "" {
+		return c
+	}
+	c.path = filepath.Join(dir, "hits")
+	if b, err := os.ReadFile(c.path); err == nil {
+		if n, perr := strconv.ParseInt(string(b), 10, 64); perr == nil {
+			c.n = n
+		}
+	}
+	return c
+}
+
+func (c *counter) inc() int64 {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.n++
+	if c.path != "" {
+		// Best-effort persist; a read-only or missing volume must not 500 the page.
+		if err := os.WriteFile(c.path, []byte(strconv.FormatInt(c.n, 10)), 0o644); err != nil {
+			slog.Warn("persist counter failed", "path", c.path, "err", err)
+		}
+	}
+	return c.n
+}
+
+func main() {
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+	slog.SetDefault(logger)
+
+	dataDir := os.Getenv("DATA_DIR")
+	hits := newCounter(dataDir)
+
+	base := instance{
+		Pod:       env("POD_NAME", "unknown"),
+		Node:      env("NODE_NAME", "unknown"),
+		Namespace: env("POD_NAMESPACE", "default"),
+		Cluster:   env("CLUSTER_NAME", "spinifex-eks"),
+		Region:    env("AWS_REGION", env("AWS_DEFAULT_REGION", "ap-southeast-2")),
+		Title:     env("APP_TITLE", "Spinifex EKS"),
+		Persisted: dataDir != "",
+	}
+
+	tmpl := template.Must(template.New("page").Parse(pageHTML))
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	})
+
+	mux.HandleFunc("/api/info", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"pod":       base.Pod,
+			"node":      base.Node,
+			"namespace": base.Namespace,
+			"cluster":   base.Cluster,
+			"region":    base.Region,
+			"hits":      hits.inc(),
+			"persisted": base.Persisted,
+		})
+	})
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		view := struct {
+			instance
+			Hits int64
+			Now  string
+		}{base, hits.inc(), time.Now().UTC().Format("2006-01-02 15:04:05 MST")}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		if err := tmpl.Execute(w, view); err != nil {
+			slog.Error("render failed", "err", err)
+		}
+	})
+
+	addr := ":" + env("PORT", "8080")
+	slog.Info("spinifex-demo listening", "addr", addr, "cluster", base.Cluster, "persisted", base.Persisted)
+	srv := &http.Server{
+		Addr:              addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+	if err := srv.ListenAndServe(); err != nil {
+		slog.Error("server exited", "err", err)
+		os.Exit(1)
+	}
+}
+
+const pageHTML = `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{.Title}}</title>
+<style>
+  :root {
+    --bg: #0b0e14; --panel: #131823; --line: #232a39;
+    --fg: #e6edf3; --muted: #8b97a7; --accent: #f5b301; --accent2: #3ddc97;
+  }
+  * { box-sizing: border-box; }
+  body {
+    margin: 0; min-height: 100vh; background:
+      radial-gradient(1200px 600px at 80% -10%, rgba(245,179,1,.10), transparent 60%),
+      radial-gradient(900px 500px at -10% 110%, rgba(61,220,151,.08), transparent 55%), var(--bg);
+    color: var(--fg); font: 15px/1.55 ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, sans-serif;
+    display: flex; align-items: center; justify-content: center; padding: 32px;
+  }
+  .card {
+    width: 100%; max-width: 640px; background: var(--panel);
+    border: 1px solid var(--line); border-radius: 16px; overflow: hidden;
+    box-shadow: 0 20px 60px rgba(0,0,0,.45);
+  }
+  .head { padding: 28px 32px; border-bottom: 1px solid var(--line); display: flex; align-items: center; gap: 16px; }
+  .logo { height: 44px; width: auto; flex: 0 0 auto; }
+  .head h1 { margin: 0; font-size: 20px; letter-spacing: .2px; }
+  .head p { margin: 2px 0 0; color: var(--muted); font-size: 13px; }
+  .body { padding: 24px 32px 8px; }
+  .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 12px 20px; }
+  .kv { padding: 12px 14px; background: #0e131d; border: 1px solid var(--line); border-radius: 10px; }
+  .kv .k { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .08em; }
+  .kv .v { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 14px; word-break: break-all; margin-top: 3px; }
+  .hits { grid-column: 1 / -1; display: flex; align-items: baseline; justify-content: space-between; }
+  .hits .n { font-size: 30px; font-weight: 700; color: var(--accent); font-family: ui-monospace, monospace; }
+  .foot { padding: 16px 32px 24px; color: var(--muted); font-size: 12px; display: flex; justify-content: space-between; gap: 12px; }
+  .pill { color: var(--accent2); border: 1px solid rgba(61,220,151,.35); border-radius: 999px; padding: 2px 10px; font-size: 11px; }
+</style>
+</head>
+<body>
+  <div class="card">
+    <div class="head">
+      <svg class="logo" viewBox="0 0 364 463" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
+        <path d="M272.523 171.205C247.773 197.595 211.323 215.815 182.103 237.425C165.663 249.575 122.633 281.165 151.093 301.825C185.533 326.835 271.333 304.985 301.013 277.585C317.193 262.645 316.693 242.485 294.703 234.075C270.913 224.975 243.483 231.925 219.353 235.755C218.543 235.885 216.823 236.805 216.993 235.345C227.053 228.885 236.103 220.925 245.933 214.135C255.403 207.585 267.053 198.225 278.643 198.015C314.693 197.345 351.803 215.135 361.203 252.215C376.833 313.885 303.333 362.255 249.833 366.635L280.913 352.235C292.563 345.305 304.353 338.105 314.653 329.215C317.213 327.005 319.073 324.145 321.243 322.035C321.753 321.535 327.053 319.265 324.003 319.125C296.253 337.335 263.793 349.415 231.323 356.335C220.173 358.715 208.323 360.995 196.963 361.285C197.243 362.845 198.643 362.525 199.663 362.845C208.823 365.795 219.313 367.545 228.843 369.005C230.163 370.005 228.733 370.415 227.683 370.555C219.063 371.665 209.923 373.305 201.293 373.745C176.843 374.985 152.203 372.435 128.573 366.425L115.293 328.755C114.153 328.865 114.073 329.895 113.783 330.725C112.613 334.015 107.893 358.025 106.313 358.365C93.6234 354.515 82.2334 345.915 72.9034 336.575L95.0434 296.915L59.7134 316.375C54.7434 305.145 53.8334 292.435 55.4934 280.355L101.213 257.645C86.9934 256.705 72.6134 258.955 58.4734 260.425C60.8734 253.495 63.8034 246.615 67.3634 240.195C68.6634 237.855 75.5634 226.305 77.3834 226.245C93.9334 229.415 111.773 230.895 128.363 227.675L92.3534 212.825L91.0334 211.045C98.3834 204.125 105.843 197.235 114.113 191.325C129.603 195.825 145.523 199.455 161.733 199.655L130.633 179.885C134.133 177.125 137.813 174.575 141.553 172.165C144.643 170.175 153.283 164.425 156.423 164.835C168.933 169.105 181.923 172.125 195.123 173.185C189.093 166.935 177.633 162.325 171.833 156.465C171.203 155.825 170.713 155.785 171.003 154.585C186.153 145.885 203.003 137.235 216.053 125.375C229.363 113.265 243.423 93.4848 233.853 75.2048C244.893 78.7148 254.393 92.6648 254.153 104.325C253.493 136.595 219.073 165.375 197.483 185.615C230.833 171.615 268.993 142.685 268.923 102.735C268.863 70.2248 239.263 57.1948 211.283 54.2348C210.103 61.3848 207.573 68.9448 203.863 75.1748C195.623 89.0148 178.803 97.6648 170.013 112.055C164.023 121.865 162.883 135.065 150.163 138.825C144.973 140.365 138.293 140.595 132.903 140.615L144.743 134.365C158.003 124.625 157.773 107.025 165.893 93.9248C164.953 92.5948 155.413 90.0748 153.183 89.5248C143.363 87.1048 126.713 84.0248 116.883 84.6948C113.843 84.9048 100.143 87.0348 98.9634 89.3748C97.2634 92.7648 105.533 107.255 107.973 110.575C108.463 111.245 109.463 111.075 109.513 111.145C109.973 111.745 109.663 113.165 108.373 112.675C107.443 112.315 101.853 108.205 100.593 107.235C90.8934 99.7648 79.8234 88.4748 71.7234 79.3648C70.4034 76.1748 68.6334 72.6148 71.7634 69.9848C88.8634 59.4048 105.793 48.3148 123.163 38.1948C174.603 8.20482 233.143 -25.6952 282.353 29.2448C323.153 74.8048 311.503 129.615 272.493 171.195L272.523 171.205ZM114.153 67.5948C127.613 65.2648 153.013 78.6248 157.193 58.8348C157.603 56.9048 156.493 55.7948 157.583 53.9148C158.403 52.5048 163.083 49.1048 164.713 47.7848C169.903 43.6048 175.823 40.3348 180.923 36.0148C161.113 39.2048 141.163 50.5648 123.853 60.5948C120.983 62.2548 117.873 63.9948 115.153 65.8548C114.453 66.3348 113.713 65.7348 114.143 67.5848L114.153 67.5948ZM21.1634 276.415C-11.8666 312.635 -4.35661 364.395 30.6734 396.135C65.5434 427.725 120.403 438.925 166.373 436.725C207.813 434.735 249.893 421.075 290.773 434.965C306.663 440.365 320.533 450.465 332.363 462.135C330.773 454.415 326.123 446.365 321.623 439.825C280.593 380.225 200.893 402.565 140.123 390.665C107.163 384.215 72.8934 369.135 52.8234 341.225C35.4734 317.105 32.7734 288.665 41.3634 260.475C39.6134 258.725 23.0334 274.355 21.1634 276.415ZM143.743 109.535C136.373 116.045 128.573 122.235 120.403 127.775L97.1634 139.135C116.843 139.885 137.193 128.635 143.753 109.545L143.743 109.535Z" fill="#f5b301"/>
+      </svg>
+      <div>
+        <h1>{{.Title}}</h1>
+        <p>Served by a pod on your Spinifex-managed Kubernetes cluster</p>
+      </div>
+    </div>
+    <div class="body">
+      <div class="grid">
+        <div class="kv"><div class="k">Pod</div><div class="v">{{.Pod}}</div></div>
+        <div class="kv"><div class="k">Node</div><div class="v">{{.Node}}</div></div>
+        <div class="kv"><div class="k">Cluster</div><div class="v">{{.Cluster}}</div></div>
+        <div class="kv"><div class="k">Region</div><div class="v">{{.Region}}</div></div>
+        <div class="kv hits">
+          <div><div class="k">Requests served</div>{{if .Persisted}}<span class="pill">persisted to EBS volume</span>{{end}}</div>
+          <div class="n">{{.Hits}}</div>
+        </div>
+      </div>
+    </div>
+    <div class="foot">
+      <span>Namespace: {{.Namespace}}</span>
+      <span>{{.Now}}</span>
+    </div>
+  </div>
+</body>
+</html>`
@@ -1,181 +0,0 @@
---
-title: "EKS with Addons + Access-Entry RBAC"
-description: "Extend a single-node EKS cluster with a managed CoreDNS addon, access-entry RBAC, and a browsable load-balanced demo app, using Terraform on Spinifex."
-category: "Terraform Workbooks"
-tags:
-  - terraform
-  - eks
-  - kubernetes
-  - addons
-  - rbac
-  - iam
-  - workbook
-resources:
-  - title: "Terraform AWS Provider"
-    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
-  - title: "Terraform Kubernetes Provider"
-    url: "https://registry.terraform.io/providers/hashicorp/kubernetes/latest"
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
-  - title: "OpenTofu"
-    url: "https://opentofu.org/"
---
-
-# Terraform: EKS with Addons + Access-Entry RBAC
-
-> Builds on the quickstart by installing a managed addon (CoreDNS) and granting a second IAM principal read-only cluster access — then deploys the same browsable, load-balanced demo app so you can see the result.
-
-## Table of Contents
-
- [Overview](#overview)
- [Instructions](#instructions)
- [Troubleshooting](#troubleshooting)
-
---
-
-## Overview
-
-This workbook takes the same single-node cluster (and the same browsable demo app) from [EKS Quickstart](../eks-quickstart) and adds the two things you reach for on almost any real cluster:
-
-1. **A managed addon.** CoreDNS is installed through the EKS API (`aws_eks_addon`) rather than `kubectl apply`, so its lifecycle is managed by the cluster. The demo app is deployed *after* CoreDNS, so a healthy page also confirms in-cluster DNS is working.
-2. **Access-entry RBAC.** A second IAM role is granted read-only access via an `aws_eks_access_entry` plus an `aws_eks_access_policy_association` bound to the AWS-managed `AmazonEKSViewPolicy`. This is the modern access-entry path (`authentication_mode = "API"`) — no `aws-auth` ConfigMap.
-
-When `apply` finishes, open the **`demo_url`** output and refresh — the `nginxdemos/hello` page reports which pod served each request.
-
-**What you'll learn:**
-
- Installing a managed EKS addon with `aws_eks_addon`
- Registering an IAM principal with `aws_eks_access_entry` and scoping it with `aws_eks_access_policy_association`
- The difference between the cluster creator's admin access and a scoped viewer
- Deploying a workload with the Kubernetes provider once the addon is healthy
-
-**What gets created**
-
-| Resource | Name | Purpose |
-|---|---|---|
-| VPC | `eks-addons-vpc` | Isolated network (10.31.0.0/16) |
-| Subnets | `eks-addons-subnet-a/-b` | Public subnets for the cluster and worker |
-| Internet Gateway | `eks-addons-igw` | Egress so the worker can pull the demo image |
-| IAM Roles | `eks-addons-cluster-role`, `eks-addons-node-role` | Control-plane and worker roles |
-| IAM Role | `eks-addons-viewer` | Stand-in principal granted read-only access |
-| EKS Cluster | `eks-addons` | Public endpoint, API auth mode |
-| Node Group | `default` | One `t3.medium` worker |
-| Addon | `coredns` | Cluster DNS, managed via the EKS API |
-| Access Entry + Assoc. | `eks-addons-viewer` → `AmazonEKSViewPolicy` | Read-only, cluster scope |
-| SG Ingress Rule | `eks-addons-demo-nodeport` | Opens the demo NodePort on the worker SG |
-| K8s Deployment + Service | `hello` | `nginxdemos/hello`, 2 replicas, NodePort |
-
-**Spinifex specifics**
-
- Addon catalog is limited. Spinifex currently serves `coredns`, `aws-load-balancer-controller` and `spinifex-noop`. There is no `vpc-cni`, `kube-proxy`, `ebs-csi` or `metrics-server`.
- **Leave `addon_version` unset.** Spinifex defaults an unset version to its catalog default (`coredns` 1.11.1). Pinning is a dead end: the AWS provider rejects a bare `1.11.1` (it requires the `v`-prefixed `v1.11.1-eksbuild.N` form), while Spinifex's catalog only accepts the bare `1.11.1` — so no single string satisfies both.
- Cluster access policies are the four AWS-managed `cluster-access-policy/Amazon EKS{ClusterAdmin,Admin,Edit,View}Policy` ARNs.
- The worker SG is auto-managed and the worker AMI is always the `eks-node` image; the NodePort is exposed by adding one rule to the looked-up worker SG.
-
-**Prerequisites:**
-
- Spinifex installed and running
- The Spinifex `eks-node` image available on the cluster
- OpenTofu or Terraform, plus `kubectl` and the AWS CLI
-
-## Instructions
-
-### Step 1. Get the Template
-
-```bash
-git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
-cd spinifex-tf
-git sparse-checkout set docs/terraform-workbooks
-cd docs/terraform-workbooks/eks-addons-access
-```
-
-Or create a `main.tf` file and paste the full configuration below.
-
-<!-- INCLUDE: main.tf lang:hcl -->
-
-### Step 2. Deploy
-
-```bash
-export AWS_PROFILE=spinifex
-tofu init
-tofu apply
-```
-
-Allow a few minutes for the cluster to reach `ACTIVE` and the worker to register. CoreDNS installs onto the worker, then Terraform deploys the demo app.
-
-> Keep `AWS_PROFILE=spinifex` exported for the whole `apply` — the Kubernetes provider runs `aws eks get-token` against the Spinifex STS endpoint.
-
-### Step 3. Open the Demo
-
-```bash
-tofu output demo_url
-```
-
-Open it in a browser and refresh — the `nginxdemos/hello` page shows the pod that answered, alternating between the two replicas.
-
-### Step 4. Verify the Addon and Access Entry
-
-```bash
-aws eks list-addons --cluster-name eks-addons
-aws eks describe-addon --cluster-name eks-addons --addon-name coredns --query 'addon.status'
-
-aws eks list-access-entries --cluster-name eks-addons
-aws eks list-associated-access-policies \
-  --cluster-name eks-addons \
-  --principal-arn "$(tofu output -raw viewer_principal_arn)"
-```
-
-The viewer principal is bound to `AmazonEKSViewPolicy` at cluster scope — it can read across the cluster but cannot mutate anything. The principal that ran `apply` keeps full cluster-admin from `bootstrap_cluster_creator_admin_permissions`.
-
-### Cleanup
-
-```bash
-tofu destroy
-```
-
-## Troubleshooting
-
-### Demo URL Doesn't Load
-
-Same chain as the quickstart — check the worker's public IP, that the demo pods are `Running`, and that the NodePort rule landed on the worker SG:
-
-```bash
-kubectl get pods -o wide
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-addons-nodegroup-sg" \
-  --query 'SecurityGroups[0].IpPermissions'
-```
-
-If the demo pods are stuck `ContainerCreating` with DNS errors, confirm CoreDNS is healthy first:
-
-```bash
-kubectl -n kube-system get deploy coredns
-```
-
-### Addon Stuck in CREATING / DEGRADED
-
-CoreDNS needs a schedulable worker. Confirm the node group is `ACTIVE` and the node is `Ready`:
-
-```bash
-aws eks describe-nodegroup --cluster-name eks-addons --nodegroup-name default --query 'nodegroup.status'
-kubectl get nodes
-```
-
-### Addon Version Rejected
-
-This workbook leaves `addon_version` unset so Spinifex picks its catalog default. If you add an explicit `addon_version`, expect a wall: the AWS provider requires the `v`-prefixed `v1.11.1-eksbuild.N` form, but Spinifex's catalog only knows the bare `1.11.1` — a value that passes one is rejected by the other. Prefer omitting the field.
-
-### Access Entry Not Taking Effect
-
-Assume the viewer role and confirm it can read but not write:
-
-```bash
-kubectl auth can-i list pods --all-namespaces   # yes
-kubectl auth can-i create deployments           # no
-```
-
-### Provider Connection Refused
-
-```bash
-sudo systemctl status spinifex.target
-curl -k https://localhost:9999/
-```
@@ -1,468 +0,0 @@
-# Example: EKS with an Addon + Access-Entry RBAC (with a visible demo app)
-#
-# Builds on eks-quickstart: same single-worker cluster and browsable demo app,
-# but adds two things you reach for on a real cluster:
-#
-#   * a managed addon (CoreDNS) installed through the EKS API, and
-#   * a second IAM principal granted scoped, read-only access via an EKS access
-#     entry + access-policy association (authentication_mode = "API").
-#
-# After `apply`, open the demo_url output — the page reports the pod that
-# served it, so a refresh shows requests landing on different replicas. CoreDNS
-# being healthy is what lets in-cluster name resolution work.
-#
-# Usage:
-#   cd spinifex/docs/terraform-workbooks/eks-addons-access
-#   export AWS_PROFILE=spinifex
-#   tofu init && tofu apply
-#   # then open the demo_url output
-
-terraform {
-  required_version = ">= 1.6.0"
-
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 5.40, < 6.0"
-    }
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = ">= 2.20"
-    }
-  }
-}
-
-# ---------------------------------------------------------------------------
-# Variables
-# ---------------------------------------------------------------------------
-
-variable "region" {
-  type    = string
-  default = "ap-southeast-2"
-}
-
-variable "cluster_name" {
-  type    = string
-  default = "eks-addons"
-}
-
-variable "k8s_version" {
-  type    = string
-  default = "1.32"
-}
-
-variable "instance_type" {
-  type    = string
-  default = "t3.medium"
-}
-
-variable "node_port" {
-  type    = number
-  default = 30080
-}
-
-variable "replicas" {
-  type    = number
-  default = 2
-}
-
-variable "browse_cidr" {
-  type        = string
-  default     = "0.0.0.0/0"
-  description = "CIDR allowed to reach the demo NodePort; tighten to your own IP in production"
-}
-
-variable "spinifex_endpoint" {
-  type    = string
-  default = "https://127.0.0.1:9999"
-}
-
-# ---------------------------------------------------------------------------
-# Providers
-# ---------------------------------------------------------------------------
-
-provider "aws" {
-  region = var.region
-
-  endpoints {
-    ec2 = var.spinifex_endpoint
-    iam = var.spinifex_endpoint
-    sts = var.spinifex_endpoint
-    eks = var.spinifex_endpoint
-  }
-
-  skip_credentials_validation = true
-  skip_metadata_api_check     = true
-  skip_requesting_account_id  = true
-  skip_region_validation      = true
-}
-
-provider "kubernetes" {
-  host                   = aws_eks_cluster.this.endpoint
-  cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
-
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    command     = "aws"
-    args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
-  }
-}
-
-data "aws_availability_zones" "available" {
-  state = "available"
-}
-
-# ---------------------------------------------------------------------------
-# VPC + two public subnets
-# ---------------------------------------------------------------------------
-
-resource "aws_vpc" "main" {
-  cidr_block           = "10.31.0.0/16"
-  enable_dns_hostnames = true
-  enable_dns_support   = true
-
-  tags = {
-    Name = "${var.cluster_name}-vpc"
-  }
-}
-
-resource "aws_internet_gateway" "igw" {
-  vpc_id = aws_vpc.main.id
-
-  tags = {
-    Name = "${var.cluster_name}-igw"
-  }
-}
-
-resource "aws_subnet" "a" {
-  vpc_id                  = aws_vpc.main.id
-  cidr_block              = "10.31.1.0/24"
-  availability_zone       = data.aws_availability_zones.available.names[0]
-  map_public_ip_on_launch = true
-
-  tags = {
-    Name = "${var.cluster_name}-subnet-a"
-  }
-}
-
-resource "aws_subnet" "b" {
-  vpc_id                  = aws_vpc.main.id
-  cidr_block              = "10.31.2.0/24"
-  availability_zone       = data.aws_availability_zones.available.names[0]
-  map_public_ip_on_launch = true
-
-  tags = {
-    Name = "${var.cluster_name}-subnet-b"
-  }
-}
-
-resource "aws_route_table" "public" {
-  vpc_id = aws_vpc.main.id
-
-  route {
-    cidr_block = "0.0.0.0/0"
-    gateway_id = aws_internet_gateway.igw.id
-  }
-
-  tags = {
-    Name = "${var.cluster_name}-public-rt"
-  }
-}
-
-resource "aws_route_table_association" "a" {
-  subnet_id      = aws_subnet.a.id
-  route_table_id = aws_route_table.public.id
-}
-
-resource "aws_route_table_association" "b" {
-  subnet_id      = aws_subnet.b.id
-  route_table_id = aws_route_table.public.id
-}
-
-# ---------------------------------------------------------------------------
-# IAM — cluster role
-# ---------------------------------------------------------------------------
-
-resource "aws_iam_role" "cluster" {
-  name = "${var.cluster_name}-cluster-role"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect    = "Allow"
-      Action    = "sts:AssumeRole"
-      Principal = { Service = "eks.amazonaws.com" }
-    }]
-  })
-}
-
-resource "aws_iam_role_policy_attachment" "cluster" {
-  role       = aws_iam_role.cluster.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
-}
-
-# ---------------------------------------------------------------------------
-# IAM — node role
-# ---------------------------------------------------------------------------
-
-resource "aws_iam_role" "node" {
-  name = "${var.cluster_name}-node-role"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect    = "Allow"
-      Action    = "sts:AssumeRole"
-      Principal = { Service = "ec2.amazonaws.com" }
-    }]
-  })
-}
-
-resource "aws_iam_role_policy_attachment" "node_worker" {
-  role       = aws_iam_role.node.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
-}
-
-resource "aws_iam_role_policy_attachment" "node_cni" {
-  role       = aws_iam_role.node.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
-}
-
-resource "aws_iam_role_policy_attachment" "node_ecr" {
-  role       = aws_iam_role.node.name
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
-}
-
-# ---------------------------------------------------------------------------
-# IAM — a second principal to grant read-only cluster access to
-# ---------------------------------------------------------------------------
-
-resource "aws_iam_role" "viewer" {
-  name = "${var.cluster_name}-viewer"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect    = "Allow"
-      Action    = "sts:AssumeRole"
-      Principal = { Service = "ec2.amazonaws.com" }
-    }]
-  })
-}
-
-# ---------------------------------------------------------------------------
-# EKS cluster
-# ---------------------------------------------------------------------------
-
-resource "aws_eks_cluster" "this" {
-  name     = var.cluster_name
-  role_arn = aws_iam_role.cluster.arn
-  version  = var.k8s_version
-
-  vpc_config {
-    subnet_ids              = [aws_subnet.a.id, aws_subnet.b.id]
-    endpoint_public_access  = true
-    endpoint_private_access = false
-  }
-
-  access_config {
-    authentication_mode                         = "API"
-    bootstrap_cluster_creator_admin_permissions = true
-  }
-
-  depends_on = [aws_iam_role_policy_attachment.cluster]
-
-  tags = {
-    Name = var.cluster_name
-  }
-}
-
-# ---------------------------------------------------------------------------
-# Node group
-# ---------------------------------------------------------------------------
-
-resource "aws_eks_node_group" "default" {
-  cluster_name    = aws_eks_cluster.this.name
-  node_group_name = "default"
-  node_role_arn   = aws_iam_role.node.arn
-  subnet_ids      = [aws_subnet.a.id, aws_subnet.b.id]
-
-  scaling_config {
-    desired_size = 1
-    min_size     = 1
-    max_size     = 2
-  }
-
-  instance_types = [var.instance_type]
-  ami_type       = "AL2_x86_64"
-
-  depends_on = [
-    aws_iam_role_policy_attachment.node_worker,
-    aws_iam_role_policy_attachment.node_cni,
-    aws_iam_role_policy_attachment.node_ecr,
-  ]
-
-  tags = {
-    Name = "${var.cluster_name}-default"
-  }
-}
-
-# ---------------------------------------------------------------------------
-# Addon — CoreDNS
-# ---------------------------------------------------------------------------
-
-resource "aws_eks_addon" "coredns" {
-  cluster_name                = aws_eks_cluster.this.name
-  addon_name                  = "coredns"
-  resolve_conflicts_on_create = "OVERWRITE"
-
-  # addon_version omitted on purpose. Spinifex defaults an unset version to its
-  # catalog default (coredns 1.11.1). The AWS provider rejects a bare "1.11.1"
-  # (it demands a v-prefixed form) which Spinifex's catalog would in turn
-  # reject — so pinning is a dead end here; let the server choose.
-  depends_on = [aws_eks_node_group.default]
-
-  tags = {
-    Name = "${var.cluster_name}-coredns"
-  }
-}
-
-# ---------------------------------------------------------------------------
-# Access entry — grant the viewer principal read-only access
-# ---------------------------------------------------------------------------
-
-resource "aws_eks_access_entry" "viewer" {
-  cluster_name  = aws_eks_cluster.this.name
-  principal_arn = aws_iam_role.viewer.arn
-  type          = "STANDARD"
-}
-
-resource "aws_eks_access_policy_association" "viewer" {
-  cluster_name  = aws_eks_cluster.this.name
-  principal_arn = aws_iam_role.viewer.arn
-  policy_arn    = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
-
-  access_scope {
-    type = "cluster"
-  }
-
-  depends_on = [aws_eks_access_entry.viewer]
-}
-
-# ---------------------------------------------------------------------------
-# Open the NodePort on the auto-managed nodegroup SG (see eks-quickstart)
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = var.browse_cidr
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-demo-nodeport"
-  }
-}
-
-# ---------------------------------------------------------------------------
-# Demo workload — deployed after CoreDNS so name resolution is up
-# ---------------------------------------------------------------------------
-
-resource "kubernetes_deployment_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-    labels    = { app = "hello" }
-  }
-
-  spec {
-    replicas = var.replicas
-
-    selector {
-      match_labels = { app = "hello" }
-    }
-
-    template {
-      metadata {
-        labels = { app = "hello" }
-      }
-
-      spec {
-        container {
-          name  = "hello"
-          image = "nginxdemos/hello"
-
-          port {
-            container_port = 80
-          }
-        }
-      }
-    }
-  }
-
-  depends_on = [aws_eks_addon.coredns]
-}
-
-resource "kubernetes_service_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-  }
-
-  spec {
-    selector = { app = "hello" }
-    type     = "NodePort"
-
-    port {
-      port        = 80
-      target_port = 80
-      node_port   = var.node_port
-    }
-  }
-
-  depends_on = [kubernetes_deployment_v1.hello]
-}
-
-data "aws_instances" "workers" {
-  instance_tags = {
-    "spinifex:eks-cluster" = aws_eks_cluster.this.name
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
-# ---------------------------------------------------------------------------
-# Outputs
-# ---------------------------------------------------------------------------
-
-output "cluster_name" {
-  value = aws_eks_cluster.this.name
-}
-
-output "viewer_principal_arn" {
-  value = aws_iam_role.viewer.arn
-}
-
-output "demo_url" {
-  value       = "http://${data.aws_instances.workers.public_ips[0]}:${var.node_port}"
-  description = "Open in a browser; refresh to see different pods answer"
-}
-
-output "update_kubeconfig" {
-  value = "aws eks update-kubeconfig --name ${aws_eks_cluster.this.name} --region ${var.region}"
-}
@@ -0,0 +1,284 @@
+---
+title: "GitOps on EKS (Argo CD + EBS-CSI)"
+description: "Deliver a Spinifex-themed app to EKS with GitOps — the Argo CD addon syncs it from a git repo, an EBS-CSI (Viperblock) PersistentVolume holds its state, and it is served over HTTPS through the AWS Load Balancer Controller + ACM, using Terraform on Spinifex."
+category: "Terraform Workbooks"
+tags:
+  - terraform
+  - eks
+  - kubernetes
+  - argocd
+  - gitops
+  - ebs-csi
+  - storage
+  - workbook
+resources:
+  - title: "Argo CD"
+    url: "https://argo-cd.readthedocs.io/"
+  - title: "AWS Load Balancer Controller"
+    url: "https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/"
+  - title: "Terraform AWS Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
+  - title: "Terraform Kubernetes Provider"
+    url: "https://registry.terraform.io/providers/hashicorp/kubernetes/latest"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+---
+
+# Terraform: GitOps on EKS (Argo CD + EBS-CSI)
+
+> The top rung of the EKS ladder. It extends [EKS HTTPS Ingress](../eks-https-ingress) and changes how the app is delivered: the **Argo CD** addon syncs a more elaborate Spinifex-themed app from a **git repo**, an **EBS-CSI** PersistentVolume (Viperblock-backed) holds its state, and it is still served over **HTTPS** through the AWS Load Balancer Controller + ACM.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Instructions](#instructions)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+This workbook keeps everything from [EKS HTTPS Ingress](../eks-https-ingress) — the public/private VPC, the NAT gateway, the LBC addon, the ACM cert, the HTTPS Ingress — and adds the parts you reach for once an app is real: **GitOps delivery** and **persistent storage**.
+
+Three changes over the previous rung:
+
+1. **Argo CD delivers the app.** Instead of Terraform applying the Deployment, the **argocd** addon syncs the app from a git repository ([`mulgadc/eks-demo-app`](https://github.com/mulgadc/eks-demo-app)). Terraform only registers the repo credential and creates the Argo CD `Application`; Argo CD reconciles the manifests and self-heals drift.
+2. **State lives on an EBS volume.** The **aws-ebs-csi-driver** addon ships a default gp3 StorageClass. The app's `PersistentVolumeClaim` (in the git repo) dynamically provisions a **Viperblock-backed EBS volume**; the demo's hit counter persists to it and survives pod restarts.
+3. **Access-entry RBAC.** A second IAM principal is granted read-only cluster access via an access entry bound to `AmazonEKSViewPolicy`.
+
+Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress; **Argo CD** manages the app's Deployment, Service, and PVC from git. The Ingress points at the Service Argo CD creates, so the app is reachable over HTTPS the moment Argo CD finishes its first sync.
+
+**What you'll learn:**
+
+- Installing the `argocd` and `aws-ebs-csi-driver` addons through the EKS API
+- Registering a private git repo with Argo CD and driving an `Application` from Terraform
+- Dynamically provisioning a Viperblock-backed EBS volume with a `PersistentVolumeClaim`
+- Splitting ownership: Terraform owns infra + Ingress, Argo CD owns the workload
+- Granting scoped read-only access with an EKS access entry
+
+**What gets created**
+
+| Resource | Name | Purpose |
+|---|---|---|
+| VPC + subnets | `eks-gitops-*` | Public/private network (10.32.0.0/16) with a NAT gateway |
+| IAM Roles | `eks-gitops-cluster-role`, `-node-role`, `-viewer` | Control-plane, worker, and read-only viewer roles |
+| ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
+| EKS Cluster | `eks-gitops` | Public + private endpoints; `managed-ingress=false` |
+| Node Group | `workers` | `node_desired_size` `t3.large` worker(s) — 1 or 3 |
+| Addons | `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver` | Ingress, GitOps delivery, persistent storage |
+| ACM Certificate | `eks-gitops-ingress` | Self-signed, imported; on the HTTPS listener |
+| Access Entry + Assoc. | `eks-gitops-viewer` → `AmazonEKSViewPolicy` | Read-only, cluster scope |
+| Argo CD repo Secret | `eks-demo-app-repo` | Credential for the private git repo |
+| Argo CD Application | `spinifex-demo` | Syncs the app from git |
+| K8s Ingress | `spinifex-demo` | `ingressClassName: alb`, HTTPS via the ACM cert |
+| Git-managed (by Argo CD) | Deployment, Service, **PVC** | The app + its Viperblock-backed volume |
+
+**Spinifex specifics**
+
+- The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
+- The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
+- **Verify dynamic provisioning on a live cluster.** EBS-CSI dynamic provisioning depends on the running Spinifex EBS/Viperblock data plane and may not be available on every build yet. If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — confirm the volume binds on a stack where the driver is healthy.
+- Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
+- The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
+
+**Prerequisites:**
+
+- Spinifex running with an `eks-node` image carrying the LBC, Argo CD, and EBS-CSI bundles
+- OpenTofu or Terraform, plus `kubectl`, the AWS CLI, and Docker
+- The demo image built and pushed to ECR (see [`../demo-app/README.md`](../demo-app/README.md))
+- A git repo with the app manifests ([`mulgadc/eks-demo-app`](https://github.com/mulgadc/eks-demo-app)); a read-only PAT if it is private
+
+## Instructions
+
+### Step 1. Get the Template
+
+```bash
+git clone --depth 1 --filter=blob:none --sparse https://github.com/mulgadc/spinifex.git spinifex-tf
+cd spinifex-tf
+git sparse-checkout set docs/terraform-workbooks
+cd docs/terraform-workbooks/eks-gitops-argocd
+```
+
+<!-- INCLUDE: main.tf lang:hcl -->
+
+### Step 2. Deploy the Cluster + Addons
+
+```bash
+export AWS_PROFILE=spinifex
+tofu init
+tofu apply        # add -var node_desired_size=3 for an HA-shaped cluster
+```
+
+Wait for the cluster to reach `ACTIVE` and for all three addons (`aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`) to report healthy:
+
+```bash
+aws eks list-addons --cluster-name eks-gitops
+aws eks describe-addon --cluster-name eks-gitops --addon-name argocd --query 'addon.status'
+```
+
+### Step 3. Build and Push the Demo Image
+
+```bash
+cd ../demo-app
+REGISTRY=$(cd ../eks-gitops-argocd && tofu output -raw ecr_repository_url)
+REGISTRY_HOST=${REGISTRY%%/*}
+aws ecr get-login-password | docker login --username AWS --password-stdin "$REGISTRY_HOST"
+docker build -t "${REGISTRY}:latest" .
+docker push "${REGISTRY}:latest"
+cd ../eks-gitops-argocd
+```
+
+Point the app's manifests at this image: set the image ref in `eks-demo-app/manifests` (a kustomize `images:` override) to your `${REGISTRY}:latest`, and push that to the git repo.
+
+### Step 4. Hand Delivery to Argo CD
+
+```bash
+cd workloads
+tofu init
+tofu apply \
+  -var git_repo_url=https://github.com/mulgadc/eks-demo-app.git \
+  -var git_token=<read-only-PAT>     # omit for a public repo
+```
+
+This registers the repo credential, creates the Argo CD `Application`, the demo app's HTTPS Ingress, and an HTTPS Ingress for the Argo CD UI itself. Argo CD then syncs the Deployment, Service, and PVC from git.
+
+<!-- INCLUDE: workloads/main.tf lang:hcl -->
+
+### Step 5. Watch the Sync and Open the App
+
+The demo app and the Argo CD UI share **one ALB** (an LBC IngressGroup). On
+`:443` the app is the catch-all and `argocd.eks-gitops.spinifex.local`
+host-routes to the UI; Argo CD also gets a hostless `:8443` listener so it is
+reachable on the raw ALB IP before DNS exists. Grab the shared ALB address once:
+
+```bash
+# Point kubectl at the cluster first (writes ~/.kube/config) — without this every
+# kubectl call fails to connect:
+aws eks update-kubeconfig --name eks-gitops --region ap-southeast-2
+
+kubectl -n argocd get applications
+kubectl get pods -o wide
+kubectl get pvc                    # the EBS-CSI volume should be Bound
+
+DNSNAME=$(kubectl get ingress spinifex-demo -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+ALB_IP=$(aws elbv2 describe-load-balancers \
+  --query "LoadBalancers[?DNSName=='${DNSNAME}'].AvailabilityZones[].LoadBalancerAddresses[].IpAddress | [0]" \
+  --output text)
+echo "$ALB_IP"
+```
+
+**Reach it by raw IP (no DNS, no host header):**
+
+| Service | URL |
+|---|---|
+| Demo app | `https://<ALB_IP>/` |
+| Argo CD UI | `https://<ALB_IP>:8443/` |
+
+```bash
+curl -k https://"$ALB_IP"/            # app — returns the Spinifex page
+curl -k https://"$ALB_IP":8443/       # Argo CD UI
+```
+
+The bare ALB IP serves the **app** on `:443`. Hitting `https://<ALB_IP>/` and
+expecting Argo CD gives the app instead — use `:8443` for the UI. Once northstar
+(or Route 53) resolves the hostnames to the ALB, `app.eks-gitops.spinifex.local`
+and `argocd.eks-gitops.spinifex.local` both work on `:443`. Self-signed cert —
+accept the browser warning.
+
+The page shows the **"persisted to EBS volume"** badge and a hit counter that keeps climbing — delete the pod (`kubectl delete pod -l app=spinifex-demo`) and the count survives, proving the volume is durable.
+
+### Step 6. Open the Argo CD UI
+
+Managing deployments through the Argo CD console is the point of this workbook, so
+the UI is exposed on the **same ALB** as the app, not behind a port-forward. The
+`workloads` module adds a `NodePort` Service in front of `argocd-server` (the addon
+ships it `ClusterIP` only) and two Ingresses in the shared group — a host-routed
+one on `:443` and a hostless one on `:8443` for raw-IP access. `argocd-server`
+serves TLS on its own port, so both use `backend-protocol: HTTPS`.
+
+**Get the admin credentials.** Argo CD generates a one-time `admin` password into
+a Secret on install; the username is always `admin`:
+
+```bash
+kubectl -n argocd get secret argocd-initial-admin-secret \
+  -o jsonpath='{.data.password}' | base64 -d ; echo
+```
+
+(`tofu output argocd_admin_password_cmd` prints this same command.)
+
+Open the UI and log in as `admin`:
+
+- By raw IP: `https://<ALB_IP>:8443/`
+- By DNS once wired: `https://argocd.eks-gitops.spinifex.local`
+
+The `spinifex-demo` Application shows the sync status, the resource tree, and live
+diffs against git — change a manifest in the repo and watch Argo CD reconcile it.
+
+If you'd rather not expose the UI at all, port-forward instead:
+
+```bash
+kubectl -n argocd port-forward svc/argocd-server 8080:443
+# then open https://localhost:8080
+```
+
+### Cleanup
+
+Destroy in reverse — the Argo CD Application + Ingress first (so Argo CD prunes the workload and the LBC tears the ALB down), then the cluster:
+
+```bash
+cd workloads
+tofu destroy
+cd ..
+tofu destroy
+```
+
+## Troubleshooting
+
+### Application Won't Sync
+
+```bash
+kubectl -n argocd get applications
+kubectl -n argocd describe application spinifex-demo     # conditions show repo/auth errors
+kubectl -n argocd logs deploy/argocd-repo-server --tail=50
+```
+
+For a private repo, confirm the `eks-demo-app-repo` Secret exists in the `argocd` namespace with the `argocd.argoproj.io/secret-type=repository` label and a valid token.
+
+### `kubernetes_manifest` Fails: no matches for kind "Application"
+
+The `argoproj.io` CRDs aren't installed yet. Apply the parent module and wait for the `argocd` addon to reach `ACTIVE` before applying `workloads/`.
+
+### PVC Stuck in Pending
+
+```bash
+kubectl get pvc
+kubectl describe pvc <name>
+kubectl get storageclass
+```
+
+Confirm the `aws-ebs-csi-driver` addon is `ACTIVE` and the default StorageClass exists. EBS-CSI dynamic provisioning depends on the live Spinifex EBS/Viperblock data plane — verify on a stack where the driver is healthy. The app still serves with an in-memory counter if the volume never binds.
+
+### The Ingress Has No Address
+
+The LBC populates `status.loadBalancer` once it provisions the ALB. Check the addon and the controller logs:
+
+```bash
+kubectl describe ingress spinifex-demo
+kubectl -n kube-system logs deploy/aws-load-balancer-controller --tail=50
+```
+
+Confirm the cluster carries `spinifex.io/managed-ingress = "false"` so the LBC owns ingress.
+
+If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ...
+403 ... AccessDenied`, the node role is missing the LBC permissions. The
+controller runs with the node instance-profile credentials, so the
+`${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be
+attached to the node role — re-run `tofu apply` on the parent module if it was
+provisioned before that policy existed.
+
+### Provider Connection Refused
+
+```bash
+sudo systemctl status spinifex.target
+curl -k https://localhost:9999/
+```
@@ -0,0 +1,745 @@
+# Example: GitOps on EKS with Argo CD + a persistent EBS-CSI volume
+#
+# The top rung of the EKS ladder. It extends eks-https-ingress (LBC + ACM HTTPS)
+# and changes how the app is delivered: instead of Terraform applying the
+# workload, the Argo CD addon syncs a more elaborate Spinifex-themed app from a
+# git repository, and the app stores state on a Viperblock-backed EBS volume
+# provisioned dynamically through the EBS-CSI driver.
+#
+# What it adds over eks-https-ingress:
+#   * The argocd addon — GitOps continuous delivery, installed via the EKS API.
+#   * The aws-ebs-csi-driver addon — dynamic EBS (Viperblock) PersistentVolumes.
+#   * An access entry granting a second IAM principal read-only cluster access.
+#
+# The workloads/ module registers the (private) git repo with Argo CD, creates an
+# Argo CD Application that syncs the app from it, and keeps the HTTPS Ingress
+# (LBC + ACM) pointing at the git-managed Service. The app's manifests — including
+# the PersistentVolumeClaim — live in the git repo (see ../../../../eks-demo-app).
+#
+# Usage:
+#   cd spinifex/docs/terraform-workbooks/eks-gitops-argocd
+#   export AWS_PROFILE=spinifex
+#   tofu init && tofu apply
+#   # build + push the demo image to the ECR repo this creates (see README),
+#   # then: cd workloads && tofu init && tofu apply -var git_repo_url=... -var git_token=...
+#   # finally: kubectl get ingress spinifex-demo -o wide  → open https://<address>
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40, < 6.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0"
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Variables
+# ---------------------------------------------------------------------------
+
+variable "region" {
+  type    = string
+  default = "ap-southeast-2"
+}
+
+variable "cluster_name" {
+  type    = string
+  default = "eks-gitops"
+}
+
+variable "k8s_version" {
+  type    = string
+  default = "1.32"
+}
+
+variable "node_instance_type" {
+  type    = string
+  default = "t3.medium"
+}
+
+variable "node_desired_size" {
+  type        = number
+  default     = 1
+  description = "Worker count. Use 1 for a single-node demo, or 3 for an HA-shaped cluster."
+
+  validation {
+    condition     = var.node_desired_size == 1 || var.node_desired_size == 3
+    error_message = "node_desired_size must be 1 or 3."
+  }
+}
+
+variable "node_port" {
+  type        = number
+  default     = 30080
+  description = "NodePort the demo Service is published on and the ALB forwards to"
+}
+
+variable "argocd_node_port" {
+  type        = number
+  default     = 30081
+  description = "NodePort the Argo CD UI Service is published on and its ALB forwards to"
+}
+
+variable "cert_common_name" {
+  type    = string
+  default = "eks-gitops.spinifex.local"
+}
+
+variable "api_public_access_cidr" {
+  type        = string
+  default     = "0.0.0.0/0"
+  description = "CIDR allowed to reach the public Kubernetes API endpoint; tighten in production"
+}
+
+variable "spinifex_endpoint" {
+  type    = string
+  default = "https://127.0.0.1:9999"
+}
+
+# ---------------------------------------------------------------------------
+# Providers
+# ---------------------------------------------------------------------------
+
+provider "aws" {
+  region = var.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+    eks = var.spinifex_endpoint
+    ecr = var.spinifex_endpoint
+    acm = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+# ---------------------------------------------------------------------------
+# VPC — public subnets for the ALB + NAT, private subnets for the workers
+# ---------------------------------------------------------------------------
+
+resource "aws_vpc" "main" {
+  cidr_block           = "10.32.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  tags = {
+    Name = "${var.cluster_name}-vpc"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.main.id
+
+  tags = {
+    Name = "${var.cluster_name}-igw"
+  }
+}
+
+resource "aws_subnet" "public_a" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.32.1.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "${var.cluster_name}-public-a"
+  }
+}
+
+resource "aws_subnet" "public_b" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.32.2.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "${var.cluster_name}-public-b"
+  }
+}
+
+resource "aws_subnet" "private_a" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.32.11.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "${var.cluster_name}-private-a"
+  }
+}
+
+resource "aws_subnet" "private_b" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.32.12.0/24"
+  availability_zone = data.aws_availability_zones.available.names[0]
+
+  tags = {
+    Name = "${var.cluster_name}-private-b"
+  }
+}
+
+resource "aws_route_table" "public" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    Name = "${var.cluster_name}-public-rt"
+  }
+}
+
+resource "aws_route_table_association" "public_a" {
+  subnet_id      = aws_subnet.public_a.id
+  route_table_id = aws_route_table.public.id
+}
+
+resource "aws_route_table_association" "public_b" {
+  subnet_id      = aws_subnet.public_b.id
+  route_table_id = aws_route_table.public.id
+}
+
+# NAT gateway gives the private workers outbound internet to pull the demo image
+# from ECR. The workers still join the cluster over the in-VPC private endpoint.
+resource "aws_eip" "nat" {
+  domain = "vpc"
+
+  tags = {
+    Name = "${var.cluster_name}-nat-eip"
+  }
+}
+
+resource "aws_nat_gateway" "nat" {
+  allocation_id = aws_eip.nat.id
+  subnet_id     = aws_subnet.public_a.id
+
+  depends_on = [aws_internet_gateway.igw]
+
+  tags = {
+    Name = "${var.cluster_name}-nat"
+  }
+}
+
+resource "aws_route_table" "private" {
+  vpc_id = aws_vpc.main.id
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = aws_nat_gateway.nat.id
+  }
+
+  tags = {
+    Name = "${var.cluster_name}-private-rt"
+  }
+}
+
+resource "aws_route_table_association" "private_a" {
+  subnet_id      = aws_subnet.private_a.id
+  route_table_id = aws_route_table.private.id
+}
+
+resource "aws_route_table_association" "private_b" {
+  subnet_id      = aws_subnet.private_b.id
+  route_table_id = aws_route_table.private.id
+}
+
+# ---------------------------------------------------------------------------
+# IAM — cluster role
+# ---------------------------------------------------------------------------
+
+resource "aws_iam_role" "cluster" {
+  name = "${var.cluster_name}-cluster-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Action    = "sts:AssumeRole"
+      Principal = { Service = "eks.amazonaws.com" }
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "cluster" {
+  role       = aws_iam_role.cluster.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
+}
+
+# ---------------------------------------------------------------------------
+# IAM — node role
+# ---------------------------------------------------------------------------
+
+resource "aws_iam_role" "node" {
+  name = "${var.cluster_name}-node-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Action    = "sts:AssumeRole"
+      Principal = { Service = "ec2.amazonaws.com" }
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_worker" {
+  role       = aws_iam_role.node.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "node_cni" {
+  role       = aws_iam_role.node.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+}
+
+resource "aws_iam_role_policy_attachment" "node_ecr" {
+  role       = aws_iam_role.node.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+}
+
+# The AWS Load Balancer Controller runs with the node's instance-profile
+# credentials (Spinifex wires creds at the node level, not IRSA — see the addon
+# block below), so the permissions it needs to manage ALBs must live on the node
+# role. This mirrors the upstream AWSLoadBalancerControllerIAMPolicy, trimmed to
+# the actions an ALB Ingress exercises (Shield/WAF/Cognito are disabled on the
+# controller). Resources are "*" since Spinifex evaluates grants by action.
+# A customer-managed policy + attachment is used (Spinifex implements
+# CreatePolicy/AttachRolePolicy, not inline PutRolePolicy).
+resource "aws_iam_policy" "node_lbc" {
+  name = "${var.cluster_name}-node-lbc"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "ec2:DescribeAccountAttributes",
+          "ec2:DescribeAddresses",
+          "ec2:DescribeAvailabilityZones",
+          "ec2:DescribeInternetGateways",
+          "ec2:DescribeVpcs",
+          "ec2:DescribeSubnets",
+          "ec2:DescribeSecurityGroups",
+          "ec2:DescribeInstances",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DescribeTags",
+          "ec2:GetCoipPoolUsage",
+          "ec2:DescribeCoipPools",
+          "ec2:CreateSecurityGroup",
+          "ec2:CreateTags",
+          "ec2:DeleteTags",
+          "ec2:AuthorizeSecurityGroupIngress",
+          "ec2:RevokeSecurityGroupIngress",
+          "ec2:DeleteSecurityGroup",
+          "elasticloadbalancing:*",
+          "acm:ListCertificates",
+          "acm:DescribeCertificate",
+          "acm:GetCertificate",
+          "iam:CreateServiceLinkedRole",
+          "iam:ListServerCertificates",
+          "iam:GetServerCertificate",
+          "tag:GetResources",
+          "tag:TagResources",
+          "wafv2:GetWebACLForResource",
+          "shield:GetSubscriptionState"
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_lbc" {
+  role       = aws_iam_role.node.name
+  policy_arn = aws_iam_policy.node_lbc.arn
+}
+
+# The EBS CSI driver also runs with the node's instance-profile credentials, so
+# the volume-lifecycle permissions it needs must live on the node role too. This
+# mirrors the upstream AmazonEBSCSIDriverPolicy. Resources are "*" since Spinifex
+# evaluates grants by action; the CreateVolume/CreateSnapshot/CreateTags grants
+# upstream scope with request/resource tag conditions, omitted here because
+# Spinifex does not yet evaluate IAM tag conditions.
+resource "aws_iam_policy" "node_ebs_csi" {
+  name = "${var.cluster_name}-node-ebs-csi"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "ec2:CreateSnapshot",
+          "ec2:AttachVolume",
+          "ec2:DetachVolume",
+          "ec2:ModifyVolume",
+          "ec2:DescribeAvailabilityZones",
+          "ec2:DescribeInstances",
+          "ec2:DescribeSnapshots",
+          "ec2:DescribeTags",
+          "ec2:DescribeVolumes",
+          "ec2:DescribeVolumesModifications",
+          "ec2:CreateTags",
+          "ec2:DeleteTags",
+          "ec2:CreateVolume",
+          "ec2:DeleteVolume",
+          "ec2:DeleteSnapshot"
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_ebs_csi" {
+  role       = aws_iam_role.node.name
+  policy_arn = aws_iam_policy.node_ebs_csi.arn
+}
+
+# ---------------------------------------------------------------------------
+# IAM — a second principal to grant read-only cluster access to
+# ---------------------------------------------------------------------------
+
+resource "aws_iam_role" "viewer" {
+  name = "${var.cluster_name}-viewer"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Action    = "sts:AssumeRole"
+      Principal = { Service = "ec2.amazonaws.com" }
+    }]
+  })
+}
+
+# ---------------------------------------------------------------------------
+# ECR — repository the workers pull the demo image from
+# ---------------------------------------------------------------------------
+
+resource "aws_ecr_repository" "demo" {
+  name = "spinifex-demo"
+
+  # tofu destroy must remove the repo even though it still holds the pushed demo
+  # image; without this, DeleteRepository (force=false) fails RepositoryNotEmpty.
+  force_delete = true
+}
+
+# ---------------------------------------------------------------------------
+# EKS cluster — public + private endpoints, LBC-owned ingress
+#
+# The spinifex.io/managed-ingress = "false" tag disables K3s' built-in
+# traefik/servicelb so the AWS Load Balancer Controller owns ingress, matching
+# how ingress works on AWS EKS.
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_cluster" "this" {
+  name     = var.cluster_name
+  role_arn = aws_iam_role.cluster.arn
+  version  = var.k8s_version
+
+  vpc_config {
+    subnet_ids              = [aws_subnet.private_a.id, aws_subnet.private_b.id]
+    endpoint_public_access  = true
+    endpoint_private_access = true
+    public_access_cidrs     = [var.api_public_access_cidr]
+  }
+
+  access_config {
+    authentication_mode                         = "API"
+    bootstrap_cluster_creator_admin_permissions = true
+  }
+
+  depends_on = [aws_iam_role_policy_attachment.cluster]
+
+  # Spinifex's DescribeCluster doesn't echo these back the way the AWS provider
+  # expects: access_config reads as absent (and its ForceNew
+  # bootstrap_cluster_creator_admin_permissions would replace the live cluster on
+  # re-add), and vpc_config.security_group_ids reads back as drift. Ignore both so
+  # later applies (e.g. adding the Argo CD NodePort rule) neither replace the
+  # cluster nor detach its security groups.
+  lifecycle {
+    ignore_changes = [access_config, vpc_config[0].security_group_ids]
+  }
+
+  tags = {
+    Name                          = var.cluster_name
+    "spinifex.io/managed-ingress" = "false"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Managed node group — workers in the private subnets
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_node_group" "workers" {
+  cluster_name    = aws_eks_cluster.this.name
+  node_group_name = "workers"
+  node_role_arn   = aws_iam_role.node.arn
+  subnet_ids      = [aws_subnet.private_a.id, aws_subnet.private_b.id]
+
+  scaling_config {
+    desired_size = var.node_desired_size
+    min_size     = var.node_desired_size
+    max_size     = var.node_desired_size * 2
+  }
+
+  instance_types = [var.node_instance_type]
+  ami_type       = "AL2_x86_64"
+
+  depends_on = [
+    aws_iam_role_policy_attachment.node_worker,
+    aws_iam_role_policy_attachment.node_cni,
+    aws_iam_role_policy_attachment.node_ecr,
+    aws_iam_role_policy_attachment.node_lbc,
+    aws_iam_role_policy_attachment.node_ebs_csi,
+  ]
+
+  tags = {
+    Name = "${var.cluster_name}-workers"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Addon — AWS Load Balancer Controller
+#
+# Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
+# node level, so no IRSA role or subnet tagging is needed here. addon_version is
+# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
+# so let Spinifex default to its catalog version.
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_addon" "lbc" {
+  cluster_name                = aws_eks_cluster.this.name
+  addon_name                  = "aws-load-balancer-controller"
+  resolve_conflicts_on_create = "OVERWRITE"
+
+  depends_on = [aws_eks_node_group.workers]
+
+  tags = {
+    Name = "${var.cluster_name}-lbc"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Addon — Argo CD (GitOps delivery)
+#
+# Installed through the EKS API; Spinifex stages the bundle host-side and the
+# worker renders it into the K3s auto-deploy dir. The workloads module registers
+# the git repo and creates the Argo CD Application that syncs the demo app.
+# addon_version omitted (see the LBC note).
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_addon" "argocd" {
+  cluster_name                = aws_eks_cluster.this.name
+  addon_name                  = "argocd"
+  resolve_conflicts_on_create = "OVERWRITE"
+
+  depends_on = [aws_eks_node_group.workers]
+
+  tags = {
+    Name = "${var.cluster_name}-argocd"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Addon — EBS CSI driver (Viperblock-backed PersistentVolumes)
+#
+# Ships a default gp3 StorageClass (provisioner ebs.csi.aws.com). A PVC in the
+# demo app's git manifests dynamically provisions a Viperblock-backed EBS volume
+# so the app's state survives pod restarts. addon_version omitted (see LBC note).
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_addon" "ebs_csi" {
+  cluster_name                = aws_eks_cluster.this.name
+  addon_name                  = "aws-ebs-csi-driver"
+  resolve_conflicts_on_create = "OVERWRITE"
+
+  depends_on = [aws_eks_node_group.workers]
+
+  tags = {
+    Name = "${var.cluster_name}-ebs-csi"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Access entry — grant the viewer principal read-only access (API auth mode)
+# ---------------------------------------------------------------------------
+
+resource "aws_eks_access_entry" "viewer" {
+  cluster_name  = aws_eks_cluster.this.name
+  principal_arn = aws_iam_role.viewer.arn
+  type          = "STANDARD"
+}
+
+resource "aws_eks_access_policy_association" "viewer" {
+  cluster_name  = aws_eks_cluster.this.name
+  principal_arn = aws_iam_role.viewer.arn
+  policy_arn    = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"
+
+  access_scope {
+    type = "cluster"
+  }
+
+  depends_on = [aws_eks_access_entry.viewer]
+}
+
+# ---------------------------------------------------------------------------
+# TLS — self-signed certificate imported into ACM
+#
+# Spinifex ACM supports ImportCertificate (not RequestCertificate), so the cert
+# is generated locally by the tls provider and imported. The workloads Ingress
+# attaches it to the ALB's HTTPS listener by ARN.
+# ---------------------------------------------------------------------------
+
+resource "tls_private_key" "ingress" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}
+
+resource "tls_self_signed_cert" "ingress" {
+  private_key_pem = tls_private_key.ingress.private_key_pem
+
+  subject {
+    common_name  = var.cert_common_name
+    organization = "Spinifex EKS Demo"
+  }
+
+  # Wildcard SAN so one ALB (shared IngressGroup) can host-route both the demo
+  # app (app.<cn>) and the Argo CD UI (argocd.<cn>) off this single cert.
+  dns_names             = [var.cert_common_name, "*.${var.cert_common_name}"]
+  validity_period_hours = 8760
+  early_renewal_hours   = 720
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+  ]
+}
+
+resource "aws_acm_certificate" "ingress" {
+  private_key      = tls_private_key.ingress.private_key_pem
+  certificate_body = tls_self_signed_cert.ingress.cert_pem
+
+  tags = {
+    Name = "${var.cluster_name}-ingress"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Let the ALB reach the workers' NodePort
+#
+# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
+# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
+# NodePort. Look the SG up by its deterministic name and add one rule.
+# ---------------------------------------------------------------------------
+
+data "aws_security_group" "nodegroup" {
+  filter {
+    name   = "group-name"
+    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
+  }
+
+  filter {
+    name   = "vpc-id"
+    values = [aws_vpc.main.id]
+  }
+
+  depends_on = [aws_eks_node_group.workers]
+}
+
+resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
+  security_group_id = data.aws_security_group.nodegroup.id
+  cidr_ipv4         = aws_vpc.main.cidr_block
+  from_port         = var.node_port
+  to_port           = var.node_port
+  ip_protocol       = "tcp"
+
+  tags = {
+    Name = "${var.cluster_name}-alb-nodeport"
+  }
+}
+
+# Same, for the Argo CD UI NodePort its ALB forwards to.
+resource "aws_vpc_security_group_ingress_rule" "argocd_nodeport_from_vpc" {
+  security_group_id = data.aws_security_group.nodegroup.id
+  cidr_ipv4         = aws_vpc.main.cidr_block
+  from_port         = var.argocd_node_port
+  to_port           = var.argocd_node_port
+  ip_protocol       = "tcp"
+
+  tags = {
+    Name = "${var.cluster_name}-argocd-nodeport"
+  }
+}
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+output "cluster_name" {
+  value = aws_eks_cluster.this.name
+}
+
+output "region" {
+  value = var.region
+}
+
+output "node_port" {
+  value = var.node_port
+}
+
+output "argocd_node_port" {
+  value = var.argocd_node_port
+}
+
+output "node_desired_size" {
+  value = var.node_desired_size
+}
+
+output "certificate_arn" {
+  value = aws_acm_certificate.ingress.arn
+}
+
+output "cert_common_name" {
+  value = var.cert_common_name
+}
+
+output "viewer_principal_arn" {
+  value = aws_iam_role.viewer.arn
+}
+
+output "ecr_repository_url" {
+  value       = aws_ecr_repository.demo.repository_url
+  description = "Push the demo-app image here, then apply the workloads module"
+}
+
+output "ingress_address_hint" {
+  value = "The demo app and Argo CD UI share one ALB (LBC IngressGroup), host-routed: app.${var.cert_common_name} and argocd.${var.cert_common_name}. Get the address: kubectl get ingress spinifex-demo -o jsonpath='{.status.loadBalancer.ingress[0].hostname}{\"\\n\"}'."
+}
+
+output "argocd_ingress_hint" {
+  value = "Same ALB as the app. Open https://argocd.${var.cert_common_name} (admin password: kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d). Resolve the host to the ALB address, or curl -k --resolve."
+}
+
+output "update_kubeconfig" {
+  value = "aws eks update-kubeconfig --name ${aws_eks_cluster.this.name} --region ${var.region}"
+}
@@ -0,0 +1,370 @@
+# GitOps workload for eks-gitops-argocd — separate root module / state.
+#
+# Instead of applying the app directly, this module hands delivery to Argo CD:
+# it registers the (private) git repo as an Argo CD repository credential and
+# creates an Argo CD Application that syncs the demo app's manifests from it. The
+# app's Deployment, Service, and PersistentVolumeClaim live in the git repo (see
+# ../../../../eks-demo-app); the HTTPS Ingress (LBC + ACM) stays here and points
+# at the Service that Argo CD creates.
+#
+# The Argo CD Application is a kubernetes_manifest, so the argoproj.io CRDs must
+# already exist — apply the parent module (which installs the argocd addon) and
+# let it reach ACTIVE before applying this module.
+#
+# Usage:
+#   cd spinifex/docs/terraform-workbooks/eks-gitops-argocd
+#   tofu init && tofu apply                       # parent: cluster + addons + ACM
+#   cd workloads && tofu init
+#   tofu apply -var git_repo_url=https://github.com/mulgadc/eks-demo-app.git \
+#              -var git_token=<a-read-only-PAT>
+#   # teardown is the reverse — destroy here first, then the parent.
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40, < 6.0"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.20"
+    }
+  }
+}
+
+variable "spinifex_endpoint" {
+  type    = string
+  default = "https://127.0.0.1:9999"
+}
+
+variable "git_repo_url" {
+  type        = string
+  default     = "https://github.com/mulgadc/eks-demo-app.git"
+  description = "Git repo Argo CD syncs the demo app from"
+}
+
+variable "git_revision" {
+  type        = string
+  default     = "main"
+  description = "Git branch, tag, or commit Argo CD tracks"
+}
+
+variable "git_path" {
+  type        = string
+  default     = "manifests"
+  description = "Path within the repo holding the app manifests"
+}
+
+variable "git_username" {
+  type        = string
+  default     = "git"
+  description = "Username for the git credential (any non-empty value for a PAT)"
+}
+
+variable "git_token" {
+  type        = string
+  default     = ""
+  sensitive   = true
+  description = "Read-only personal access token for the private repo. Leave empty for a public repo."
+}
+
+provider "aws" {
+  region = data.terraform_remote_state.infra.outputs.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+    eks = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+data "terraform_remote_state" "infra" {
+  backend = "local"
+
+  config = {
+    path = "../terraform.tfstate"
+  }
+}
+
+locals {
+  cluster_name     = data.terraform_remote_state.infra.outputs.cluster_name
+  region           = data.terraform_remote_state.infra.outputs.region
+  cert_arn         = data.terraform_remote_state.infra.outputs.certificate_arn
+  argocd_node_port = data.terraform_remote_state.infra.outputs.argocd_node_port
+  cert_cn          = data.terraform_remote_state.infra.outputs.cert_common_name
+  private_repo     = var.git_token != ""
+
+  # The demo app and the Argo CD UI share one ALB via an LBC IngressGroup; LBC
+  # gives each backend its own target group and host-routes between them.
+  alb_group   = local.cluster_name
+  app_host    = "app.${local.cert_cn}"
+  argocd_host = "argocd.${local.cert_cn}"
+}
+
+data "aws_eks_cluster" "this" {
+  name = local.cluster_name
+}
+
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.this.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
+
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
+  }
+}
+
+# Repository credential for the private repo. Argo CD picks up Secrets in its
+# namespace labelled argocd.argoproj.io/secret-type=repository. Created only when
+# a token is supplied (a public repo needs no credential).
+resource "kubernetes_secret_v1" "repo" {
+  count = local.private_repo ? 1 : 0
+
+  metadata {
+    name      = "eks-demo-app-repo"
+    namespace = "argocd"
+    labels = {
+      "argocd.argoproj.io/secret-type" = "repository"
+    }
+  }
+
+  data = {
+    type     = "git"
+    url      = var.git_repo_url
+    username = var.git_username
+    password = var.git_token
+  }
+}
+
+# Argo CD Application: syncs the demo app from the git repo into the default
+# namespace, self-healing and pruning so the cluster tracks the repo.
+resource "kubernetes_manifest" "demo_app" {
+  manifest = {
+    apiVersion = "argoproj.io/v1alpha1"
+    kind       = "Application"
+
+    # The resources finalizer makes deleting this Application cascade-prune the
+    # synced resources (Deployment, Service, PVC). Without it, tofu destroy drops
+    # the Application CR but orphans the live PVC, whose Delete-reclaim PV then
+    # leaks an EBS volume once the cluster (and ebs-csi) is torn down.
+    metadata = {
+      name       = "spinifex-demo"
+      namespace  = "argocd"
+      finalizers = ["resources-finalizer.argocd.argoproj.io"]
+    }
+
+    spec = {
+      project = "default"
+
+      source = {
+        repoURL        = var.git_repo_url
+        targetRevision = var.git_revision
+        path           = var.git_path
+      }
+
+      destination = {
+        server    = "https://kubernetes.default.svc"
+        namespace = "default"
+      }
+
+      syncPolicy = {
+        automated = {
+          prune    = true
+          selfHeal = true
+        }
+        syncOptions = ["CreateNamespace=true"]
+      }
+    }
+  }
+
+  depends_on = [kubernetes_secret_v1.repo]
+}
+
+# HTTPS Ingress (LBC + ACM), carried over from eks-https-ingress. It points at
+# the spinifex-demo Service that Argo CD creates from the git manifests.
+resource "kubernetes_ingress_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+
+    annotations = {
+      "alb.ingress.kubernetes.io/scheme"           = "internet-facing"
+      "alb.ingress.kubernetes.io/target-type"      = "instance"
+      "alb.ingress.kubernetes.io/group.name"       = local.alb_group
+      "alb.ingress.kubernetes.io/listen-ports"     = "[{\"HTTP\":80},{\"HTTPS\":443}]"
+      "alb.ingress.kubernetes.io/certificate-arn"  = local.cert_arn
+      "alb.ingress.kubernetes.io/ssl-redirect"     = "443"
+      "alb.ingress.kubernetes.io/healthcheck-path" = "/healthz"
+    }
+  }
+
+  spec {
+    ingress_class_name = "alb"
+
+    # No host condition: the app is the ALB's catch-all on :443, so it answers on
+    # the raw ALB IP as well as app.<cn> once DNS is wired. Argo CD's host rule is
+    # more specific and still wins for argocd.<cn>.
+    rule {
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+
+          backend {
+            service {
+              name = "spinifex-demo"
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  depends_on = [kubernetes_manifest.demo_app]
+}
+
+# ---------------------------------------------------------------------------
+# Argo CD UI exposure — managing the cluster through the GitOps console is the
+# point of this demo, so the UI gets the same HTTPS-Ingress treatment as the app
+# rather than a port-forward. The argocd addon ships argocd-server as a ClusterIP
+# only; expose it via a NodePort the ALB targets. argocd-server serves TLS on
+# 8080 (no --insecure), so the ALB speaks HTTPS to the backend.
+#
+# Same group.name as the demo Ingress, so LBC folds both onto ONE ALB. On :443
+# the app is the catch-all and argocd.<cn> host-routes to the UI. Argo CD also
+# gets a hostless :8443 listener (argocd_ip below) so the UI is reachable on the
+# raw ALB IP before DNS exists — https://<alb-ip>:8443.
+# ---------------------------------------------------------------------------
+
+resource "kubernetes_service_v1" "argocd_server_nodeport" {
+  metadata {
+    name      = "argocd-server-nodeport"
+    namespace = "argocd"
+  }
+
+  spec {
+    type     = "NodePort"
+    selector = { "app.kubernetes.io/name" = "argocd-server" }
+
+    port {
+      port        = 443
+      target_port = 8080
+      node_port   = local.argocd_node_port
+      protocol    = "TCP"
+    }
+  }
+}
+
+resource "kubernetes_ingress_v1" "argocd" {
+  metadata {
+    name      = "argocd-server"
+    namespace = "argocd"
+
+    annotations = {
+      "alb.ingress.kubernetes.io/scheme"               = "internet-facing"
+      "alb.ingress.kubernetes.io/target-type"          = "instance"
+      "alb.ingress.kubernetes.io/group.name"           = local.alb_group
+      "alb.ingress.kubernetes.io/listen-ports"         = "[{\"HTTPS\":443}]"
+      "alb.ingress.kubernetes.io/certificate-arn"      = local.cert_arn
+      "alb.ingress.kubernetes.io/backend-protocol"     = "HTTPS"
+      "alb.ingress.kubernetes.io/healthcheck-protocol" = "HTTPS"
+      "alb.ingress.kubernetes.io/healthcheck-path"     = "/healthz"
+    }
+  }
+
+  spec {
+    ingress_class_name = "alb"
+
+    rule {
+      host = local.argocd_host
+
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+
+          backend {
+            service {
+              name = kubernetes_service_v1.argocd_server_nodeport.metadata[0].name
+              port {
+                number = 443
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+# Hostless Argo CD listener on :8443 of the same ALB, so the UI is reachable on
+# the raw ALB IP (https://<alb-ip>:8443) without DNS or a host header. argocd.<cn>
+# on :443 still works via the host-routed ingress above.
+resource "kubernetes_ingress_v1" "argocd_ip" {
+  metadata {
+    name      = "argocd-server-ip"
+    namespace = "argocd"
+
+    annotations = {
+      "alb.ingress.kubernetes.io/scheme"               = "internet-facing"
+      "alb.ingress.kubernetes.io/target-type"          = "instance"
+      "alb.ingress.kubernetes.io/group.name"           = local.alb_group
+      "alb.ingress.kubernetes.io/listen-ports"         = "[{\"HTTPS\":8443}]"
+      "alb.ingress.kubernetes.io/certificate-arn"      = local.cert_arn
+      "alb.ingress.kubernetes.io/backend-protocol"     = "HTTPS"
+      "alb.ingress.kubernetes.io/healthcheck-protocol" = "HTTPS"
+      "alb.ingress.kubernetes.io/healthcheck-path"     = "/healthz"
+    }
+  }
+
+  spec {
+    ingress_class_name = "alb"
+
+    rule {
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+
+          backend {
+            service {
+              name = kubernetes_service_v1.argocd_server_nodeport.metadata[0].name
+              port {
+                number = 443
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+output "application_name" {
+  value = kubernetes_manifest.demo_app.manifest.metadata.name
+}
+
+output "alb_address_hint" {
+  value = "One shared ALB. Get its IP: kubectl get ingress spinifex-demo -o jsonpath='{.status.loadBalancer.ingress[0].ip}{\"\\n\"}' (or [0].hostname). Reach by raw IP — app: https://<alb-ip>/  Argo CD: https://<alb-ip>:8443/ . By DNS — app: ${local.app_host}  Argo CD: ${local.argocd_host}."
+}
+
+output "argocd_admin_password_cmd" {
+  value = "kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d ; echo"
+}
+
+output "argocd_url_hint" {
+  value = "Open https://${local.argocd_host} (admin password: kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d). Resolve the host to the ALB address (northstar/CNAME), or curl -k --resolve ${local.argocd_host}:443:<alb-ip>."
+}
@@ -1,31 +1,30 @@
 ---
-title: "3-Node EKS behind an HTTPS ALB (ACM TLS)"
-description: "Provision an HA-shaped EKS cluster with public + private API endpoints and three workers, deploy a demo app, and publish it through an internet-facing ALB that terminates TLS with an ACM-imported certificate, using Terraform on Spinifex."
+title: "EKS HTTPS Ingress (LBC + ACM)"
+description: "Serve a Spinifex-themed demo app over HTTPS on EKS using the AWS Load Balancer Controller addon and an ACM certificate — an internet-facing ALB provisioned from a Kubernetes Ingress, using Terraform on Spinifex."
 category: "Terraform Workbooks"
 tags:
  - terraform
  - eks
  - kubernetes
-  - acm
-  - tls
+  - ingress
  - alb
-  - elbv2
-  - vpc
+  - acm
+  - https
  - workbook
 resources:
+  - title: "AWS Load Balancer Controller"
+    url: "https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/"
  - title: "Terraform AWS Provider"
    url: "https://registry.terraform.io/providers/hashicorp/aws/latest"
  - title: "Terraform Kubernetes Provider"
    url: "https://registry.terraform.io/providers/hashicorp/kubernetes/latest"
  - title: "Spinifex Repository"
    url: "https://github.com/mulgadc/spinifex"
-  - title: "OpenTofu"
-    url: "https://opentofu.org/"
 ---

-# Terraform: 3-Node EKS behind an HTTPS ALB
+# Terraform: EKS HTTPS Ingress (LBC + ACM)

-> The largest EKS workbook, and the clearest demo. An HA-shaped cluster with public + private API endpoints and three workers, running a demo app published over HTTPS through an internet-facing ALB that terminates TLS with an ACM-imported certificate.
+> Builds on the quickstart. Instead of a raw NodePort, it serves the Spinifex-themed demo app over **HTTPS** through an ALB that the **AWS Load Balancer Controller** provisions from a Kubernetes **Ingress**, terminating TLS with a certificate held in **ACM** — the same pattern you'd use on AWS EKS.

 ## Table of Contents

@@ -37,59 +36,50 @@ resources:

 ## Overview

-This workbook is sized for a **3-node Spinifex deployment** and is built to show off. It brings together the pieces from the smaller EKS examples, deploys a demo app across three workers, and fronts it with a TLS-terminating load balancer. When `apply` finishes, fetch the ALB's public IP and open `https://<ip>` — the `nginxdemos/hello` page reports which pod answered, and refreshing moves between the three replicas spread across the cluster. That single page demonstrates a multi-node Kubernetes cluster load-balancing behind HTTPS.
+This is the second rung of the EKS ladder. It takes the cluster + demo app from [EKS Quickstart](../eks-quickstart) and publishes the app the way you would in production: a Kubernetes `Ingress` reconciled by the **AWS Load Balancer Controller (LBC)** into an internet-facing **Application Load Balancer** that terminates TLS with an **ACM** certificate.

- **Public + private API endpoints.** `endpoint_public_access` serves `kubectl` and the Kubernetes provider from your host (CIDR-restricted); `endpoint_private_access` exposes an in-VPC private endpoint the workers use to join.
- **Private workers + NAT.** The three workers sit in private subnets. They *join* the cluster over the in-VPC private endpoint, and a NAT gateway gives them outbound access to pull the demo container image.
- **CoreDNS** installed as a managed addon.
- **TLS via ACM.** A self-signed certificate is generated with the `tls` provider and **imported into ACM** (Spinifex ACM supports `ImportCertificate`, not `RequestCertificate`). The ALB's HTTPS listener references the ACM ARN and forwards to the workers' NodePort.
- **The app, deployed by Terraform.** The Kubernetes provider deploys the Deployment + NodePort Service, and one security-group rule lets the ALB reach the NodePort.
+To make room for an internet-facing ALB with private workers, the network grows from the quickstart's two public subnets to a public/private split: the ALB and a **NAT gateway** sit in the public subnets, the workers sit in the private subnets and reach ECR through the NAT. The cluster is tagged `spinifex.io/managed-ingress = "false"`, which disables K3s' built-in traefik/servicelb so the LBC owns ingress.

-<p align="center">
-  <img src="../../../.github/assets/diagrams/tf-eks-https-ingress.svg" alt="3-node EKS — public+private API endpoints, ALB in public subnets terminating TLS, three workers in private subnets behind a NAT gateway" width="900">
-</p>
+The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires the controller's AWS credentials and the cluster's ELB-eligible subnets at the node level and injects those subnets into the `alb` `IngressClassParams`, so the Ingress needs **no IRSA role and no subnet tags** — just `ingressClassName: alb` and a handful of annotations.

 **What you'll learn:**

- Running an EKS cluster with both public and private API endpoints
- Placing workers in private subnets that join over the private endpoint and egress via NAT
- Importing a certificate into ACM and terminating TLS on an ALB HTTPS listener
- Wiring an ALB target group to managed node-group workers via a NodePort, including the SG rule that allows it
- Deploying a load-balanced workload with the Kubernetes provider
+- Installing the `aws-load-balancer-controller` addon and letting it own ingress
+- Disabling the built-in K3s ingress with the `spinifex.io/managed-ingress` tag
+- Importing a self-signed certificate into ACM and attaching it to an ALB listener
+- Driving an ALB entirely from a Kubernetes `Ingress` (`ingressClassName: alb`)
+- A public/private VPC with a NAT gateway for private workers

 **What gets created**

 | Resource | Name | Purpose |
 |---|---|---|
-| VPC | `eks-https-vpc` | Isolated network (10.32.0.0/16) |
-| Public Subnets | `eks-https-public-a/-b` | Host the ALB and NAT gateway |
-| Private Subnets | `eks-https-private-a/-b` | Host the workers |
-| Internet Gateway | `eks-https-igw` | Internet for the public subnets |
-| NAT Gateway (+ EIP) | `eks-https-nat` | Outbound for the private workers (image pulls) |
-| Route Tables | `eks-https-public-rt`, `eks-https-private-rt` | Public via IGW; private via NAT |
+| VPC | `eks-https-vpc` | Isolated network (10.31.0.0/16) |
+| Subnets | `eks-https-public-a/-b`, `eks-https-private-a/-b` | Public (ALB + NAT) and private (workers) |
+| Internet + NAT Gateway | `eks-https-igw`, `eks-https-nat` | Public egress; private-worker egress to ECR |
 | IAM Roles | `eks-https-cluster-role`, `eks-https-node-role` | Control-plane and worker roles |
-| EKS Cluster | `eks-https` | Public (CIDR-restricted) + private endpoints |
-| Node Group | `workers` | Three `t3.large` workers in the private subnets |
-| Addon | `coredns` | Cluster DNS |
-| ACM Cert | `eks-https-ingress` | Self-signed cert imported into ACM |
-| ALB + HTTPS Listener | `eks-https-alb` | Internet-facing, terminates TLS on :443 |
-| Target Group | `eks-https-tg` | Instance targets on the NodePort |
-| SG Ingress Rule | `eks-https-alb-nodeport` | Lets the ALB SG reach the worker NodePort |
-| K8s Deployment + Service | `hello` | `nginxdemos/hello`, 3 replicas, NodePort |
+| ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
+| EKS Cluster | `eks-https` | Public + private endpoints; `managed-ingress=false` |
+| Node Group | `workers` | `node_desired_size` `t3.large` worker(s) — 1 or 3 |
+| Addon | `aws-load-balancer-controller` | Provisions the ALB from the Ingress |
+| ACM Certificate | `eks-https-ingress` | Self-signed, imported; attached to the HTTPS listener |
+| SG Ingress Rule | `eks-https-alb-nodeport` | Admits the VPC CIDR to the workers' NodePort |
+| K8s Deployment + Service | `spinifex-demo` | Themed demo image, 2 replicas, NodePort |
+| K8s Ingress | `spinifex-demo` | `ingressClassName: alb`, HTTPS via the ACM cert |

 **Spinifex specifics**

- ACM supports `ImportCertificate`, `DescribeCertificate`, `ListCertificates`, `DeleteCertificate` — there is no `RequestCertificate`/DNS-validation flow, hence the self-signed import.
- The ALB HTTPS listener resolves the cert from ACM. Valid SSL policies are `ELBSecurityPolicy-2016-08` (default) and `ELBSecurityPolicy-TLS13-1-2-2021-06`.
- The worker SG is auto-managed (`vpc_config.security_group_ids` is ignored) and admits only intra-cluster traffic. The ALB→NodePort rule is added by looking the SG up by its deterministic name and admitting the ALB SG.
- Worker discovery uses the `spinifex:eks-cluster` tag Spinifex stamps on node-group instances; the target-group attachment count is the known `node_desired_size`.
- Leave `addon_version` unset (see the addons workbook for why pinning it can't satisfy both the provider and the catalog).
+- **The LBC needs no IRSA or subnet tagging here.** Credentials and ELB-eligible subnets are injected by Spinifex at the node level and into the `alb` `IngressClassParams`. The Ingress just sets `ingressClassName: alb` plus the `alb.ingress.kubernetes.io/*` annotations.
+- **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
+- The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
+- Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
+- **Leave `addon_version` unset** (see the [quickstart/addons notes](../eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.

 **Prerequisites:**

- A 3-node Spinifex deployment, installed and running
- The Spinifex `eks-node` image available on the cluster
- OpenTofu or Terraform, plus `kubectl` and the AWS CLI
+- Spinifex installed and running, with the `eks-node` image (carrying the LBC bundle) available
+- OpenTofu or Terraform, plus `kubectl`, the AWS CLI, and Docker
+- The demo image built and pushed to ECR (see [`../demo-app/README.md`](../demo-app/README.md))

 ## Instructions

@@ -106,7 +96,9 @@ Or create a `main.tf` file and paste the full configuration below.

 <!-- INCLUDE: main.tf lang:hcl -->

-### Step 2. Deploy
+### Step 2. Deploy the Cluster
+
+The cluster and the demo app are **two root modules with separate state**. Apply the cluster first (set `node_desired_size=3` for an HA-shaped cluster):

 ```bash
 export AWS_PROFILE=spinifex
@@ -114,110 +106,98 @@ tofu init
 tofu apply
 ```

-Expect several minutes: the control plane bootstraps, three workers launch and join over the private endpoint, the NAT gateway comes up, CoreDNS installs, the ALB is provisioned, and the Kubernetes provider rolls out the demo app.
+This brings up the cluster, the workers, the NAT gateway, the LBC addon, the ACM cert, and the `spinifex-demo` ECR repository. Give it a few minutes to reach `ACTIVE` and for the LBC addon to report healthy.

-> **Keep `AWS_PROFILE=spinifex` exported** for the whole `apply` — the Kubernetes provider authenticates with `aws eks get-token` against the Spinifex STS endpoint.
-
-> **Tighten access in production.** `api_public_access_cidr` and `alb_ingress_cidr` both default to `0.0.0.0/0`. Set them to your own CIDR:
->
-> ```bash
-> export TF_VAR_api_public_access_cidr="203.0.113.10/32"
-> export TF_VAR_alb_ingress_cidr="203.0.113.10/32"
-> ```
-
-### Step 3. Open the Demo over HTTPS
-
-The ALB DNS name (`*.elb.spinifex.local`) won't resolve from your host, so fetch its public IP:
+### Step 3. Build and Push the Demo Image

 ```bash
-ALB_IP=$(aws elbv2 describe-load-balancers --names eks-https-alb \
-  --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
-  --output text)
+cd ../demo-app
+REGISTRY=$(cd ../eks-https-ingress && tofu output -raw ecr_repository_url)
+REGISTRY_HOST=${REGISTRY%%/*}
+aws ecr get-login-password | docker login --username AWS --password-stdin "$REGISTRY_HOST"
+docker build -t "${REGISTRY}:latest" .
+docker push "${REGISTRY}:latest"
+cd ../eks-https-ingress
 ```

-The certificate is self-signed, so a browser will warn (proceed anyway) and `curl` needs `-k`:
+### Step 4. Deploy the App and Ingress

 ```bash
-curl -k https://$ALB_IP
-curl -k https://$ALB_IP   # refresh: the "Server name" changes between pods
+cd workloads
+tofu init
+tofu apply
 ```

-Open `https://$ALB_IP` in a browser and refresh to watch requests land on different pods across the three nodes.
+> **Why two modules?** The Kubernetes provider in `workloads/` reads the cluster endpoint from a live `data "aws_eks_cluster"` source, so it's only configured while the cluster exists. Always destroy `workloads/` before the cluster, or the provider falls back to `http://localhost:80` and reports `connection refused`. Keep `AWS_PROFILE=spinifex` exported for both applies.

-### Step 4. Inspect (optional)
+<!-- INCLUDE: workloads/main.tf lang:hcl -->
+
+### Step 5. Open the Demo over HTTPS
+
+The LBC takes a minute or two to provision the ALB after the Ingress is created. The Ingress publishes the ALB's **DNS name**, which has no resolver yet (northstar will add one) — resolve it to the ALB's public IP with `describe-load-balancers` so you can reach it without DNS or `/etc/hosts`:

 ```bash
+# Point kubectl at the cluster first (writes ~/.kube/config) — without this every
+# kubectl call fails to connect:
 aws eks update-kubeconfig --name eks-https --region ap-southeast-2
-kubectl get nodes
-kubectl get pods -o wide          # three replicas, one per node

-TG_ARN=$(aws elbv2 describe-target-groups --names eks-https-tg \
-  --query 'TargetGroups[0].TargetGroupArn' --output text)
-aws elbv2 describe-target-health --target-group-arn "$TG_ARN"
+kubectl get ingress spinifex-demo -o wide
+DNSNAME=$(kubectl get ingress spinifex-demo -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+IP=$(aws elbv2 describe-load-balancers \
+  --query "LoadBalancers[?DNSName=='${DNSNAME}'].AvailabilityZones[].LoadBalancerAddresses[].IpAddress | [0]" \
+  --output text)
+curl -k "https://$IP"          # self-signed cert: -k skips verification
 ```

+Open `https://$IP` in a browser (accept the self-signed certificate). The Spinifex-themed page reports the pod, node, cluster, and region that answered — refresh to watch requests land on different replicas, now over HTTPS through the ALB.
+
 ### Cleanup

+Destroy in reverse — the app/Ingress first (so the LBC tears the ALB down while the cluster is up), then the cluster:
+
 ```bash
+cd workloads
+tofu destroy
+cd ..
 tofu destroy
 ```

 ## Troubleshooting

-### Targets Unhealthy / ALB Returns 5xx
+### The Ingress Has No Address

-The ALB health-checks `/` on the NodePort. Targets turn healthy once the demo pods are running *and* the SG rule admits the ALB. Check, in order:
+The LBC populates `status.loadBalancer` once it has provisioned the ALB. If it stays empty:

 ```bash
-kubectl get pods -o wide                                  # pods Running?
-aws ec2 describe-security-groups \
-  --filters "Name=group-name,Values=eks-cluster-eks-https-nodegroup-sg" \
-  --query 'SecurityGroups[0].IpPermissions'               # NodePort rule present?
-aws elbv2 describe-target-health --target-group-arn "$TG_ARN"
+kubectl describe ingress spinifex-demo            # events show LBC reconcile errors
+kubectl -n kube-system logs deploy/aws-load-balancer-controller --tail=50
+aws eks describe-addon --cluster-name eks-https --addon-name aws-load-balancer-controller --query 'addon.status'
 ```

-If pods are `ImagePullBackOff`, the workers can't reach Docker Hub — confirm the NAT gateway is `available`:
+Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingress = "false"` (otherwise the built-in ingress competes).
+
+> **Verify on a live cluster.** The LBC reconcile path (ALB creation, target registration, listener wiring) exercises the running Spinifex EKS data plane end to end. Stand this up on a live stack to confirm the ALB serves the page; the manifests here are the AWS-parity shape of that flow.
+
+### HTTPS Loads but Returns 502 / 504
+
+The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:

 ```bash
-aws ec2 describe-nat-gateways --query 'NatGateways[].[NatGatewayId,State]'
+kubectl get pods -o wide
+kubectl get svc spinifex-demo
+aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-https-nodegroup-sg" \
+  --query 'SecurityGroups[0].IpPermissions'
 ```

-### Workers Never Join
+The `eks-https-alb-nodeport` rule must admit the VPC CIDR on the NodePort, and the target group health check (`/healthz`) must pass.

-The workers join over the private endpoint. Confirm it's enabled and the node group is progressing:
+### Addon Fails With "no baked bundle"

-```bash
-aws eks describe-cluster --name eks-https --query 'cluster.resourcesVpcConfig.endpointPrivateAccess'
-aws eks describe-nodegroup --cluster-name eks-https --nodegroup-name workers --query 'nodegroup.status'
-```
+Spinifex only installs addons whose manifests are baked into the `eks-node` AMI. If `describe-addon` reports `CREATE_FAILED` with `no baked bundle`, the running worker image predates the LBC bundle — rebuild and republish the `eks-node` image, then recreate the addon.

-### Target-Group Attachment Errors at Apply
+### Pods Stuck in ImagePullBackOff

-The attachment indexes the discovered worker IDs by `node_desired_size`. If `apply` errors with an index out of range, the workers weren't all tagged/running when the `aws_instances` lookup ran. Re-run `tofu apply` — the node group will be `ACTIVE` by then:
-
-```bash
-aws ec2 describe-instances --filters "Name=tag:spinifex:eks-cluster,Values=eks-https" \
-  --query 'Reservations[].Instances[].[InstanceId,State.Name]' --output text
-```
-
-### kubernetes provider: connection refused / Unauthorized
-
-The provider runs `aws eks get-token` against the Spinifex STS endpoint. If the cluster was still `CREATING` when the provider first connected, re-run `tofu apply`. Otherwise confirm:
-
-```bash
-aws sts get-caller-identity
-aws eks describe-cluster --name eks-https --query 'cluster.status'
-```
-
-### ACM Import / HTTPS Listener Errors
-
-```bash
-aws acm list-certificates
-aws acm describe-certificate --certificate-arn "$(tofu output -raw certificate_arn)" \
-  --query 'Certificate.Status'
-```
-
-A `CertificateNotFound` on the listener means the ACM ARN didn't resolve — check the cert exists and belongs to the same account/profile.
+The private workers pull from ECR through the NAT gateway. Confirm the image was pushed (`tofu output ecr_repository_url`), the NAT route is in place, and the node role carries `AmazonEC2ContainerRegistryReadOnly`.

 ### Provider Connection Refused

@@ -1,32 +1,31 @@
-# Example: 3-Node EKS behind an HTTPS ALB (ACM TLS) with a load-balanced app
+# Example: EKS HTTPS Ingress via the AWS Load Balancer Controller + ACM
 #
-# The largest EKS workbook, and the one with the clearest payoff. It stands up
-# an HA-shaped cluster sized for a 3-node Spinifex deployment, deploys a demo
-# web app across the workers, and publishes it through an internet-facing ALB
-# that terminates TLS with a certificate held in ACM.
+# Builds on eks-quickstart. Instead of poking a NodePort open on the worker, this
+# workbook serves the Spinifex-themed demo app over HTTPS through an Application
+# Load Balancer that the AWS Load Balancer Controller (LBC) provisions from a
+# Kubernetes Ingress — the same pattern you'd use on AWS EKS.
 #
-# After `apply`, fetch the ALB's public IP (see alb_public_ip_hint) and open
-# https://<ip> in a browser. The page reports which pod answered; refresh and
-# it moves between the three replicas spread across the cluster — a live picture
-# of a multi-node Kubernetes cluster load-balancing behind HTTPS.
+# What it adds over the quickstart:
+#   * Public subnets for the ALB + a NAT gateway, private subnets for the workers.
+#   * The aws-load-balancer-controller addon, installed through the EKS API.
+#   * The cluster tag spinifex.io/managed-ingress = "false", which disables K3s'
+#     built-in traefik/servicelb so the LBC owns ingress (AWS parity).
+#   * A self-signed certificate imported into ACM and attached to the ALB's HTTPS
+#     listener via an Ingress annotation.
 #
-# What it builds:
-#   * VPC with two public subnets (ALB + NAT) and two private subnets (workers).
-#   * A NAT gateway so the private workers can pull the demo container image.
-#   * EKS cluster with BOTH a public endpoint (kubectl from your host) and a
-#     private endpoint (the in-VPC path the workers use to join).
-#   * A managed node group of three workers (t3.large) in the private subnets.
-#   * CoreDNS addon.
-#   * A self-signed cert imported into ACM (Spinifex ACM supports
-#     ImportCertificate, not RequestCertificate), wired to the ALB's HTTPS
-#     listener, which forwards to the workers' NodePort.
-#   * The demo Deployment + NodePort Service, deployed by the Kubernetes
-#     provider, plus the one SG rule that lets the ALB reach the NodePort.
+# The workloads/ module then creates a Deployment, a NodePort Service, and an
+# Ingress (ingressClassName: alb). The LBC reconciles that Ingress into an
+# internet-facing ALB that terminates TLS with the ACM cert and forwards to the
+# workers' NodePort. The cluster's ELB-eligible subnets are injected by Spinifex
+# into the alb IngressClassParams, so the Ingress needs no subnet annotations.
 #
 # Usage:
 #   cd spinifex/docs/terraform-workbooks/eks-https-ingress
 #   export AWS_PROFILE=spinifex
 #   tofu init && tofu apply
+#   # build + push the demo image to the ECR repo this creates (see README),
+#   # then: cd workloads && tofu init && tofu apply
+#   # finally: kubectl get ingress spinifex-demo -o wide  → open https://<address>

 terraform {
  required_version = ">= 1.6.0"
@@ -40,10 +39,6 @@ terraform {
      source  = "hashicorp/tls"
      version = ">= 4.0"
    }
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = ">= 2.20"
-    }
  }
 }

@@ -68,19 +63,24 @@ variable "k8s_version" {

 variable "node_instance_type" {
  type    = string
-  default = "t3.large"
+  default = "t3.medium"
 }

 variable "node_desired_size" {
  type        = number
-  default     = 3
-  description = "Worker count; spread across a 3-node Spinifex deployment for HA"
+  default     = 1
+  description = "Worker count. Use 1 for a single-node demo, or 3 for an HA-shaped cluster."
+
+  validation {
+    condition     = var.node_desired_size == 1 || var.node_desired_size == 3
+    error_message = "node_desired_size must be 1 or 3."
+  }
 }

 variable "node_port" {
  type        = number
  default     = 30080
-  description = "NodePort the ALB forwards to and the demo Service is published on"
+  description = "NodePort the demo Service is published on and the ALB forwards to"
 }

 variable "cert_common_name" {
@@ -94,12 +94,6 @@ variable "api_public_access_cidr" {
  description = "CIDR allowed to reach the public Kubernetes API endpoint; tighten in production"
 }

-variable "alb_ingress_cidr" {
-  type        = string
-  default     = "0.0.0.0/0"
-  description = "CIDR allowed to reach the ALB HTTPS listener"
-}
-
 variable "spinifex_endpoint" {
  type    = string
  default = "https://127.0.0.1:9999"
@@ -113,12 +107,12 @@ provider "aws" {
  region = var.region

  endpoints {
-    ec2                    = var.spinifex_endpoint
-    iam                    = var.spinifex_endpoint
-    sts                    = var.spinifex_endpoint
-    eks                    = var.spinifex_endpoint
-    acm                    = var.spinifex_endpoint
-    elasticloadbalancingv2 = var.spinifex_endpoint
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+    eks = var.spinifex_endpoint
+    ecr = var.spinifex_endpoint
+    acm = var.spinifex_endpoint
  }

  skip_credentials_validation = true
@@ -127,17 +121,6 @@ provider "aws" {
  skip_region_validation      = true
 }

-provider "kubernetes" {
-  host                   = aws_eks_cluster.this.endpoint
-  cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
-
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    command     = "aws"
-    args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
-  }
-}
-
 data "aws_availability_zones" "available" {
  state = "available"
 }
@@ -147,7 +130,7 @@ data "aws_availability_zones" "available" {
 # ---------------------------------------------------------------------------

 resource "aws_vpc" "main" {
-  cidr_block           = "10.32.0.0/16"
+  cidr_block           = "10.31.0.0/16"
  enable_dns_hostnames = true
  enable_dns_support   = true

@@ -166,7 +149,7 @@ resource "aws_internet_gateway" "igw" {

 resource "aws_subnet" "public_a" {
  vpc_id                  = aws_vpc.main.id
-  cidr_block              = "10.32.1.0/24"
+  cidr_block              = "10.31.1.0/24"
  availability_zone       = data.aws_availability_zones.available.names[0]
  map_public_ip_on_launch = true

@@ -177,7 +160,7 @@ resource "aws_subnet" "public_a" {

 resource "aws_subnet" "public_b" {
  vpc_id                  = aws_vpc.main.id
-  cidr_block              = "10.32.2.0/24"
+  cidr_block              = "10.31.2.0/24"
  availability_zone       = data.aws_availability_zones.available.names[0]
  map_public_ip_on_launch = true

@@ -188,7 +171,7 @@ resource "aws_subnet" "public_b" {

 resource "aws_subnet" "private_a" {
  vpc_id            = aws_vpc.main.id
-  cidr_block        = "10.32.11.0/24"
+  cidr_block        = "10.31.11.0/24"
  availability_zone = data.aws_availability_zones.available.names[0]

  tags = {
@@ -198,7 +181,7 @@ resource "aws_subnet" "private_a" {

 resource "aws_subnet" "private_b" {
  vpc_id            = aws_vpc.main.id
-  cidr_block        = "10.32.12.0/24"
+  cidr_block        = "10.31.12.0/24"
  availability_zone = data.aws_availability_zones.available.names[0]

  tags = {
@@ -229,9 +212,8 @@ resource "aws_route_table_association" "public_b" {
  route_table_id = aws_route_table.public.id
 }

-# NAT gateway gives the private workers outbound internet to pull the demo
-# container image. The workers still *join* the cluster over the in-VPC private
-# endpoint, not through the NAT.
+# NAT gateway gives the private workers outbound internet to pull the demo image
+# from ECR. The workers still join the cluster over the in-VPC private endpoint.
 resource "aws_eip" "nat" {
  domain = "vpc"

@@ -328,8 +310,82 @@ resource "aws_iam_role_policy_attachment" "node_ecr" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
 }

+# The AWS Load Balancer Controller runs with the node's instance-profile
+# credentials (Spinifex wires creds at the node level, not IRSA — see the addon
+# block below), so the permissions it needs to manage ALBs must live on the node
+# role. This mirrors the upstream AWSLoadBalancerControllerIAMPolicy, trimmed to
+# the actions an ALB Ingress exercises (Shield/WAF/Cognito are disabled on the
+# controller). Resources are "*" since Spinifex evaluates grants by action.
+# A customer-managed policy + attachment is used (Spinifex implements
+# CreatePolicy/AttachRolePolicy, not inline PutRolePolicy).
+resource "aws_iam_policy" "node_lbc" {
+  name = "${var.cluster_name}-node-lbc"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "ec2:DescribeAccountAttributes",
+          "ec2:DescribeAddresses",
+          "ec2:DescribeAvailabilityZones",
+          "ec2:DescribeInternetGateways",
+          "ec2:DescribeVpcs",
+          "ec2:DescribeSubnets",
+          "ec2:DescribeSecurityGroups",
+          "ec2:DescribeInstances",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DescribeTags",
+          "ec2:GetCoipPoolUsage",
+          "ec2:DescribeCoipPools",
+          "ec2:CreateSecurityGroup",
+          "ec2:CreateTags",
+          "ec2:DeleteTags",
+          "ec2:AuthorizeSecurityGroupIngress",
+          "ec2:RevokeSecurityGroupIngress",
+          "ec2:DeleteSecurityGroup",
+          "elasticloadbalancing:*",
+          "acm:ListCertificates",
+          "acm:DescribeCertificate",
+          "acm:GetCertificate",
+          "iam:CreateServiceLinkedRole",
+          "iam:ListServerCertificates",
+          "iam:GetServerCertificate",
+          "tag:GetResources",
+          "tag:TagResources",
+          "wafv2:GetWebACLForResource",
+          "shield:GetSubscriptionState"
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_lbc" {
+  role       = aws_iam_role.node.name
+  policy_arn = aws_iam_policy.node_lbc.arn
+}
+
 # ---------------------------------------------------------------------------
-# EKS cluster — public + private endpoints
+# ECR — repository the workers pull the demo image from
+# ---------------------------------------------------------------------------
+
+resource "aws_ecr_repository" "demo" {
+  name = "spinifex-demo"
+
+  # tofu destroy must remove the repo even though it still holds the pushed demo
+  # image; without this, DeleteRepository (force=false) fails RepositoryNotEmpty.
+  force_delete = true
+}
+
+# ---------------------------------------------------------------------------
+# EKS cluster — public + private endpoints, LBC-owned ingress
+#
+# The spinifex.io/managed-ingress = "false" tag disables K3s' built-in
+# traefik/servicelb so the AWS Load Balancer Controller owns ingress, matching
+# how ingress works on AWS EKS.
 # ---------------------------------------------------------------------------

 resource "aws_eks_cluster" "this" {
@@ -352,12 +408,13 @@ resource "aws_eks_cluster" "this" {
  depends_on = [aws_iam_role_policy_attachment.cluster]

  tags = {
-    Name = var.cluster_name
+    Name                          = var.cluster_name
+    "spinifex.io/managed-ingress" = "false"
  }
 }

 # ---------------------------------------------------------------------------
-# Managed node group — three workers in the private subnets
+# Managed node group — workers in the private subnets
 # ---------------------------------------------------------------------------

 resource "aws_eks_node_group" "workers" {
@@ -387,27 +444,32 @@ resource "aws_eks_node_group" "workers" {
 }

 # ---------------------------------------------------------------------------
-# Addon — CoreDNS
+# Addon — AWS Load Balancer Controller
+#
+# Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
+# node level, so no IRSA role or subnet tagging is needed here. addon_version is
+# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
+# so let Spinifex default to its catalog version.
 # ---------------------------------------------------------------------------

-resource "aws_eks_addon" "coredns" {
+resource "aws_eks_addon" "lbc" {
  cluster_name                = aws_eks_cluster.this.name
-  addon_name                  = "coredns"
+  addon_name                  = "aws-load-balancer-controller"
  resolve_conflicts_on_create = "OVERWRITE"

-  # addon_version omitted on purpose. Spinifex defaults an unset version to its
-  # catalog default (coredns 1.11.1). The AWS provider also rejects a bare
-  # "1.11.1" (it demands a v-prefixed form) which Spinifex's catalog would in
-  # turn reject — so pinning is a dead end here; let the server choose.
  depends_on = [aws_eks_node_group.workers]

  tags = {
-    Name = "${var.cluster_name}-coredns"
+    Name = "${var.cluster_name}-lbc"
  }
 }

 # ---------------------------------------------------------------------------
 # TLS — self-signed certificate imported into ACM
+#
+# Spinifex ACM supports ImportCertificate (not RequestCertificate), so the cert
+# is generated locally by the tls provider and imported. The workloads Ingress
+# attaches it to the ALB's HTTPS listener by ARN.
 # ---------------------------------------------------------------------------

 resource "tls_private_key" "ingress" {
@@ -443,107 +505,12 @@ resource "aws_acm_certificate" "ingress" {
  }
 }

-# ---------------------------------------------------------------------------
-# ALB — internet-facing, HTTPS terminated with the ACM cert
-# ---------------------------------------------------------------------------
-
-resource "aws_security_group" "alb" {
-  name        = "${var.cluster_name}-alb-sg"
-  description = "ALB HTTPS inbound, NodePort to workers"
-  vpc_id      = aws_vpc.main.id
-
-  ingress {
-    description = "HTTPS"
-    from_port   = 443
-    to_port     = 443
-    protocol    = "tcp"
-    cidr_blocks = [var.alb_ingress_cidr]
-  }
-
-  egress {
-    description = "All outbound"
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
-  tags = {
-    Name = "${var.cluster_name}-alb-sg"
-  }
-}
-
-resource "aws_lb" "ingress" {
-  name               = "${var.cluster_name}-alb"
-  internal           = false
-  load_balancer_type = "application"
-  security_groups    = [aws_security_group.alb.id]
-  subnets            = [aws_subnet.public_a.id, aws_subnet.public_b.id]
-
-  tags = {
-    Name = "${var.cluster_name}-alb"
-  }
-}
-
-resource "aws_lb_target_group" "workers" {
-  name        = "${var.cluster_name}-tg"
-  port        = var.node_port
-  protocol    = "HTTP"
-  target_type = "instance"
-  vpc_id      = aws_vpc.main.id
-
-  health_check {
-    path                = "/"
-    protocol            = "HTTP"
-    matcher             = "200-399"
-    healthy_threshold   = 2
-    unhealthy_threshold = 3
-    timeout             = 5
-    interval            = 10
-  }
-
-  tags = {
-    Name = "${var.cluster_name}-tg"
-  }
-}
-
-# Discover the launched workers by their Spinifex EKS tag so the ALB can target
-# them. depends_on the node group: the lookup must run after the workers exist.
-data "aws_instances" "workers" {
-  instance_tags = {
-    "spinifex:eks-cluster" = aws_eks_cluster.this.name
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-# count is the known desired size, so it's resolvable at plan time even though
-# the worker IDs themselves are only known after the node group launches.
-resource "aws_lb_target_group_attachment" "workers" {
-  count            = var.node_desired_size
-  target_group_arn = aws_lb_target_group.workers.arn
-  target_id        = data.aws_instances.workers.ids[count.index]
-  port             = var.node_port
-}
-
-resource "aws_lb_listener" "https" {
-  load_balancer_arn = aws_lb.ingress.arn
-  port              = 443
-  protocol          = "HTTPS"
-  ssl_policy        = "ELBSecurityPolicy-TLS13-1-2-2021-06"
-  certificate_arn   = aws_acm_certificate.ingress.arn
-
-  default_action {
-    type             = "forward"
-    target_group_arn = aws_lb_target_group.workers.arn
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Let the ALB reach the workers' NodePort
 #
 # Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# Look it up by its deterministic name and admit the ALB SG on the NodePort.
+# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
+# NodePort. Look the SG up by its deterministic name and add one rule.
 # ---------------------------------------------------------------------------

 data "aws_security_group" "nodegroup" {
@@ -560,88 +527,18 @@ data "aws_security_group" "nodegroup" {
  depends_on = [aws_eks_node_group.workers]
 }

-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_alb" {
-  security_group_id            = data.aws_security_group.nodegroup.id
-  referenced_security_group_id = aws_security_group.alb.id
-  from_port                    = var.node_port
-  to_port                      = var.node_port
-  ip_protocol                  = "tcp"
+resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
+  security_group_id = data.aws_security_group.nodegroup.id
+  cidr_ipv4         = aws_vpc.main.cidr_block
+  from_port         = var.node_port
+  to_port           = var.node_port
+  ip_protocol       = "tcp"

  tags = {
    Name = "${var.cluster_name}-alb-nodeport"
  }
 }

-# ---------------------------------------------------------------------------
-# Demo workload — three replicas spread across the workers
-# ---------------------------------------------------------------------------
-
-resource "kubernetes_deployment_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-    labels    = { app = "hello" }
-  }
-
-  spec {
-    replicas = var.node_desired_size
-
-    selector {
-      match_labels = { app = "hello" }
-    }
-
-    template {
-      metadata {
-        labels = { app = "hello" }
-      }
-
-      spec {
-        container {
-          name  = "hello"
-          image = "nginxdemos/hello"
-
-          port {
-            container_port = 80
-          }
-        }
-
-        # Spread one replica per node so a refresh visibly hits different nodes.
-        topology_spread_constraint {
-          max_skew           = 1
-          topology_key       = "kubernetes.io/hostname"
-          when_unsatisfiable = "ScheduleAnyway"
-
-          label_selector {
-            match_labels = { app = "hello" }
-          }
-        }
-      }
-    }
-  }
-
-  depends_on = [aws_eks_addon.coredns]
-}
-
-resource "kubernetes_service_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-  }
-
-  spec {
-    selector = { app = "hello" }
-    type     = "NodePort"
-
-    port {
-      port        = 80
-      target_port = 80
-      node_port   = var.node_port
-    }
-  }
-
-  depends_on = [kubernetes_deployment_v1.hello]
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------
@@ -650,20 +547,29 @@ output "cluster_name" {
  value = aws_eks_cluster.this.name
 }

+output "region" {
+  value = var.region
+}
+
+output "node_port" {
+  value = var.node_port
+}
+
+output "node_desired_size" {
+  value = var.node_desired_size
+}
+
 output "certificate_arn" {
  value = aws_acm_certificate.ingress.arn
 }

-output "alb_dns_name" {
-  value = aws_lb.ingress.dns_name
+output "ecr_repository_url" {
+  value       = aws_ecr_repository.demo.repository_url
+  description = "Push the demo-app image here, then apply the workloads module"
 }

-output "alb_public_ip_hint" {
-  value = "aws elbv2 describe-load-balancers --names ${var.cluster_name}-alb --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' --output text"
-}
-
-output "demo_url_hint" {
-  value = "Fetch the ALB IP via alb_public_ip_hint, then open https://<that-ip> (self-signed cert: curl -k). Refresh to see different pods answer."
+output "ingress_address_hint" {
+  value = "After applying workloads: kubectl get ingress spinifex-demo -o jsonpath='{.status.loadBalancer.ingress[0].hostname}{\"\\n\"}' — then open https://<that-address> (self-signed cert: curl -k)."
 }

 output "update_kubeconfig" {
@@ -0,0 +1,258 @@
+# Demo workload for eks-https-ingress — separate root module / state.
+#
+# Kept apart from the cluster root module on purpose: the kubernetes provider
+# below reads the cluster endpoint from a *live* data source, so it is only ever
+# configured while the cluster exists. Destroy this module before the parent and
+# the provider never falls back to localhost.
+#
+# Creates the Spinifex-themed demo Deployment, a NodePort Service, and an Ingress
+# (ingressClassName: alb). The AWS Load Balancer Controller reconciles the
+# Ingress into an internet-facing ALB that terminates TLS with the imported ACM
+# certificate and forwards to the workers' NodePort.
+#
+# Usage:
+#   cd spinifex/docs/terraform-workbooks/eks-https-ingress
+#   tofu init && tofu apply                    # parent: cluster + LBC + ACM
+#   cd workloads && tofu init && tofu apply    # this module: demo app + Ingress
+#   # teardown is the reverse — destroy here first, then the parent.
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40, < 6.0"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.20"
+    }
+  }
+}
+
+variable "spinifex_endpoint" {
+  type    = string
+  default = "https://127.0.0.1:9999"
+}
+
+variable "replicas" {
+  type    = number
+  default = 2
+}
+
+variable "demo_image" {
+  type        = string
+  default     = ""
+  description = "Demo image ref. Defaults to the parent's ECR repository URL at :latest."
+}
+
+variable "inbound_cidr" {
+  type        = string
+  default     = "0.0.0.0/0"
+  description = "CIDR allowed to reach the ALB HTTPS listener"
+}
+
+provider "aws" {
+  region = data.terraform_remote_state.infra.outputs.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+    eks = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# Cluster identity comes from the parent module's state; the live endpoint/CA
+# come from a data source so the provider is only configured while the cluster
+# is up.
+data "terraform_remote_state" "infra" {
+  backend = "local"
+
+  config = {
+    path = "../terraform.tfstate"
+  }
+}
+
+locals {
+  cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
+  region       = data.terraform_remote_state.infra.outputs.region
+  node_port    = data.terraform_remote_state.infra.outputs.node_port
+  cert_arn     = data.terraform_remote_state.infra.outputs.certificate_arn
+  demo_image   = var.demo_image != "" ? var.demo_image : "${data.terraform_remote_state.infra.outputs.ecr_repository_url}:latest"
+}
+
+data "aws_eks_cluster" "this" {
+  name = local.cluster_name
+}
+
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.this.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
+
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
+  }
+}
+
+# The Spinifex-themed demo app reports the pod, node, cluster, and region that
+# served the request. The pod and node names come from the downward API.
+resource "kubernetes_deployment_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+    labels    = { app = "spinifex-demo" }
+  }
+
+  spec {
+    replicas = var.replicas
+
+    selector {
+      match_labels = { app = "spinifex-demo" }
+    }
+
+    template {
+      metadata {
+        labels = { app = "spinifex-demo" }
+      }
+
+      spec {
+        container {
+          name  = "spinifex-demo"
+          image = local.demo_image
+
+          port {
+            container_port = 8080
+          }
+
+          env {
+            name = "POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+
+          env {
+            name = "NODE_NAME"
+            value_from {
+              field_ref {
+                field_path = "spec.nodeName"
+              }
+            }
+          }
+
+          env {
+            name = "POD_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+
+          env {
+            name  = "CLUSTER_NAME"
+            value = local.cluster_name
+          }
+
+          env {
+            name  = "AWS_REGION"
+            value = local.region
+          }
+
+          env {
+            name  = "APP_TITLE"
+            value = "Spinifex EKS — HTTPS Ingress"
+          }
+
+          readiness_probe {
+            http_get {
+              path = "/healthz"
+              port = 8080
+            }
+            initial_delay_seconds = 3
+            period_seconds        = 10
+          }
+        }
+      }
+    }
+  }
+}
+
+# NodePort Service: the ALB target group registers the workers on this port.
+resource "kubernetes_service_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+  }
+
+  spec {
+    selector = { app = "spinifex-demo" }
+    type     = "NodePort"
+
+    port {
+      port        = 80
+      target_port = 8080
+      node_port   = local.node_port
+    }
+  }
+
+  depends_on = [kubernetes_deployment_v1.demo]
+}
+
+# Ingress reconciled by the AWS Load Balancer Controller into an internet-facing
+# ALB. target-type instance registers the workers' NodePort; the ACM cert ARN
+# attaches to the HTTPS:443 listener; HTTP is redirected to HTTPS. ELB subnets
+# are injected by Spinifex into the alb IngressClassParams, so none are set here.
+resource "kubernetes_ingress_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+
+    annotations = {
+      "alb.ingress.kubernetes.io/scheme"           = "internet-facing"
+      "alb.ingress.kubernetes.io/target-type"      = "instance"
+      "alb.ingress.kubernetes.io/listen-ports"     = "[{\"HTTP\":80},{\"HTTPS\":443}]"
+      "alb.ingress.kubernetes.io/certificate-arn"  = local.cert_arn
+      "alb.ingress.kubernetes.io/ssl-redirect"     = "443"
+      "alb.ingress.kubernetes.io/healthcheck-path" = "/healthz"
+      "alb.ingress.kubernetes.io/inbound-cidrs"    = var.inbound_cidr
+    }
+  }
+
+  spec {
+    ingress_class_name = "alb"
+
+    rule {
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+
+          backend {
+            service {
+              name = kubernetes_service_v1.demo.metadata[0].name
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+output "ingress_name" {
+  value = kubernetes_ingress_v1.demo.metadata[0].name
+}
@@ -1,6 +1,6 @@
 ---
 title: "EKS Quickstart"
-description: "Stand up a minimal managed-Kubernetes cluster and a browsable demo app — VPC, IAM roles, an EKS cluster, a single-worker node group, and a load-balanced web page — using Terraform on Spinifex."
+description: "Stand up a minimal managed-Kubernetes cluster and a browsable Spinifex-themed demo app — VPC, IAM roles, an EKS cluster, a one- or three-worker node group, an ECR repository, and a load-balanced web page — using Terraform on Spinifex."
 category: "Terraform Workbooks"
 tags:
  - terraform
@@ -22,7 +22,7 @@ resources:

 # Terraform: EKS Quickstart

-> The smallest end-to-end EKS example on Spinifex: a VPC, the two IAM roles EKS needs, a single-worker cluster, and a demo web app you can open in a browser.
+> The smallest end-to-end EKS example on Spinifex: a VPC, the two IAM roles EKS needs, a one- or three-worker cluster, an ECR repository, and a Spinifex-themed demo app you can open in a browser.

 ## Table of Contents

@@ -34,9 +34,9 @@ resources:

 ## Overview

-Provision a managed Kubernetes cluster on Spinifex with Terraform/OpenTofu — and finish with something you can actually see. As well as the cluster, this workbook deploys a small demo app onto it and publishes it on the worker's public IP. When `apply` finishes, open the **`demo_url`** output: the page is served by `nginxdemos/hello` and prints the name of the pod that answered. Refresh it and the answering pod changes, so even a non-technical viewer can watch Kubernetes scheduling and load-balancing in real time.
+Provision a managed Kubernetes cluster on Spinifex with Terraform/OpenTofu — and finish with something you can actually see. As well as the cluster, this workbook creates an **ECR repository**, then deploys the Spinifex-themed demo app onto the cluster and publishes it on the worker's public IP. When `apply` finishes, open the **`demo_url`** output: the page reports the **pod, node, cluster, and region** that answered. Refresh it and the answering pod changes, so even a non-technical viewer can watch Kubernetes scheduling and load-balancing in real time.

-Under the hood it keeps things minimal: a VPC with two public subnets, an IAM role for the control plane, an IAM role for the workers with the three managed policies AWS-managed node groups expect, a public-endpoint `aws_eks_cluster`, and a single-node `aws_eks_node_group`. Terraform then uses the **Kubernetes provider** (authenticating exactly like `kubectl` does, via `aws eks get-token`) to deploy the Deployment + NodePort Service.
+Under the hood it keeps things minimal: a VPC with two public subnets, an IAM role for the control plane, an IAM role for the workers with the three managed policies AWS-managed node groups expect, a public-endpoint `aws_eks_cluster`, and a node group sized by **`node_desired_size`** (1 for a single-node demo, or 3 for an HA-shaped cluster). You build the demo image and push it to the ECR repository, then Terraform uses the **Kubernetes provider** (authenticating exactly like `kubectl` does, via `aws eks get-token`) to deploy the Deployment + NodePort Service.

 **What you'll learn:**

@@ -54,11 +54,12 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 | Subnets | `eks-quickstart-subnet-a/-b` | Public subnets for the cluster and worker |
 | Internet Gateway | `eks-quickstart-igw` | Egress so the worker can pull the demo image |
 | IAM Roles | `eks-quickstart-cluster-role`, `eks-quickstart-node-role` | Control-plane and worker roles |
+| ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
 | EKS Cluster | `eks-quickstart` | Public API endpoint, API auth mode, Kubernetes 1.32 |
-| Node Group | `default` | One `t3.medium` worker |
+| Node Group | `default` | `node_desired_size` `t3.medium` worker(s) — 1 or 3 |
 | SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the auto-managed worker SG |
-| K8s Deployment | `hello` | `nginxdemos/hello`, 2 replicas |
-| K8s Service | `hello` | NodePort publishing the demo on the worker |
+| K8s Deployment | `spinifex-demo` | The themed demo image from ECR, 2 replicas |
+| K8s Service | `spinifex-demo` | NodePort publishing the demo on the worker |

 **Spinifex specifics**

@@ -72,6 +73,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 - Spinifex installed and running (see [Installing Spinifex](/docs/install))
 - The Spinifex `eks-node` image available on the cluster
 - OpenTofu or Terraform, plus `kubectl` and the AWS CLI
+- Docker, to build and push the demo image to ECR

 ## Instructions

@@ -90,23 +92,57 @@ Or create a `main.tf` file and paste the full configuration below.

 ### Step 2. Deploy

+The cluster and the demo app are **two root modules with separate state**. Apply the cluster first:
+
 ```bash
 export AWS_PROFILE=spinifex
 tofu init
 tofu apply
 ```

-This single `apply` does the whole job: it creates the cluster (which bootstraps a control-plane VM and brings up k3s — a few minutes in `CREATING`), launches the worker, opens the NodePort, then deploys the demo app and waits for the pods to roll out.
+This creates the cluster (which bootstraps a control-plane VM and brings up k3s — a few minutes in `CREATING`), launches the worker(s), creates the `spinifex-demo` ECR repository, and opens the NodePort.

-> **Same profile for the Kubernetes provider.** The Kubernetes provider authenticates by shelling out to `aws eks get-token`, which has to reach the Spinifex STS endpoint. Keep `AWS_PROFILE=spinifex` exported for the whole `apply`.
+Set the worker count with `node_desired_size` (`1` for a single node, `3` for an HA-shaped cluster):
+
+```bash
+tofu apply -var node_desired_size=3
+```
+
+Once the cluster is `ACTIVE`, **build and push the demo image** to the ECR repository this created (full commands in [`../demo-app/README.md`](../demo-app/README.md)):
+
+```bash
+cd ../demo-app
+REGISTRY=$(cd ../eks-quickstart && tofu output -raw ecr_repository_url)
+REGISTRY_HOST=${REGISTRY%%/*}
+aws ecr get-login-password | docker login --username AWS --password-stdin "$REGISTRY_HOST"
+docker build -t "${REGISTRY}:latest" .
+docker push "${REGISTRY}:latest"
+cd ../eks-quickstart
+```
+
+Then deploy the demo app from the nested `workloads/` module — it defaults `demo_image` to the parent's ECR repository at `:latest`:
+
+```bash
+cd workloads
+tofu init
+tofu apply
+```
+
+> **Why two modules?** The Kubernetes provider in `workloads/` reads the cluster endpoint from a live `data "aws_eks_cluster"` source, so it's only ever configured while the cluster exists. Keeping it out of the cluster module means `destroy` never tries to refresh a workload against a cluster that's already gone — the failure mode where the provider falls back to `http://localhost:80` and reports `connection refused`. Always destroy `workloads/` before the cluster.
+
+> **Same profile for the Kubernetes provider.** The Kubernetes provider authenticates by shelling out to `aws eks get-token`, which has to reach the Spinifex STS endpoint. Keep `AWS_PROFILE=spinifex` exported for both applies.
+
+<!-- INCLUDE: workloads/main.tf lang:hcl -->

 ### Step 3. Open the Demo

+Run from the cluster module directory (`cd ..` if you're still in `workloads/`):
+
 ```bash
 tofu output demo_url
 ```

-Open that URL in a browser. You'll see the `nginxdemos/hello` page showing the **Server name** (the pod that handled the request). Refresh a few times — with two replicas, the pod name alternates, demonstrating that the Service is load-balancing across the cluster.
+Open that URL in a browser. You'll see the Spinifex-themed page reporting the **pod, node, cluster, and region** that handled the request. Refresh a few times — with two replicas, the pod name alternates, demonstrating that the Service is load-balancing across the cluster.

 ### Step 4. Inspect with kubectl (optional)

@@ -120,7 +156,12 @@ kubectl get pods -o wide

 ### Cleanup

+Destroy in reverse — the demo app first (while the cluster is still up), then the cluster:
+
 ```bash
+cd workloads
+tofu destroy
+cd ..
 tofu destroy
 ```

@@ -143,7 +184,7 @@ aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-e
  --query 'SecurityGroups[0].IpPermissions'
 ```

-If the pods are `Pending`, the worker may not be `Ready` yet — give it a moment. If they're `ImagePullBackOff`, the worker can't reach Docker Hub; confirm the IGW route and the worker's public IP.
+If the pods are `Pending`, the worker may not be `Ready` yet — give it a moment. If they're `ImagePullBackOff`, the worker can't pull from ECR: confirm you built and pushed the image (`tofu output ecr_repository_url`), that the IGW route and the worker's public IP are in place, and that the node role carries `AmazonEC2ContainerRegistryReadOnly`.

 ### kubernetes provider: connection refused / Unauthorized

@@ -1,23 +1,27 @@
 # Example: EKS Quickstart on Spinifex
 #
 # A minimal managed-Kubernetes cluster that ends in something you can see: a
-# VPC, the two IAM roles EKS needs, a single-worker cluster, and a demo web app
-# deployed onto it by Terraform and exposed on the worker's public IP.
+# VPC, the two IAM roles EKS needs, a one- or three-worker cluster, an ECR
+# repository, and the Spinifex-themed demo app deployed onto it by Terraform and
+# exposed on the worker's public IP.
 #
-# Once `apply` finishes, open the demo_url output in a browser. The page is
-# served by the nginxdemos/hello image and prints the pod name that answered —
-# refresh and it bounces between the app's replicas, showing the cluster is
-# really scheduling and load-balancing pods.
+# Once `apply` finishes, open the demo_url output in a browser. The page reports
+# the pod, node, cluster, and region that answered — refresh and it bounces
+# between the app's replicas, showing the cluster is really scheduling and
+# load-balancing pods.
 #
 # Demonstrates: VPC + subnets, EKS cluster + node IAM roles, an aws_eks_cluster
-# and managed aws_eks_node_group, the Kubernetes provider authenticating to the
-# cluster, and a Deployment + NodePort Service reachable from your browser.
+# and managed aws_eks_node_group, an ECR repository the workers pull from, the
+# Kubernetes provider authenticating to the cluster, and a Deployment + NodePort
+# Service reachable from your browser.
 #
 # Usage:
 #   cd spinifex/docs/terraform-workbooks/eks-quickstart
 #   export AWS_PROFILE=spinifex
 #   tofu init && tofu apply
-#   # then open the demo_url output
+#   # build + push the demo image to the ECR repo this creates (see README),
+#   # then: cd workloads && tofu init && tofu apply
+#   # finally open the demo_url output

 terraform {
  required_version = ">= 1.6.0"
@@ -27,10 +31,6 @@ terraform {
      source  = "hashicorp/aws"
      version = ">= 5.40, < 6.0"
    }
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = ">= 2.20"
-    }
  }
 }

@@ -59,18 +59,23 @@ variable "instance_type" {
  default = "t3.medium"
 }

+variable "node_desired_size" {
+  type        = number
+  default     = 1
+  description = "Worker count. Use 1 for a single-node demo, or 3 for an HA-shaped cluster."
+
+  validation {
+    condition     = var.node_desired_size == 1 || var.node_desired_size == 3
+    error_message = "node_desired_size must be 1 or 3."
+  }
+}
+
 variable "node_port" {
  type        = number
  default     = 30080
  description = "NodePort the demo Service is published on"
 }

-variable "replicas" {
-  type        = number
-  default     = 2
-  description = "Demo app replicas; refresh the page to see requests land on different pods"
-}
-
 variable "browse_cidr" {
  type        = string
  default     = "0.0.0.0/0"
@@ -95,6 +100,7 @@ provider "aws" {
    iam = var.spinifex_endpoint
    sts = var.spinifex_endpoint
    eks = var.spinifex_endpoint
+    ecr = var.spinifex_endpoint
  }

  skip_credentials_validation = true
@@ -103,19 +109,6 @@ provider "aws" {
  skip_region_validation      = true
 }

-# Authenticates to the cluster with the same `aws eks get-token` exec flow the
-# generated kubeconfig uses, so the Kubernetes provider can deploy the demo app.
-provider "kubernetes" {
-  host                   = aws_eks_cluster.this.endpoint
-  cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data)
-
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    command     = "aws"
-    args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name, "--region", var.region]
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Data sources
 # ---------------------------------------------------------------------------
@@ -245,6 +238,22 @@ resource "aws_iam_role_policy_attachment" "node_ecr" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
 }

+# ---------------------------------------------------------------------------
+# ECR — repository the workers pull the demo image from
+#
+# Build and push the demo-app image here before applying the workloads module
+# (see ../demo-app/README.md). The node role already carries
+# AmazonEC2ContainerRegistryReadOnly, so workers can pull from it.
+# ---------------------------------------------------------------------------
+
+resource "aws_ecr_repository" "demo" {
+  name = "spinifex-demo"
+
+  # tofu destroy must remove the repo even though it still holds the pushed demo
+  # image; without this, DeleteRepository (force=false) fails RepositoryNotEmpty.
+  force_delete = true
+}
+
 # ---------------------------------------------------------------------------
 # EKS cluster — public API endpoint, API (access-entry) auth mode
 # ---------------------------------------------------------------------------
@@ -283,9 +292,9 @@ resource "aws_eks_node_group" "default" {
  subnet_ids      = [aws_subnet.a.id, aws_subnet.b.id]

  scaling_config {
-    desired_size = 1
-    min_size     = 1
-    max_size     = 2
+    desired_size = var.node_desired_size
+    min_size     = var.node_desired_size
+    max_size     = var.node_desired_size * 2
  }

  instance_types = [var.instance_type]
@@ -336,68 +345,6 @@ resource "aws_vpc_security_group_ingress_rule" "nodeport" {
  }
 }

-# ---------------------------------------------------------------------------
-# Demo workload — deployed onto the cluster by Terraform
-#
-# nginxdemos/hello renders a page showing which pod served the request; with
-# multiple replicas, refreshing the demo_url alternates between them.
-# ---------------------------------------------------------------------------
-
-resource "kubernetes_deployment_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-    labels    = { app = "hello" }
-  }
-
-  spec {
-    replicas = var.replicas
-
-    selector {
-      match_labels = { app = "hello" }
-    }
-
-    template {
-      metadata {
-        labels = { app = "hello" }
-      }
-
-      spec {
-        container {
-          name  = "hello"
-          image = "nginxdemos/hello"
-
-          port {
-            container_port = 80
-          }
-        }
-      }
-    }
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
-resource "kubernetes_service_v1" "hello" {
-  metadata {
-    name      = "hello"
-    namespace = "default"
-  }
-
-  spec {
-    selector = { app = "hello" }
-    type     = "NodePort"
-
-    port {
-      port        = 80
-      target_port = 80
-      node_port   = var.node_port
-    }
-  }
-
-  depends_on = [kubernetes_deployment_v1.hello]
-}
-
 # ---------------------------------------------------------------------------
 # Discover the worker's public IP for the demo URL
 # ---------------------------------------------------------------------------
@@ -418,6 +365,23 @@ output "cluster_name" {
  value = aws_eks_cluster.this.name
 }

+output "region" {
+  value = var.region
+}
+
+output "node_port" {
+  value = var.node_port
+}
+
+output "node_desired_size" {
+  value = var.node_desired_size
+}
+
+output "ecr_repository_url" {
+  value       = aws_ecr_repository.demo.repository_url
+  description = "Push the demo-app image here, then apply the workloads module"
+}
+
 output "demo_url" {
  value       = "http://${data.aws_instances.workers.public_ips[0]}:${var.node_port}"
  description = "Open in a browser; refresh to see different pods answer"
@@ -0,0 +1,198 @@
+# Demo workload for eks-quickstart — separate root module / state.
+#
+# Kept apart from the cluster root module on purpose: the kubernetes provider
+# below reads the cluster endpoint from a *live* data source, so it is only ever
+# configured while the cluster exists. Destroy this module before the parent and
+# the provider never falls back to localhost.
+#
+# Usage:
+#   cd spinifex/docs/terraform-workbooks/eks-quickstart
+#   tofu init && tofu apply                    # parent: cluster + infra
+#   cd workloads && tofu init && tofu apply    # this module: demo app
+#   # teardown is the reverse — destroy here first, then the parent.
+
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.40, < 6.0"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.20"
+    }
+  }
+}
+
+variable "spinifex_endpoint" {
+  type        = string
+  default     = "https://127.0.0.1:9999"
+  description = "Spinifex AWS gateway endpoint"
+}
+
+variable "replicas" {
+  type        = number
+  default     = 2
+  description = "Demo app replicas; refresh the page to see requests land on different pods"
+}
+
+variable "demo_image" {
+  type        = string
+  default     = ""
+  description = "Demo image ref. Defaults to the parent's ECR repository URL at :latest."
+}
+
+provider "aws" {
+  region = data.terraform_remote_state.infra.outputs.region
+
+  endpoints {
+    ec2 = var.spinifex_endpoint
+    iam = var.spinifex_endpoint
+    sts = var.spinifex_endpoint
+    eks = var.spinifex_endpoint
+  }
+
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+  skip_region_validation      = true
+}
+
+# Cluster identity comes from the parent module's state; the live endpoint/CA
+# come from a data source so the provider is only configured while the cluster
+# is up.
+data "terraform_remote_state" "infra" {
+  backend = "local"
+
+  config = {
+    path = "../terraform.tfstate"
+  }
+}
+
+locals {
+  cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
+  region       = data.terraform_remote_state.infra.outputs.region
+  node_port    = data.terraform_remote_state.infra.outputs.node_port
+  demo_image   = var.demo_image != "" ? var.demo_image : "${data.terraform_remote_state.infra.outputs.ecr_repository_url}:latest"
+}
+
+data "aws_eks_cluster" "this" {
+  name = local.cluster_name
+}
+
+# Authenticates with the same `aws eks get-token` exec flow the generated
+# kubeconfig uses, so the Kubernetes provider can deploy the demo app.
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.this.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data)
+
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", local.cluster_name, "--region", local.region]
+  }
+}
+
+# The Spinifex-themed demo app reports the pod, node, cluster, and region that
+# served the request. With multiple replicas, refreshing the demo_url alternates
+# between them. The pod and node names come from the downward API.
+resource "kubernetes_deployment_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+    labels    = { app = "spinifex-demo" }
+  }
+
+  spec {
+    replicas = var.replicas
+
+    selector {
+      match_labels = { app = "spinifex-demo" }
+    }
+
+    template {
+      metadata {
+        labels = { app = "spinifex-demo" }
+      }
+
+      spec {
+        container {
+          name  = "spinifex-demo"
+          image = local.demo_image
+
+          port {
+            container_port = 8080
+          }
+
+          env {
+            name = "POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+
+          env {
+            name = "NODE_NAME"
+            value_from {
+              field_ref {
+                field_path = "spec.nodeName"
+              }
+            }
+          }
+
+          env {
+            name = "POD_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+
+          env {
+            name  = "CLUSTER_NAME"
+            value = local.cluster_name
+          }
+
+          env {
+            name  = "AWS_REGION"
+            value = local.region
+          }
+
+          readiness_probe {
+            http_get {
+              path = "/healthz"
+              port = 8080
+            }
+            initial_delay_seconds = 3
+            period_seconds        = 10
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service_v1" "demo" {
+  metadata {
+    name      = "spinifex-demo"
+    namespace = "default"
+  }
+
+  spec {
+    selector = { app = "spinifex-demo" }
+    type     = "NodePort"
+
+    port {
+      port        = 80
+      target_port = 8080
+      node_port   = local.node_port
+    }
+  }
+
+  depends_on = [kubernetes_deployment_v1.demo]
+}
@@ -13,12 +13,12 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.8.0
-	github.com/mulgadc/viperblock v1.8.0
+	github.com/mulgadc/predastore v1.9.0
+	github.com/mulgadc/viperblock v1.9.0
 	github.com/nats-io/nats-server/v2 v2.14.2
 	github.com/nats-io/nats.go v1.52.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.3.1
+	github.com/pelletier/go-toml/v2 v2.4.0
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
@@ -85,7 +85,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.6 // indirect
-	github.com/klauspost/reedsolomon v1.14.0 // indirect
+	github.com/klauspost/reedsolomon v1.14.1 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
@@ -51,8 +51,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUG
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvIGc1XnX076H5ZXLE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
 github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8=
 github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
@@ -238,8 +238,8 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
-github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
+github.com/klauspost/reedsolomon v1.14.1 h1:swE9kzyWXD/wVG+l5Pe8bWnQ0giIY7D1GjCBKk3kG2U=
+github.com/klauspost/reedsolomon v1.14.1/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -288,10 +288,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.8.0 h1:Di/PlAP8nIVjXMmjVPuS9UtqQkqMjR41RrC5J21vHmw=
-github.com/mulgadc/predastore v1.8.0/go.mod h1:TC6I4Z27RCe2gPqlxqUehJ2J+/jKggnH+4DyMq1H/ng=
-github.com/mulgadc/viperblock v1.8.0 h1:kVyM5RxomokVa1uA2kbhtS02kZ9aDXKVj4MvGeqsKiM=
-github.com/mulgadc/viperblock v1.8.0/go.mod h1:18zxN4szsp5h6w8nTqkZWP0SWmIrLU3NdpKXDYNanls=
+github.com/mulgadc/predastore v1.9.0 h1:nXwyblTso9ICbDkzSVx7j2j1FpCgPe/4c1du5Ge0k18=
+github.com/mulgadc/predastore v1.9.0/go.mod h1:FBFtu96XZlslhIJ2+Tv9NdR1S3Ks5XrWvJ+Q59iBxVA=
+github.com/mulgadc/viperblock v1.9.0 h1:rogwhMefiHIo7T8ApDYMQEs6UNGFNHtH8Bj2W7RzXU8=
+github.com/mulgadc/viperblock v1.9.0/go.mod h1:tq3msiqnmURyqEFd0oeMrVv3hpIgefXLurEn9exz+rI=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -310,8 +310,8 @@ github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
-github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.4.0 h1:Mwu0mAkUKbittDs3/ADDWXqMmq3EOK2VHiuCkV00Row=
+github.com/pelletier/go-toml/v2 v2.4.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -0,0 +1,443 @@
+# aws-ebs-csi-driver 1.40.1 — RBAC, ServiceAccounts, CSIDriver, PDB.
+# Vendored from kubernetes-sigs/aws-ebs-csi-driver v1.41.0 deploy/kubernetes/base;
+# namespace kube-system + controller-SA IRSA role-arn annotation injected.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-controller-sa
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+  annotations:
+    eks.amazonaws.com/role-arn: '{{SERVICE_ACCOUNT_ROLE_ARN}}'
+automountServiceAccountToken: true
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-node-sa
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+automountServiceAccountToken: true
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-attacher-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - nodes
+  verbs:
+  - get
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-provisioner-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattributesclasses
+  verbs:
+  - get
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-resizer-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - patch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattributesclasses
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-snapshotter-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - groupsnapshot.storage.k8s.io
+  resources:
+  - volumegroupsnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - groupsnapshot.storage.k8s.io
+  resources:
+  - volumegroupsnapshotcontents
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - groupsnapshot.storage.k8s.io
+  resources:
+  - volumegroupsnapshotcontents/status
+  verbs:
+  - update
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-attacher-binding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-getter-binding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-node-sa
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-csi-node-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-provisioner-binding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-resizer-binding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshotter-binding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-leases-role
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-leases-rolebinding
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: ebs-csi-leases-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: ebs.csi.aws.com
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+  fsGroupPolicy: File
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: ebs-csi-controller
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  maxUnavailable: 1
@@ -0,0 +1,276 @@
+# aws-ebs-csi-driver 1.40.1 — controller Deployment + gateway-CA Secret.
+# AWS credentials filled by mulga-eks-addon-sync via the IRSA env/volume/mount
+# markers below: web-identity (projected SA token aud sts.amazonaws.com + role
+# ARN + STS endpoint + CA) when a service account role ARN is staged, otherwise
+# node instance-profile creds over IMDS. AWS_EC2_ENDPOINT / AWS_ENDPOINT_URL_EC2
+# point EBS calls at the gateway in both modes. Static aws-secret key auth removed.
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: spinifex-gateway-ca
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+type: Opaque
+data:
+  ca.pem: '{{GATEWAY_CA_PEM_B64}}'
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-controller
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+spec:
+  replicas: 2
+  revisionHistoryLimit: 10
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/name: aws-ebs-csi-driver
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: ebs-csi-controller-sa
+      priorityClassName: system-cluster-critical
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/compute-type
+                operator: NotIn
+                values:
+                - fargate
+                - auto
+                - hybrid
+            weight: 1
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - ebs-csi-controller
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      securityContext:
+        fsGroup: 1000
+        runAsGroup: 1000
+        runAsNonRoot: true
+        runAsUser: 1000
+      containers:
+      - name: ebs-plugin
+        image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.40.1
+        imagePullPolicy: IfNotPresent
+        args:
+        - controller
+        - --endpoint=$(CSI_ENDPOINT)
+        - --batching=true
+        - --logging-format=text
+        - --user-agent-extra=kustomize
+        - --v=2
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: AWS_EC2_ENDPOINT
+          value: '{{GATEWAY_ENDPOINT}}'
+        - name: AWS_ENDPOINT_URL_EC2
+          value: '{{GATEWAY_ENDPOINT}}'
+{{IRSA_ENV}}
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /var/lib/csi/sockets/pluginproxy/
+{{IRSA_VOLUME_MOUNT}}
+        ports:
+        - name: healthz
+          containerPort: 9808
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 3
+          periodSeconds: 10
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 3
+          periodSeconds: 10
+          failureThreshold: 5
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
+      - name: csi-provisioner
+        image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v5.2.0-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --timeout=60s
+        - --csi-address=$(ADDRESS)
+        - --v=2
+        - --feature-gates=Topology=true
+        - --extra-create-metadata
+        - --leader-election=true
+        - --default-fstype=ext4
+        - --kube-api-qps=20
+        - --kube-api-burst=100
+        - --worker-threads=100
+        - --retry-interval-max=30m
+        - --feature-gates=VolumeAttributesClass=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /var/lib/csi/sockets/pluginproxy/
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
+      - name: csi-attacher
+        image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.8.1-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --timeout=60s
+        - --csi-address=$(ADDRESS)
+        - --v=2
+        - --leader-election=true
+        - --kube-api-qps=20
+        - --kube-api-burst=100
+        - --worker-threads=100
+        - --retry-interval-max=5m
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /var/lib/csi/sockets/pluginproxy/
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
+      - name: csi-snapshotter
+        image: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v8.2.1-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --csi-address=$(ADDRESS)
+        - --leader-election=true
+        - --v=2
+        - --extra-create-metadata
+        - --kube-api-qps=20
+        - --kube-api-burst=100
+        - --worker-threads=100
+        - --retry-interval-max=30m
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /var/lib/csi/sockets/pluginproxy/
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
+      - name: csi-resizer
+        image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.13.2-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --timeout=60s
+        - --extra-modify-metadata
+        - --csi-address=$(ADDRESS)
+        - --v=2
+        - --handle-volume-inuse-error=false
+        - --leader-election=true
+        - --kube-api-qps=20
+        - --kube-api-burst=100
+        - --workers=100
+        - --retry-interval-max=30m
+        - --feature-gates=VolumeAttributesClass=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /var/lib/csi/sockets/pluginproxy/
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
+      - name: liveness-probe
+        image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.14.0-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --csi-address=/csi/csi.sock
+        volumeMounts:
+        - name: socket-dir
+          mountPath: /csi
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+      volumes:
+      - name: socket-dir
+        emptyDir: {}
+{{IRSA_VOLUME}}
@@ -0,0 +1,182 @@
+# aws-ebs-csi-driver 1.40.1 — node DaemonSet. Mounts host /dev so the plugin
+# resolves /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_<volid> (minted by
+# mulga-ebs-byid). No AWS creds: the node plugin does not call EC2.
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+  namespace: kube-system
+spec:
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 10%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/name: aws-ebs-csi-driver
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: eks.amazonaws.com/compute-type
+                operator: NotIn
+                values:
+                - fargate
+                - auto
+                - hybrid
+              - key: node.kubernetes.io/instance-type
+                operator: NotIn
+                values:
+                - a1.medium
+                - a1.large
+                - a1.xlarge
+                - a1.2xlarge
+                - a1.4xlarge
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: ebs-csi-node-sa
+      terminationGracePeriodSeconds: 30
+      priorityClassName: system-node-critical
+      hostNetwork: false
+      securityContext:
+        fsGroup: 0
+        runAsGroup: 0
+        runAsNonRoot: false
+        runAsUser: 0
+      containers:
+      - name: ebs-plugin
+        image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.40.1
+        imagePullPolicy: IfNotPresent
+        args:
+        - node
+        - --endpoint=$(CSI_ENDPOINT)
+        - --csi-mount-point-prefix=/var/lib/kubelet/plugins/kubernetes.io/csi/ebs.csi.aws.com/
+        - --logging-format=text
+        - --v=2
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:/csi/csi.sock
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: kubelet-dir
+          mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+        - name: plugin-dir
+          mountPath: /csi
+        - name: device-dir
+          mountPath: /dev
+        ports:
+        - name: healthz
+          containerPort: 9808
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 3
+          periodSeconds: 10
+          failureThreshold: 5
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          privileged: true
+          readOnlyRootFilesystem: true
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/aws-ebs-csi-driver
+              - pre-stop-hook
+      - name: node-driver-registrar
+        image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.13.0-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --csi-address=$(ADDRESS)
+        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - --v=2
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+        livenessProbe:
+          exec:
+            command:
+            - /csi-node-driver-registrar
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --mode=kubelet-registration-probe
+          initialDelaySeconds: 30
+          periodSeconds: 90
+          timeoutSeconds: 15
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        - name: registration-dir
+          mountPath: /registration
+        - name: probe-dir
+          mountPath: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+      - name: liveness-probe
+        image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.14.0-eks-1-32-7
+        imagePullPolicy: IfNotPresent
+        args:
+        - --csi-address=/csi/csi.sock
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        resources:
+          limits:
+            memory: 256Mi
+          requests:
+            cpu: 10m
+            memory: 40Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+      volumes:
+      - name: kubelet-dir
+        hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+      - name: plugin-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+          type: DirectoryOrCreate
+      - name: registration-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+      - name: device-dir
+        hostPath:
+          path: /dev
+          type: Directory
+      - name: probe-dir
+        emptyDir: {}
@@ -0,0 +1,18 @@
+# Default gp3 StorageClass — PVCs without an explicit class provision a
+# Viperblock-backed EBS volume via the Spinifex EBS API. WaitForFirstConsumer
+# defers CreateVolume until a pod schedules so the volume lands in the node's AZ.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ebs-gp3
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  labels:
+    app.kubernetes.io/name: aws-ebs-csi-driver
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: gp3
+  csi.storage.k8s.io/fstype: ext4
@@ -0,0 +1,688 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: ingressclassparams.elbv2.k8s.aws
+spec:
+  group: elbv2.k8s.aws
+  names:
+    kind: IngressClassParams
+    listKind: IngressClassParamsList
+    plural: ingressclassparams
+    singular: ingressclassparams
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: The Ingress Group name
+      jsonPath: .spec.group.name
+      name: GROUP-NAME
+      type: string
+    - description: The AWS Load Balancer scheme
+      jsonPath: .spec.scheme
+      name: SCHEME
+      type: string
+    - description: The AWS Load Balancer ipAddressType
+      jsonPath: .spec.ipAddressType
+      name: IP-ADDRESS-TYPE
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: AGE
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: IngressClassParams is the Schema for the IngressClassParams API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: IngressClassParamsSpec defines the desired state of IngressClassParams
+            properties:
+              certificateArn:
+                description: CertificateArn specifies the ARN of the certificates
+                  for all Ingresses that belong to IngressClass with this IngressClassParams.
+                items:
+                  type: string
+                type: array
+              group:
+                description: Group defines the IngressGroup for all Ingresses that
+                  belong to IngressClass with this IngressClassParams.
+                properties:
+                  name:
+                    description: Name is the name of IngressGroup.
+                    type: string
+                required:
+                - name
+                type: object
+              inboundCIDRs:
+                description: InboundCIDRs specifies the CIDRs that are allowed to
+                  access the Ingresses that belong to IngressClass with this IngressClassParams.
+                items:
+                  type: string
+                type: array
+              ipAddressType:
+                description: IPAddressType defines the ip address type for all Ingresses
+                  that belong to IngressClass with this IngressClassParams.
+                enum:
+                - ipv4
+                - dualstack
+                - dualstack-without-public-ipv4
+                type: string
+              listeners:
+                description: Listeners define a list of listeners with their protocol,
+                  port and attributes.
+                items:
+                  properties:
+                    listenerAttributes:
+                      description: The attributes of the listener
+                      items:
+                        description: Attributes defines custom attributes on resources.
+                        properties:
+                          key:
+                            description: The key of the attribute.
+                            type: string
+                          value:
+                            description: The value of the attribute.
+                            type: string
+                        required:
+                        - key
+                        - value
+                        type: object
+                      type: array
+                    port:
+                      description: The port of the listener
+                      format: int32
+                      type: integer
+                    protocol:
+                      description: The protocol of the listener
+                      type: string
+                  type: object
+                type: array
+              loadBalancerAttributes:
+                description: LoadBalancerAttributes define the custom attributes to
+                  LoadBalancers for all Ingress that that belong to IngressClass with
+                  this IngressClassParams.
+                items:
+                  description: Attributes defines custom attributes on resources.
+                  properties:
+                    key:
+                      description: The key of the attribute.
+                      type: string
+                    value:
+                      description: The value of the attribute.
+                      type: string
+                  required:
+                  - key
+                  - value
+                  type: object
+                type: array
+              minimumLoadBalancerCapacity:
+                description: MinimumLoadBalancerCapacity define the capacity reservation
+                  for LoadBalancers for all Ingress that belong to IngressClass with
+                  this IngressClassParams.
+                properties:
+                  capacityUnits:
+                    description: The Capacity Units Value.
+                    format: int32
+                    type: integer
+                required:
+                - capacityUnits
+                type: object
+              namespaceSelector:
+                description: |-
+                  NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams.
+                  * if absent or present but empty, it selects all namespaces.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              scheme:
+                description: Scheme defines the scheme for all Ingresses that belong
+                  to IngressClass with this IngressClassParams.
+                enum:
+                - internal
+                - internet-facing
+                type: string
+              sslPolicy:
+                description: SSLPolicy specifies the SSL Policy for all Ingresses
+                  that belong to IngressClass with this IngressClassParams.
+                type: string
+              subnets:
+                description: Subnets defines the subnets for all Ingresses that belong
+                  to IngressClass with this IngressClassParams.
+                properties:
+                  ids:
+                    description: IDs specify the resource IDs of subnets. Exactly
+                      one of this or `tags` must be specified.
+                    items:
+                      description: SubnetID specifies a subnet ID.
+                      pattern: subnet-[0-9a-f]+
+                      type: string
+                    minItems: 1
+                    type: array
+                  tags:
+                    additionalProperties:
+                      items:
+                        type: string
+                      type: array
+                    description: |-
+                      Tags specifies subnets in the load balancer's VPC where each
+                      tag specified in the map key contains one of the values in the corresponding
+                      value list.
+                      Exactly one of this or `ids` must be specified.
+                    type: object
+                type: object
+              tags:
+                description: Tags defines list of Tags on AWS resources provisioned
+                  for Ingresses that belong to IngressClass with this IngressClassParams.
+                items:
+                  description: Tag defines a AWS Tag on resources.
+                  properties:
+                    key:
+                      description: The key of the tag.
+                      type: string
+                    value:
+                      description: The value of the tag.
+                      type: string
+                  required:
+                  - key
+                  - value
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: targetgroupbindings.elbv2.k8s.aws
+spec:
+  group: elbv2.k8s.aws
+  names:
+    kind: TargetGroupBinding
+    listKind: TargetGroupBindingList
+    plural: targetgroupbindings
+    singular: targetgroupbinding
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The Kubernetes Service's name
+      jsonPath: .spec.serviceRef.name
+      name: SERVICE-NAME
+      type: string
+    - description: The Kubernetes Service's port
+      jsonPath: .spec.serviceRef.port
+      name: SERVICE-PORT
+      type: string
+    - description: The AWS TargetGroup's TargetType
+      jsonPath: .spec.targetType
+      name: TARGET-TYPE
+      type: string
+    - description: The AWS TargetGroup's Amazon Resource Name
+      jsonPath: .spec.targetGroupARN
+      name: ARN
+      priority: 1
+      type: string
+    - description: The AWS TargetGroup's Name
+      jsonPath: .spec.targetGroupName
+      name: NAME
+      priority: 2
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: AGE
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: TargetGroupBinding is the Schema for the TargetGroupBinding API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding
+            properties:
+              multiClusterTargetGroup:
+                description: MultiClusterTargetGroup Denotes if the TargetGroup is
+                  shared among multiple clusters
+                type: boolean
+              networking:
+                description: networking provides the networking setup for ELBV2 LoadBalancer
+                  to access targets in TargetGroup.
+                properties:
+                  ingress:
+                    description: List of ingress rules to allow ELBV2 LoadBalancer
+                      to access targets in TargetGroup.
+                    items:
+                      properties:
+                        from:
+                          description: |-
+                            List of peers which should be able to access the targets in TargetGroup.
+                            At least one NetworkingPeer should be specified.
+                          items:
+                            description: NetworkingPeer defines the source/destination
+                              peer for networking rules.
+                            properties:
+                              ipBlock:
+                                description: |-
+                                  IPBlock defines an IPBlock peer.
+                                  If specified, none of the other fields can be set.
+                                properties:
+                                  cidr:
+                                    description: |-
+                                      CIDR is the network CIDR.
+                                      Both IPV4 or IPV6 CIDR are accepted.
+                                    type: string
+                                required:
+                                - cidr
+                                type: object
+                              securityGroup:
+                                description: |-
+                                  SecurityGroup defines a SecurityGroup peer.
+                                  If specified, none of the other fields can be set.
+                                properties:
+                                  groupID:
+                                    description: GroupID is the EC2 SecurityGroupID.
+                                    type: string
+                                required:
+                                - groupID
+                                type: object
+                            type: object
+                          type: array
+                        ports:
+                          description: |-
+                            List of ports which should be made accessible on the targets in TargetGroup.
+                            If ports is empty or unspecified, it defaults to all ports with TCP.
+                          items:
+                            properties:
+                              port:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                description: |-
+                                  The port which traffic must match.
+                                  When NodePort endpoints(instance TargetType) is used, this must be a numerical port.
+                                  When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods.
+                                  if port is unspecified, it defaults to all ports.
+                                x-kubernetes-int-or-string: true
+                              protocol:
+                                description: |-
+                                  The protocol which traffic must match.
+                                  If protocol is unspecified, it defaults to TCP.
+                                enum:
+                                - TCP
+                                - UDP
+                                type: string
+                            type: object
+                          type: array
+                      required:
+                      - from
+                      - ports
+                      type: object
+                    type: array
+                type: object
+              serviceRef:
+                description: serviceRef is a reference to a Kubernetes Service and
+                  ServicePort.
+                properties:
+                  name:
+                    description: Name is the name of the Service.
+                    type: string
+                  port:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: Port is the port of the ServicePort.
+                    x-kubernetes-int-or-string: true
+                required:
+                - name
+                - port
+                type: object
+              targetGroupARN:
+                description: targetGroupARN is the Amazon Resource Name (ARN) for
+                  the TargetGroup.
+                type: string
+              targetGroupName:
+                description: targetGroupName is the Name of the TargetGroup.
+                type: string
+              targetType:
+                description: targetType is the TargetType of TargetGroup. If unspecified,
+                  it will be automatically inferred.
+                enum:
+                - instance
+                - ip
+                type: string
+            required:
+            - serviceRef
+            type: object
+          status:
+            description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding
+            properties:
+              observedGeneration:
+                description: The generation observed by the TargetGroupBinding controller.
+                format: int64
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The Kubernetes Service's name
+      jsonPath: .spec.serviceRef.name
+      name: SERVICE-NAME
+      type: string
+    - description: The Kubernetes Service's port
+      jsonPath: .spec.serviceRef.port
+      name: SERVICE-PORT
+      type: string
+    - description: The AWS TargetGroup's TargetType
+      jsonPath: .spec.targetType
+      name: TARGET-TYPE
+      type: string
+    - description: The AWS TargetGroup's Amazon Resource Name
+      jsonPath: .spec.targetGroupARN
+      name: ARN
+      priority: 1
+      type: string
+    - description: The AWS TargetGroup's Name
+      jsonPath: .spec.targetGroupName
+      name: NAME
+      priority: 2
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: AGE
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: TargetGroupBinding is the Schema for the TargetGroupBinding API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding
+            properties:
+              ipAddressType:
+                description: ipAddressType specifies whether the target group is of
+                  type IPv4 or IPv6. If unspecified, it will be automatically inferred.
+                enum:
+                - ipv4
+                - ipv6
+                type: string
+              multiClusterTargetGroup:
+                description: MultiClusterTargetGroup Denotes if the TargetGroup is
+                  shared among multiple clusters
+                type: boolean
+              networking:
+                description: networking defines the networking rules to allow ELBV2
+                  LoadBalancer to access targets in TargetGroup.
+                properties:
+                  ingress:
+                    description: List of ingress rules to allow ELBV2 LoadBalancer
+                      to access targets in TargetGroup.
+                    items:
+                      description: NetworkingIngressRule defines a particular set
+                        of traffic that is allowed to access TargetGroup's targets.
+                      properties:
+                        from:
+                          description: |-
+                            List of peers which should be able to access the targets in TargetGroup.
+                            At least one NetworkingPeer should be specified.
+                          items:
+                            description: NetworkingPeer defines the source/destination
+                              peer for networking rules.
+                            properties:
+                              ipBlock:
+                                description: |-
+                                  IPBlock defines an IPBlock peer.
+                                  If specified, none of the other fields can be set.
+                                properties:
+                                  cidr:
+                                    description: |-
+                                      CIDR is the network CIDR.
+                                      Both IPV4 or IPV6 CIDR are accepted.
+                                    type: string
+                                required:
+                                - cidr
+                                type: object
+                              securityGroup:
+                                description: |-
+                                  SecurityGroup defines a SecurityGroup peer.
+                                  If specified, none of the other fields can be set.
+                                properties:
+                                  groupID:
+                                    description: GroupID is the EC2 SecurityGroupID.
+                                    type: string
+                                required:
+                                - groupID
+                                type: object
+                            type: object
+                          type: array
+                        ports:
+                          description: |-
+                            List of ports which should be made accessible on the targets in TargetGroup.
+                            If ports is empty or unspecified, it defaults to all ports with TCP.
+                          items:
+                            description: NetworkingPort defines the port and protocol
+                              for networking rules.
+                            properties:
+                              port:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                description: |-
+                                  The port which traffic must match.
+                                  When NodePort endpoints(instance TargetType) is used, this must be a numerical port.
+                                  When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods.
+                                  if port is unspecified, it defaults to all ports.
+                                x-kubernetes-int-or-string: true
+                              protocol:
+                                description: |-
+                                  The protocol which traffic must match.
+                                  If protocol is unspecified, it defaults to TCP.
+                                enum:
+                                - TCP
+                                - UDP
+                                type: string
+                            type: object
+                          type: array
+                      required:
+                      - from
+                      - ports
+                      type: object
+                    type: array
+                type: object
+              nodeSelector:
+                description: node selector for instance type target groups to only
+                  register certain nodes
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              serviceRef:
+                description: serviceRef is a reference to a Kubernetes Service and
+                  ServicePort.
+                properties:
+                  name:
+                    description: Name is the name of the Service.
+                    type: string
+                  port:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: Port is the port of the ServicePort.
+                    x-kubernetes-int-or-string: true
+                required:
+                - name
+                - port
+                type: object
+              targetGroupARN:
+                description: targetGroupARN is the Amazon Resource Name (ARN) for
+                  the TargetGroup.
+                type: string
+              targetGroupName:
+                description: targetGroupName is the Name of the TargetGroup.
+                type: string
+              targetType:
+                description: targetType is the TargetType of TargetGroup. If unspecified,
+                  it will be automatically inferred.
+                enum:
+                - instance
+                - ip
+                type: string
+              vpcID:
+                description: VpcID is the VPC of the TargetGroup. If unspecified,
+                  it will be automatically inferred.
+                type: string
+            required:
+            - serviceRef
+            type: object
+          status:
+            description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding
+            properties:
+              observedGeneration:
+                description: The generation observed by the TargetGroupBinding controller.
+                format: int64
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
@@ -0,0 +1,552 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: aws-load-balancer-controller
+    app.kubernetes.io/version: "v2.11.0"
+  annotations:
+    eks.amazonaws.com/role-arn: {{SERVICE_ACCOUNT_ROLE_ARN}}
+automountServiceAccountToken: true
+---
+# spinifex: node-minted webhook serving cert (render_addon mints a self-signed
+# cert for the webhook service SANs; cert-manager is not present on the node).
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-load-balancer-webhook-tls
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+type: kubernetes.io/tls
+data:
+  ca.crt: "{{WEBHOOK_CA_B64}}"
+  tls.crt: "{{WEBHOOK_TLS_CRT_B64}}"
+  tls.key: "{{WEBHOOK_TLS_KEY_B64}}"
+---
+# spinifex: the AWS gateway CA, mounted as AWS_CA_BUNDLE so the web-identity
+# credential bootstrap + elbv2/ec2/acm calls trust the gateway TLS endpoint.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: spinifex-gateway-ca
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+type: Opaque
+data:
+  ca.pem: "{{GATEWAY_CA_PEM_B64}}"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-controller-leader-election-role
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - aws-load-balancer-controller-leader
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - aws-load-balancer-controller-leader
+  resources:
+  - leases
+  verbs:
+  - get
+  - update
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-controller-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - elbv2.k8s.aws
+  resources:
+  - ingressclassparams
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - elbv2.k8s.aws
+  resources:
+  - targetgroupbindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - elbv2.k8s.aws
+  resources:
+  - targetgroupbindings/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - patch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-controller-leader-election-rolebinding
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: aws-load-balancer-controller-leader-election-role
+subjects:
+- kind: ServiceAccount
+  name: aws-load-balancer-controller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-controller-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-load-balancer-controller-role
+subjects:
+- kind: ServiceAccount
+  name: aws-load-balancer-controller
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-webhook-service
+  namespace: kube-system
+spec:
+  ports:
+  - port: 443
+    targetPort: 9443
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: aws-load-balancer-controller
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/name: aws-load-balancer-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: controller
+        app.kubernetes.io/name: aws-load-balancer-controller
+    spec:
+      containers:
+      - args:
+        - --cluster-name={{CLUSTER_NAME}}
+        - --ingress-class=alb
+        - --feature-gates=ALBSingleSubnet=true
+        - --aws-region={{AWS_REGION}}
+        - --aws-vpc-id={{AWS_VPC_ID}}
+        - --aws-api-endpoints=EC2={{GATEWAY_ENDPOINT}},Elastic Load Balancing v2={{GATEWAY_ENDPOINT}},ACM={{GATEWAY_ENDPOINT}},STS={{GATEWAY_ENDPOINT}}
+        - --enable-shield=false
+        - --enable-waf=false
+        - --enable-wafv2=false
+        env:
+{{IRSA_ENV}}
+        image: public.ecr.aws/eks/aws-load-balancer-controller:v2.11.0
+        livenessProbe:
+          failureThreshold: 2
+          httpGet:
+            path: /healthz
+            port: 61779
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        name: controller
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 200m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+{{IRSA_VOLUME_MOUNT}}
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1337
+      serviceAccountName: aws-load-balancer-controller
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: aws-load-balancer-webhook-tls
+{{IRSA_VOLUME}}
+---
+apiVersion: v1
+kind: List
+metadata:
+  name: ingress-class
+items:
+- apiVersion: elbv2.k8s.aws/v1beta1
+  kind: IngressClassParams
+  metadata:
+    name: alb
+    labels:
+      app.kubernetes.io/name: aws-load-balancer-controller
+      app.kubernetes.io/version: "v2.11.0"
+{{ELB_INGRESS_PARAMS_SPEC}}
+- apiVersion: networking.k8s.io/v1
+  kind: IngressClass
+  metadata:
+    name: alb
+    labels:
+      app.kubernetes.io/name: aws-load-balancer-controller
+      app.kubernetes.io/version: "v2.11.0"
+  spec:
+    controller: ingress.k8s.aws/alb
+    parameters:
+      apiGroup: elbv2.k8s.aws
+      kind: IngressClassParams
+      name: alb
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-webhook
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /mutate-v1-service
+  failurePolicy: Fail
+  name: mservice.elbv2.k8s.aws
+  objectSelector:
+    matchExpressions:
+    - key: app.kubernetes.io/name
+      operator: NotIn
+      values:
+      - aws-load-balancer-controller
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - services
+  sideEffects: None
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /mutate-v1-pod
+  failurePolicy: Fail
+  name: mpod.elbv2.k8s.aws
+  namespaceSelector:
+    matchExpressions:
+    - key: elbv2.k8s.aws/pod-readiness-gate-inject
+      operator: In
+      values:
+      - enabled
+  objectSelector:
+    matchExpressions:
+    - key: app.kubernetes.io/name
+      operator: NotIn
+      values:
+      - aws-load-balancer-controller
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  sideEffects: None
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /mutate-elbv2-k8s-aws-v1beta1-targetgroupbinding
+  failurePolicy: Fail
+  name: mtargetgroupbinding.elbv2.k8s.aws
+  rules:
+  - apiGroups:
+    - elbv2.k8s.aws
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - targetgroupbindings
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: aws-load-balancer-controller
+  name: aws-load-balancer-webhook
+webhooks:
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /validate-elbv2-k8s-aws-v1beta1-ingressclassparams
+  failurePolicy: Fail
+  name: vingressclassparams.elbv2.k8s.aws
+  objectSelector:
+    matchExpressions:
+    - key: app.kubernetes.io/name
+      operator: NotIn
+      values:
+      - aws-load-balancer-controller
+  rules:
+  - apiGroups:
+    - elbv2.k8s.aws
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingressclassparams
+  sideEffects: None
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /validate-elbv2-k8s-aws-v1beta1-targetgroupbinding
+  failurePolicy: Fail
+  name: vtargetgroupbinding.elbv2.k8s.aws
+  rules:
+  - apiGroups:
+    - elbv2.k8s.aws
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - targetgroupbindings
+  sideEffects: None
+- admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    caBundle: "{{WEBHOOK_CA_B64}}"
+    service:
+      name: aws-load-balancer-webhook-service
+      namespace: kube-system
+      path: /validate-networking-v1-ingress
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: vingress.elbv2.k8s.aws
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  sideEffects: None
@@ -0,0 +1,6 @@
+# Admission-webhook serving-cert SANs for this addon. render_addon mints a
+# self-signed cert for these names (via eks-webhook-cert) and embeds it as the
+# webhook Secret + every caBundle. Presence of this file is what opts a bundle
+# into webhook cert-gen.
+WEBHOOK_CN=aws-load-balancer-webhook-service.kube-system.svc
+WEBHOOK_DNS=aws-load-balancer-webhook-service.kube-system.svc,aws-load-balancer-webhook-service.kube-system.svc.cluster.local
@@ -0,0 +1,232 @@
+# Mulga-owned CoreDNS for the eks-node AMI. Replaces k3s's bundled CoreDNS
+# (suppressed via a coredns.yaml.skip marker staged by k3s.initd) so DNS is
+# node-local and never depends on the cross-node pod overlay: a DaemonSet runs
+# CoreDNS on every node and kube-dns uses internalTrafficPolicy=Local, so each
+# node resolves through its own CoreDNS. In the managed-control-plane topology
+# the worker<->CP overlay cannot cross the VPC boundary, so a CP-resident
+# CoreDNS is unreachable from worker pods.
+#
+# Faithful to k3s v1.32.5+k3s1 (image, Corefile, RBAC, Service) except the two
+# locality changes above. Refresh image/Corefile when bumping K3S_VERSION in
+# setup.sh.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: system:coredns
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - pods
+      - namespaces
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+  - kind: ServiceAccount
+    name: coredns
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        hosts /etc/coredns/NodeHosts {
+          ttl 60
+          reload 15s
+          fallthrough
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+        import /etc/coredns/custom/*.override
+    }
+    import /etc/coredns/custom/*.server
+  NodeHosts: ""
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      # CriticalAddonsOnly (Exists, all effects) tolerates the CP's
+      # CriticalAddonsOnly=true:NoExecute taint so a CoreDNS pod also runs on
+      # the control-plane node; the control-plane/master tolerations cover any
+      # role taints. Required for full per-node coverage with internalTrafficPolicy=Local.
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      nodeSelector:
+        kubernetes.io/os: linux
+      containers:
+        - name: coredns
+          image: rancher/mirrored-coredns-coredns:1.12.1
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              memory: 170Mi
+            requests:
+              cpu: 100m
+              memory: 70Mi
+          args: ["-conf", "/etc/coredns/Corefile"]
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/coredns
+              readOnly: true
+            - name: custom-config-volume
+              mountPath: /etc/coredns/custom
+              readOnly: true
+          ports:
+            - containerPort: 53
+              name: dns
+              protocol: UDP
+            - containerPort: 53
+              name: dns-tcp
+              protocol: TCP
+            - containerPort: 9153
+              name: metrics
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
+              drop:
+                - all
+            readOnlyRootFilesystem: true
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8181
+              scheme: HTTP
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+              - key: Corefile
+                path: Corefile
+              - key: NodeHosts
+                path: NodeHosts
+        - name: custom-config-volume
+          configMap:
+            name: coredns-custom
+            optional: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: 10.43.0.10
+  # Local: kube-proxy routes each node's 10.43.0.10 queries only to that node's
+  # CoreDNS DaemonSet pod, so DNS never crosses the worker<->CP boundary.
+  internalTrafficPolicy: Local
+  ports:
+    - name: dns
+      port: 53
+      protocol: UDP
+      targetPort: 53
+    - name: dns-tcp
+      port: 53
+      protocol: TCP
+      targetPort: 53
+    - name: metrics
+      port: 9153
+      protocol: TCP
+      targetPort: 9153
@@ -35,4 +35,27 @@ start_pre() {
        eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env"
        return 1
    fi
+
+    # Stamp the node providerID as aws:///<az>/<instance-id> (EKS parity) so the
+    # AWS Load Balancer Controller can resolve nodes as EC2 instances for ip-mode
+    # target registration; k3s' default k3s://<name> providerID makes it reject
+    # the node. Instance-id + AZ come from IMDSv2 (token-gated GET). On any miss
+    # we leave the default providerID rather than block the agent from starting.
+    _imds=http://169.254.169.254/latest
+    _curl="curl -fsS --connect-timeout 3 --max-time 5"
+    _tok=$(${_curl} -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 120' \
+        "${_imds}/api/token" 2>/dev/null)
+    if [ -n "${_tok}" ]; then
+        _iid=$(${_curl} -H "X-aws-ec2-metadata-token: ${_tok}" \
+            "${_imds}/meta-data/instance-id" 2>/dev/null)
+        _az=$(${_curl} -H "X-aws-ec2-metadata-token: ${_tok}" \
+            "${_imds}/meta-data/placement/availability-zone" 2>/dev/null)
+        if [ -n "${_iid}" ] && [ -n "${_az}" ]; then
+            command_args="agent --kubelet-arg=provider-id=aws:///${_az}/${_iid}"
+        else
+            ewarn "IMDS instance-id/az empty; keeping default providerID"
+        fi
+    else
+        ewarn "IMDS token request failed; keeping default providerID"
+    fi
 }
@@ -34,20 +34,18 @@ start_pre() {
        fi
    fi

-    # Deferred built-in ingress: when this cluster wants traefik
-    # (EKS_DEFER_TRAEFIK=1) but the control plane has not yet stabilised (no
-    # traefik-enabled sentinel), stage a deploy-controller .skip marker so k3s
-    # writes traefik.yaml/traefik-crd.yaml but does NOT apply them. Staged here,
-    # pre-start, so the marker exists before the deploy controller first scans —
-    # racing it post-start could apply traefik (and crash etcd) anyway.
-    # mulga-eks-state-report removes the marker once the apiserver is healthy.
-    if [ -f /etc/spinifex-eks/first-boot.env ]; then
-        defer=$(. /etc/spinifex-eks/first-boot.env 2>/dev/null; echo "${EKS_DEFER_TRAEFIK:-0}")
-        if [ "${defer}" = "1" ] && [ ! -f /var/lib/spinifex-eks/traefik-enabled ]; then
-            mkdir -p /var/lib/rancher/k3s/server/manifests
-            : > /var/lib/rancher/k3s/server/manifests/traefik.yaml.skip
-            : > /var/lib/rancher/k3s/server/manifests/traefik-crd.yaml.skip
-        fi
+    # Node-local CoreDNS: suppress k3s's bundled CoreDNS (a single Deployment that
+    # tolerates the CP taint and strands itself on the control-plane node) and
+    # deploy the Mulga DaemonSet variant so every node resolves via its own
+    # CoreDNS. Worker pods cannot reach a CP-resident CoreDNS — the worker<->CP
+    # overlay does not cross the VPC boundary. Staged pre-start so the deploy
+    # controller never applies the bundled manifest. Guarded on the baked bundle:
+    # an older image without it keeps k3s's CoreDNS rather than losing DNS.
+    if [ -f /usr/share/spinifex-eks/coredns/coredns-mulga.yaml ]; then
+        mkdir -p /var/lib/rancher/k3s/server/manifests
+        : > /var/lib/rancher/k3s/server/manifests/coredns.yaml.skip
+        cp /usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
+            /var/lib/rancher/k3s/server/manifests/coredns-mulga.yaml
    fi

    # The apiserver is configured with --authentication-token-webhook-config-file
@@ -18,9 +18,9 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-
 # eks-gateway-publish relays the bootstrap envelopes + state reports to the AWS
 # gateway over SigV4 HTTPS (server role only). The k3s binary is downloaded +
 # SHA256-verified by setup.sh (network in chroot).
-BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch"
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider"

-INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch"
+INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"

 # All role services ship; none but the selector are enabled at bake time.
 # eks-node-role picks server vs agent at first boot and enables the rest.
@@ -36,13 +36,26 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
+scripts/images/eks-node/mulga-ebs-byid.initd:/etc/init.d/mulga-ebs-byid \
+scripts/images/eks-node/mulga-ebs-byid.sh:/usr/local/sbin/mulga-ebs-byid \
+scripts/images/eks-node/mulga-eks-provider-id.initd:/etc/init.d/mulga-eks-provider-id \
+scripts/images/eks-node/mulga-eks-provider-id.sh:/usr/local/sbin/mulga-eks-provider-id \
 scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \
+scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \
+scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml \
+scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml \
+scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/webhook.conf:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/webhook.conf \
+scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/00-rbac.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/00-rbac.yaml \
+scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml \
+scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml \
+scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml \
+scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
 scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"

 # Only crond (etcd snapshot) + the role selector at bake. The selector adds the
 # role-specific services (k3s/eks-token-webhook/k3s-first-boot/state-report or
 # k3s-agent) to the default runlevel on first boot.
-ENABLE_SERVICES="crond eks-node-role"
+ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id"

 SETUP_SCRIPT="scripts/images/eks-node/setup.sh"

@@ -0,0 +1,22 @@
+#!/sbin/openrc-run
+# mulga-ebs-byid — role-agnostic boot oneshot that enables the EBS by-id
+# symlink bridge for the aws-ebs-csi-driver node plugin. Points the kernel
+# hotplug helper at busybox mdev (Alpine has no eudev/netlink device manager),
+# so a hot-attached virtio-blk volume triggers /etc/mdev.conf, then coldplugs
+# any volume already present at boot. Runs on every node (server + agent).
+
+description="Mulga EBS /dev/disk/by-id symlink bridge (mdev hotplug)"
+
+depend() {
+    after dev
+    before k3s k3s-agent
+}
+
+start() {
+    ebegin "Enabling EBS by-id hotplug bridge"
+    if [ -w /proc/sys/kernel/hotplug ]; then
+        echo /sbin/mdev > /proc/sys/kernel/hotplug
+    fi
+    ACTION=coldplug /usr/local/sbin/mulga-ebs-byid
+    eend $?
+}
@@ -0,0 +1,62 @@
+#!/bin/sh
+# mulga-ebs-byid — mint the EBS-style /dev/disk/by-id symlink the upstream
+# aws-ebs-csi-driver node plugin resolves. Spinifex attaches Viperblock volumes
+# as virtio-blk with serial = volume-id (dashes stripped); the kernel exposes it
+# at /sys/block/<dev>/serial. Alpine uses busybox mdev (no eudev), so the
+# nvme-Amazon_Elastic_Block_Store_<serial> link is created here from that serial.
+#
+# The mdev rule routes every vd* event here (busybox mdev stops at the first
+# matching line, so this helper supersedes the stock vd* rule rather than
+# stacking with it). Stock by-id/by-path/block links are preserved by
+# delegating to /lib/mdev/persistent-storage first.
+#
+# Invoked by mdev (env MDEV=<dev>, ACTION add|remove) and at boot for coldplug.
+set -eu
+
+PERSISTENT_STORAGE=/lib/mdev/persistent-storage
+BYID_DIR=/dev/disk/by-id
+PREFIX=nvme-Amazon_Elastic_Block_Store_
+
+dev="${MDEV:-${1:-}}"
+[ -n "${dev}" ] || exit 0
+
+link_dev() {
+    _d=$1
+    _sf="/sys/block/${_d}/serial"
+    [ -r "${_sf}" ] || return 0
+    _serial=$(cat "${_sf}" 2>/dev/null) || return 0
+    case "${_serial}" in
+        vol*) mkdir -p "${BYID_DIR}"; ln -sf "../../${_d}" "${BYID_DIR}/${PREFIX}${_serial}" ;;
+    esac
+}
+
+unlink_dev() {
+    _d=$1
+    for _l in "${BYID_DIR}/${PREFIX}"*; do
+        [ -L "${_l}" ] || continue
+        [ "$(readlink "${_l}")" = "../../${_d}" ] && rm -f "${_l}"
+    done
+}
+
+delegate() {
+    [ -x "${PERSISTENT_STORAGE}" ] || return 0
+    MDEV="${dev}" "${PERSISTENT_STORAGE}" || true
+}
+
+case "${ACTION:-add}" in
+    remove)
+        unlink_dev "${dev}"
+        delegate
+        ;;
+    coldplug)
+        for _p in /sys/block/vd*; do
+            [ -e "${_p}" ] || continue
+            link_dev "$(basename "${_p}")"
+        done
+        ;;
+    *)
+        delegate
+        link_dev "${dev}"
+        ;;
+esac
+exit 0
@@ -26,6 +26,7 @@ set -a
 set +a

 : "${EKS_GATEWAY_URL:?}"
+: "${EKS_ADDON_GATEWAY_URL:?}"
 : "${EKS_ACCESS_KEY:?}"
 : "${EKS_SECRET_KEY:?}"
 : "${EKS_ACCOUNT_ID:?}"
@@ -40,6 +41,10 @@ export KUBECONFIG
 BAKED_DIR=/usr/share/spinifex-eks/addons
 DEPLOY_DIR=/var/lib/rancher/k3s/server/manifests
 RENDER_PREFIX=spinifex-addon-
+# Per-cluster admission-webhook serving certs are minted once and cached here so
+# the rendered manifest is byte-stable across sync ticks (a churning cert would
+# re-apply the addon every tick).
+WEBHOOK_CERT_DIR=/var/lib/spinifex-eks/webhook-certs

 # log mirrors to syslog (on-VM /var/log) and, best-effort, to the serial
 # console — the host captures ttyS0 to a per-instance log, so this is the only
@@ -59,8 +64,10 @@ report() {
 }

 # render_addon ADDON VERSION ROLE_ARN — render the baked bundle into the
-# auto-deploy dir, substituting the IRSA role ARN. Returns non-zero (and reports
-# failed) when the baked bundle for ADDON/VERSION is absent.
+# auto-deploy dir, substituting the IRSA role ARN plus the cluster/gateway
+# placeholders an in-cluster AWS-SDK addon (e.g. the LB controller) needs to
+# reach the gateway. Returns non-zero (and reports failed) when the baked bundle
+# for ADDON/VERSION is absent. Placeholders not present in a bundle are no-ops.
 render_addon() {
    _addon=$1; _version=$2; _role=$3
    _src="${BAKED_DIR}/${_addon}/${_version}"
@@ -72,11 +79,156 @@ render_addon() {
        return 1
    fi

+    # Gateway CA as single-line base64 for a kube Secret .data entry the addon
+    # mounts as AWS_CA_BUNDLE. tr -d so busybox base64 (no -w0) stays one line.
+    _ca_b64=""
+    if [ -n "${EKS_GATEWAY_CA:-}" ] && [ -f "${EKS_GATEWAY_CA}" ]; then
+        _ca_b64=$(base64 < "${EKS_GATEWAY_CA}" | tr -d '\n')
+    fi
+
+    # Webhook serving cert (opt-in): a bundle ships webhook.conf with its service
+    # SANs when it runs an admission webhook. Mint/reuse a self-signed cert on the
+    # node (the AMI has no openssl) and embed it as the Secret + every caBundle.
+    _wh_ca=""; _wh_crt=""; _wh_key=""
+    if [ -f "${_src}/webhook.conf" ]; then
+        WEBHOOK_CN=""; WEBHOOK_DNS=""
+        # shellcheck disable=SC1090,SC1091
+        . "${_src}/webhook.conf"
+        if [ -n "${WEBHOOK_CN}" ] && [ -n "${WEBHOOK_DNS}" ]; then
+            _wh_err="${_dst}.whcert.$$"
+            if ! _wh_out=$(eks-webhook-cert -dir "${WEBHOOK_CERT_DIR}/${_addon}" \
+                    -cn "${WEBHOOK_CN}" -dns "${WEBHOOK_DNS}" 2>"${_wh_err}"); then
+                log "addon ${_addon}: webhook cert gen failed: $(tr '\n' ' ' < "${_wh_err}")"
+                rm -f "${_wh_err}"
+                report failed "${_addon}" "${_version}" "webhook cert generation failed"
+                return 1
+            fi
+            rm -f "${_wh_err}"
+            _wh_ca=$(printf '%s' "${_wh_out}" | cut -f1)
+            _wh_crt=$(printf '%s' "${_wh_out}" | cut -f2)
+            _wh_key=$(printf '%s' "${_wh_out}" | cut -f3)
+        fi
+    fi
+
+    # Reusable AWS-credential injection. A bundle opts in by placing the
+    # {{IRSA_ENV}} / {{IRSA_VOLUME}} / {{IRSA_VOLUME_MOUNT}} markers on their own
+    # line; awk swaps each for the standard block, then the sed pass fills the
+    # nested role/region/gateway placeholders. Bundles without the markers are
+    # untouched. Blocks assume a Deployment pod spec (env 8sp, volumes 6sp,
+    # volumeMounts 8sp).
+    #
+    # Gateway-CA trust is injected in both modes: AWS_CA_BUNDLE covers the
+    # standard-config clients, and SSL_CERT_FILE adds the gateway CA to the Go
+    # system trust pool. The LBC ec2/elbv2/acm clients install their own
+    # per-client BaseEndpoint resolver and build a fresh HTTP client that
+    # ignores AWS_CA_BUNDLE, falling back to the system pool — so without
+    # SSL_CERT_FILE they reject the gateway cert as "unknown authority".
+    #
+    # IRSA (web-identity) env is injected ONLY when a service-account role ARN is
+    # staged. The token (aud sts.amazonaws.com) + role ARN + gateway STS endpoint
+    # are what AssumeRoleWithWebIdentity needs. Without a role ARN, those are
+    # omitted entirely: injecting an empty AWS_ROLE_ARN beside the token file
+    # half-arms the SDK web-identity provider, which then signs anonymously
+    # instead of falling through the default chain to the node instance-profile
+    # credentials served over IMDS. Omitting them restores that node-credential
+    # fallback.
+    if [ -n "${_role}" ]; then
+        _irsa_env='        - name: AWS_ROLE_ARN
+          value: "{{SERVICE_ACCOUNT_ROLE_ARN}}"
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token
+        - name: AWS_STS_REGIONAL_ENDPOINTS
+          value: regional
+        - name: AWS_REGION
+          value: "{{AWS_REGION}}"
+        - name: AWS_DEFAULT_REGION
+          value: "{{AWS_REGION}}"
+        - name: AWS_ENDPOINT_URL_STS
+          value: "{{GATEWAY_ENDPOINT}}"
+        - name: AWS_CA_BUNDLE
+          value: /etc/spinifex/gateway-ca/ca.pem
+        - name: SSL_CERT_FILE
+          value: /etc/spinifex/gateway-ca/ca.pem'
+        _irsa_vol='      - name: aws-iam-token
+        projected:
+          defaultMode: 420
+          sources:
+          - serviceAccountToken:
+              audience: sts.amazonaws.com
+              expirationSeconds: 86400
+              path: token
+      - name: spinifex-gateway-ca
+        secret:
+          defaultMode: 420
+          secretName: spinifex-gateway-ca'
+        _irsa_mnt='        - name: aws-iam-token
+          mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
+          readOnly: true
+        - name: spinifex-gateway-ca
+          mountPath: /etc/spinifex/gateway-ca
+          readOnly: true'
+    else
+        _irsa_env='        - name: AWS_REGION
+          value: "{{AWS_REGION}}"
+        - name: AWS_DEFAULT_REGION
+          value: "{{AWS_REGION}}"
+        - name: AWS_CA_BUNDLE
+          value: /etc/spinifex/gateway-ca/ca.pem
+        - name: SSL_CERT_FILE
+          value: /etc/spinifex/gateway-ca/ca.pem'
+        _irsa_vol='      - name: spinifex-gateway-ca
+        secret:
+          defaultMode: 420
+          secretName: spinifex-gateway-ca'
+        _irsa_mnt='        - name: spinifex-gateway-ca
+          mountPath: /etc/spinifex/gateway-ca
+          readOnly: true'
+    fi
+
+    # IngressClassParams spec injection. EKS_ELB_SUBNET_IDS is the cluster's
+    # ELB-eligible subnets (CSV, deduped to one per AZ by the daemon). Inject them
+    # as explicit .spec.subnets.ids so every Ingress takes LBC's explicit-subnet
+    # path, the only one that honors ALBSingleSubnet; a single-AZ cluster otherwise
+    # fails reconcile. Empty value drops the marker line, leaving tag auto-discovery.
+    _icp_spec=''
+    if [ -n "${EKS_ELB_SUBNET_IDS:-}" ]; then
+        _icp_spec='  spec:
+    subnets:
+      ids:'
+        _oldifs=$IFS
+        IFS=','
+        for _sn in ${EKS_ELB_SUBNET_IDS}; do
+            [ -n "${_sn}" ] || continue
+            _icp_spec="${_icp_spec}
+      - ${_sn}"
+        done
+        IFS=$_oldifs
+    fi
+
    _tmp="${_dst}.tmp.$$"
    : > "${_tmp}"
    for _f in "${_src}"/*.yaml; do
        [ -e "${_f}" ] || continue
-        sed -e "s|{{SERVICE_ACCOUNT_ROLE_ARN}}|${_role}|g" "${_f}" >> "${_tmp}"
+        # Each block marker must be the ONLY token on its line (whitespace aside);
+        # matching a substring would also fire on a comment that merely names the
+        # marker, injecting the block mid-document and producing invalid YAML.
+        awk -v env="${_irsa_env}" -v vol="${_irsa_vol}" -v mnt="${_irsa_mnt}" -v icp="${_icp_spec}" '
+            $0 ~ /^[[:space:]]*[{][{]IRSA_ENV[}][}][[:space:]]*$/                { print env; next }
+            $0 ~ /^[[:space:]]*[{][{]IRSA_VOLUME_MOUNT[}][}][[:space:]]*$/       { print mnt; next }
+            $0 ~ /^[[:space:]]*[{][{]IRSA_VOLUME[}][}][[:space:]]*$/             { print vol; next }
+            $0 ~ /^[[:space:]]*[{][{]ELB_INGRESS_PARAMS_SPEC[}][}][[:space:]]*$/ { if (icp != "") print icp; next }
+            { print }
+        ' "${_f}" \
+        | sed -e "s|{{SERVICE_ACCOUNT_ROLE_ARN}}|${_role}|g" \
+            -e "s|{{CLUSTER_NAME}}|${EKS_CLUSTER_NAME}|g" \
+            -e "s|{{AWS_REGION}}|${EKS_REGION:-}|g" \
+            -e "s|{{AWS_VPC_ID}}|${EKS_VPC_ID:-}|g" \
+            -e "s|{{GATEWAY_ENDPOINT}}|${EKS_ADDON_GATEWAY_URL}|g" \
+            -e "s|{{GATEWAY_CA_PEM_B64}}|${_ca_b64}|g" \
+            -e "s|{{WEBHOOK_CA_B64}}|${_wh_ca}|g" \
+            -e "s|{{WEBHOOK_TLS_CRT_B64}}|${_wh_crt}|g" \
+            -e "s|{{WEBHOOK_TLS_KEY_B64}}|${_wh_key}|g" \
+            >> "${_tmp}"
        printf '\n---\n' >> "${_tmp}"
    done
    # Idempotent: only replace the auto-deploy file when the rendered content
@@ -0,0 +1,19 @@
+#!/sbin/openrc-run
+# mulga-eks-provider-id — boot oneshot that writes a K3s drop-in from IMDS
+# before K3s starts: the kubelet provider-id plus region/zone topology labels,
+# the node metadata the aws-ebs-csi-driver needs for NodeGetInfo + provisioning.
+# Ordered after local (which installs the IMDS on-link route) and before both
+# K3s roles. Role-agnostic: enabled on every node (server + agent).
+
+description="Set K3s provider-id + topology labels from IMDS for EBS CSI"
+
+depend() {
+    after net local
+    before k3s k3s-agent
+}
+
+start() {
+    ebegin "Writing K3s provider-id drop-in from IMDS"
+    /usr/local/sbin/mulga-eks-provider-id
+    eend $?
+}
@@ -0,0 +1,91 @@
+#!/bin/sh
+set -eu
+
+# mulga-eks-provider-id — writes a K3s config drop-in that supplies the node
+# metadata the aws-ebs-csi-driver needs, read from IMDS at boot: the kubelet
+# provider-id (aws:///<az>/<instance-id>) plus the region/zone topology labels
+# and the instance-type label.
+#
+# The node plugin's metadata init first tries IMDS, whose ENI path the spinifex
+# IMDS does not serve, then falls back to Kubernetes: instance ID from
+# .spec.providerID, region/zone from the well-known topology labels, and the
+# instance type from node.kubernetes.io/instance-type. K3s runs no AWS
+# cloud-provider, so none of those are set unless kubelet is told. Without the
+# topology labels NodeGetInfo never runs, the CSINode carries no topology key,
+# and the provisioner fails "no topology key found"; without a parseable
+# instance type NodeGetInfo panics in IsNitroInstanceType ("cannot determine
+# family"). This drop-in supplies all four; region is derived from the AZ
+# (trailing zone letters stripped).
+#
+# K3s merges /etc/rancher/k3s/config.yaml.d/*.yaml over config.yaml; list keys
+# such as kubelet-arg are concatenated, so this adds the flag without touching
+# the cloud-init-written main config. Role-agnostic: runs on server + agent.
+
+IMDS="http://169.254.169.254/latest"
+DROPIN_DIR=/etc/rancher/k3s/config.yaml.d
+DROPIN="${DROPIN_DIR}/10-provider-id.yaml"
+LOGTAG="mulga-eks-provider-id"
+
+log() { echo "[${LOGTAG}] $*"; }
+
+if [ -f "${DROPIN}" ]; then
+    log "drop-in ${DROPIN} present — already configured"
+    exit 0
+fi
+
+# IMDSv2 token if the service enforces it; tokenless (v1) is the fallback.
+fetch() {
+    if [ -n "${_tok}" ]; then
+        curl -fsS --max-time 3 -H "X-aws-ec2-metadata-token: ${_tok}" "${IMDS}/meta-data/$1" 2>/dev/null
+    else
+        curl -fsS --max-time 3 "${IMDS}/meta-data/$1" 2>/dev/null
+    fi
+}
+
+# Retry: the IMDS on-link route + networking may settle shortly after boot.
+iid=""
+az=""
+itype=""
+i=0
+while [ "${i}" -lt 30 ]; do
+    _tok=$(curl -fsS --max-time 3 -X PUT "${IMDS}/api/token" \
+        -H "X-aws-ec2-metadata-token-ttl-seconds: 300" 2>/dev/null || true)
+    iid=$(fetch instance-id || true)
+    az=$(fetch placement/availability-zone || true)
+    itype=$(fetch instance-type || true)
+    case "${iid}" in
+        i-*) [ -n "${az}" ] && break ;;
+    esac
+    iid=""
+    az=""
+    i=$((i + 1))
+    sleep 2
+done
+
+if [ -z "${iid}" ] || [ -z "${az}" ]; then
+    log "IMDS instance-id/availability-zone unavailable after retries; provider-id not set"
+    exit 0
+fi
+
+# Region = AZ with the trailing zone letter(s) removed (ap-southeast-2a ->
+# ap-southeast-2). The CSI metadata fallback reads it from the region label.
+region=$(printf '%s' "${az}" | sed 's/[a-z]*$//')
+
+# instance-type must parse as family.size or the node plugin panics in
+# IsNitroInstanceType. Fall back to a generic type only to keep it parseable;
+# the real spinifex type (e.g. sys.medium) already satisfies the format.
+case "${itype}" in
+    *.*) : ;;
+    *) itype="m5.large" ;;
+esac
+
+mkdir -p "${DROPIN_DIR}"
+cat > "${DROPIN}" <<EOF
+kubelet-arg:
+  - "provider-id=aws:///${az}/${iid}"
+node-label:
+  - "topology.kubernetes.io/region=${region}"
+  - "topology.kubernetes.io/zone=${az}"
+  - "node.kubernetes.io/instance-type=${itype}"
+EOF
+log "wrote provider-id aws:///${az}/${iid}, region=${region} zone=${az} type=${itype}"
@@ -35,36 +35,6 @@ set +a
 KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
 export KUBECONFIG

-# --- Deferred built-in ingress enable -------------------------------------
-# traefik is deferred at boot (its helm-install Job overloads the embedded etcd
-# during bootstrap and can crash the control plane). k3s writes traefik.yaml but
-# k3s.initd pre-stages a <name>.yaml.skip marker the deploy controller honours,
-# so traefik is not applied. Once the apiserver has been healthy for
-# TRAEFIK_ENABLE_AFTER consecutive reports — a crash resets the streak, so this
-# only fires post-stabilisation — remove the skip markers so the deploy
-# controller installs traefik with etcd headroom. Gated on EKS_DEFER_TRAEFIK=1
-# (set only for clusters that want built-in ingress) and run once via a sentinel.
-TRAEFIK_ENABLE_AFTER=${TRAEFIK_ENABLE_AFTER:-4}
-TRAEFIK_ENABLED_SENTINEL=/var/lib/spinifex-eks/traefik-enabled
-K3S_MANIFESTS=/var/lib/rancher/k3s/server/manifests
-healthy_streak=0
-
-maybe_enable_traefik() {
-    [ "${EKS_DEFER_TRAEFIK:-0}" = "1" ] || return 0
-    [ -f "${TRAEFIK_ENABLED_SENTINEL}" ] && return 0
-    if [ "${1}" = "ok" ]; then
-        healthy_streak=$((healthy_streak + 1))
-    else
-        healthy_streak=0
-        return 0
-    fi
-    [ "${healthy_streak}" -ge "${TRAEFIK_ENABLE_AFTER}" ] || return 0
-    rm -f "${K3S_MANIFESTS}/traefik.yaml.skip" "${K3S_MANIFESTS}/traefik-crd.yaml.skip"
-    : > "${TRAEFIK_ENABLED_SENTINEL}"
-    logger -t mulga-eks-state-report \
-        "enabled deferred built-in ingress (traefik) after ${healthy_streak} healthy reports"
-}
-
 publish_report() {
    if curl -sk --max-time 3 https://127.0.0.1:6443/healthz | grep -q '^ok$'; then
        health=ok
@@ -84,7 +54,6 @@ publish_report() {
 if [ -n "${STATE_REPORT_INTERVAL:-}" ] && [ "${STATE_REPORT_INTERVAL}" -gt 0 ] 2>/dev/null; then
    while true; do
        publish_report
-        maybe_enable_traefik "${health}"
        sleep "${STATE_REPORT_INTERVAL}"
    done
 else
@@ -47,8 +47,24 @@ chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
    /etc/init.d/mulga-eks-addon-sync
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
+chmod 0755 /etc/init.d/mulga-ebs-byid /usr/local/sbin/mulga-ebs-byid
+chmod 0755 /etc/init.d/mulga-eks-provider-id /usr/local/sbin/mulga-eks-provider-id
 chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot

+# EBS by-id bridge: route every virtio-blk event through mulga-ebs-byid, which
+# delegates to the stock persistent-storage helper and then mints the
+# nvme-Amazon_Elastic_Block_Store_<serial> link the EBS CSI node plugin resolves.
+# busybox mdev stops at the first matching rule, so the stock vd* persistent-
+# storage line is replaced in place — appending a second vd* rule would be
+# shadowed and never fire. The leading '*' runs the command on add and remove.
+sed -i \
+    's#^vd\[a-z\]\.\*[[:space:]].*persistent-storage#vd[a-z].*\troot:disk 0660 */usr/local/sbin/mulga-ebs-byid#' \
+    /etc/mdev.conf
+grep -q 'mulga-ebs-byid' /etc/mdev.conf || {
+    echo "[eks-node-setup] failed to wire mulga-ebs-byid into /etc/mdev.conf"
+    exit 1
+}
+
 # K3s server config — skeleton; cloud-init / first-boot fills in the
 # per-cluster fields (cluster-cidr, service-cidr, token-file, etc). Agents
 # ignore this file (they run `k3s agent`).
@@ -94,4 +110,12 @@ sed -i \
    's|console=ttyS0,115200n8 console=ttyAMA0,115200n8 console=tty0|console=tty0 console=ttyAMA0,115200n8 console=ttyS0,115200n8|' \
    /etc/update-extlinux.conf /boot/extlinux.conf

+# Cut the boot-menu countdown from 10s to ~1s. The stock cloud image waits 10s at
+# the SYSLINUX menu before auto-booting — a fixed, network-independent tax on every
+# VM start. Patch the generator config (timeout in seconds) and the rendered output
+# (TIMEOUT in 1/10s) so a regenerate keeps the short value. A small nonzero value is
+# kept so the menu stays interruptible over serial; TIMEOUT 0 would wait forever.
+sed -i 's/^timeout=.*/timeout=1/' /etc/update-extlinux.conf
+sed -i 's/^TIMEOUT[[:space:]].*/TIMEOUT 10/' /boot/extlinux.conf
+
 echo "[eks-node-setup] done"
@@ -221,6 +221,7 @@ install_apt_deps() {
            pciutils \
            jq curl iproute2 netcat-openbsd wget unzip xz-utils file \
            ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base \
+            chrony \
            > /dev/null

        info "System dependencies installed"
@@ -60,6 +60,10 @@ type ConfigSettings struct {
 	// Predastore multi-node
 	PredastoreNodeID int

+	// CompactionIntervalSeconds gates the predastore [compaction] block. Zero
+	// means unset: no block is emitted and predastore keeps its built-in default.
+	CompactionIntervalSeconds int
+
 	// Node capabilities
 	Services []string

@@ -155,7 +159,22 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings
 	return nil
 }

-func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) {
+// AWSGWServiceDNSNames builds the AWS-parity TLS SANs for the awsgw cert from
+// the cluster region and internal suffix: the exact ECR control-plane host and
+// the wildcard covering per-account registry hosts. Returns nil if either input
+// is empty so callers omit the SANs rather than emitting malformed names.
+func AWSGWServiceDNSNames(region, suffix string) []string {
+	if region == "" || suffix == "" {
+		return nil
+	}
+	base := "ecr." + region + "." + suffix
+	return []string{
+		base,            // control plane: ecr.{region}.{suffix}
+		"*.dkr." + base, // registry: *.dkr.ecr.{region}.{suffix}
+	}
+}
+
+func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, awsRegion, internalSuffix string) (caCertPath string) {
 	caCertPath = filepath.Join(configDir, "ca.pem")
 	caKeyPath := filepath.Join(configDir, "ca.key")
 	serverCertPath := filepath.Join(configDir, "server.pem")
@@ -177,7 +196,8 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (
 		fmt.Printf("   CA Certificate: %s\n", caCertPath)
 		fmt.Printf("   CA Key: %s\n", caKeyPath)

-		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil {
+		extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
+		if err := GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 			os.Exit(1)
 		}
@@ -200,8 +220,9 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (
 }

 // GenerateServerCertOnly generates a server certificate signed by an existing CA.
-// Used by joining nodes that receive the CA from the leader.
-func GenerateServerCertOnly(configDir string, bindIP string) error {
+// Used by joining nodes that receive the CA from the leader. awsRegion and
+// internalSuffix add the AWS-parity ECR SANs; empty values omit them.
+func GenerateServerCertOnly(configDir string, bindIP, awsRegion, internalSuffix string) error {
 	caCertPath := filepath.Join(configDir, "ca.pem")
 	caKeyPath := filepath.Join(configDir, "ca.key")
 	serverCertPath := filepath.Join(configDir, "server.pem")
@@ -211,7 +232,8 @@ func GenerateServerCertOnly(configDir string, bindIP string) error {
 		return fmt.Errorf("CA files not found in %s", configDir)
 	}

-	return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP)
+	extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
+	return GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS)
 }

 func CreateServiceDirectories(spxRoot string) {
@@ -860,20 +882,21 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
 // GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
 // multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
 // and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
-func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string) (string, error) {
+func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string, compactionIntervalSeconds int) (string, error) {
 	if len(nodes) < 2 {
 		return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes))
 	}

 	data := struct {
-		Nodes     []PredastoreNodeConfig
-		AccessKey string
-		SecretKey string
-		Region    string
-		NatsToken string
-		ConfigDir string
-		BindIP    string
-	}{nodes, accessKey, secretKey, region, natsToken, configDir, bindIP}
+		Nodes                     []PredastoreNodeConfig
+		AccessKey                 string
+		SecretKey                 string
+		Region                    string
+		NatsToken                 string
+		ConfigDir                 string
+		BindIP                    string
+		CompactionIntervalSeconds int
+	}{nodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionIntervalSeconds}

 	tmpl, err := template.New("predastore-multinode").Parse(templateStr)
 	if err != nil {
@@ -621,7 +621,7 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) {
 	dir := t.TempDir()

 	// First call generates all certs
-	caCertPath := GenerateCertificatesIfNeeded(dir, false, "")
+	caCertPath := GenerateCertificatesIfNeeded(dir, false, "", "us-east-1", "spinifex.internal")
 	assert.Equal(t, filepath.Join(dir, "ca.pem"), caCertPath)
 	assert.True(t, FileExists(filepath.Join(dir, "ca.pem")))
 	assert.True(t, FileExists(filepath.Join(dir, "ca.key")))
@@ -632,7 +632,7 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) {
 		caInfo, _ := os.Stat(filepath.Join(dir, "ca.pem"))
 		origModTime := caInfo.ModTime()

-		GenerateCertificatesIfNeeded(dir, false, "")
+		GenerateCertificatesIfNeeded(dir, false, "", "us-east-1", "spinifex.internal")
 		caInfo2, _ := os.Stat(filepath.Join(dir, "ca.pem"))
 		assert.Equal(t, origModTime, caInfo2.ModTime())
 	})
@@ -640,7 +640,7 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) {
 	t.Run("ForceRegenerates", func(t *testing.T) {
 		origCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))

-		GenerateCertificatesIfNeeded(dir, true, "")
+		GenerateCertificatesIfNeeded(dir, true, "", "us-east-1", "spinifex.internal")
 		newCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
 		assert.NotEqual(t, origCA, newCA)
 	})
@@ -660,7 +660,7 @@ func TestGenerateServerCertOnly(t *testing.T) {
 		require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), caCert, 0600))
 		require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.key"), caKey, 0600))

-		err := GenerateServerCertOnly(dir, "10.0.0.5")
+		err := GenerateServerCertOnly(dir, "10.0.0.5", "us-east-1", "spinifex.internal")
 		require.NoError(t, err)
 		assert.True(t, FileExists(filepath.Join(dir, "server.pem")))
 		assert.True(t, FileExists(filepath.Join(dir, "server.key")))
@@ -676,16 +676,34 @@ func TestGenerateServerCertOnly(t *testing.T) {
 			}
 		}
 		assert.True(t, hasBindIP)
+
+		// AWS-parity ECR SANs must be present.
+		assert.Contains(t, cert.DNSNames, "ecr.us-east-1.spinifex.internal")
+		assert.Contains(t, cert.DNSNames, "*.dkr.ecr.us-east-1.spinifex.internal")
 	})

 	t.Run("MissingCA", func(t *testing.T) {
 		dir := t.TempDir()
-		err := GenerateServerCertOnly(dir, "10.0.0.5")
+		err := GenerateServerCertOnly(dir, "10.0.0.5", "us-east-1", "spinifex.internal")
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "CA files not found")
 	})
 }

+func TestAWSGWServiceDNSNames(t *testing.T) {
+	t.Parallel()
+
+	got := AWSGWServiceDNSNames("us-east-1", "spinifex.internal")
+	assert.Equal(t, []string{
+		"ecr.us-east-1.spinifex.internal",
+		"*.dkr.ecr.us-east-1.spinifex.internal",
+	}, got)
+
+	assert.Nil(t, AWSGWServiceDNSNames("", "spinifex.internal"))
+	assert.Nil(t, AWSGWServiceDNSNames("us-east-1", ""))
+	assert.Nil(t, AWSGWServiceDNSNames("", ""))
+}
+
 // --- Directory creation ---

 func TestCreateServiceDirectories_CreatesAll(t *testing.T) {
@@ -727,7 +745,7 @@ host = "{{.Host}}"
 		{ID: 3, Host: "10.0.0.3"},
 	}

-	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
+	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1", 0)
 	require.NoError(t, err)
 	assert.Contains(t, result, `host = "10.0.0.1"`)
 	assert.Contains(t, result, `host = "10.0.0.3"`)
@@ -738,7 +756,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {

 	_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
 		{ID: 1, Host: "10.0.0.1"},
-	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
+	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1", 0)
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "at least 2 nodes")
 }
@@ -746,7 +764,7 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
 func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
 	_, err := GenerateMultiNodePredastoreConfig("{{.Unclosed", []PredastoreNodeConfig{
 		{ID: 1, Host: "a"}, {ID: 2, Host: "b"}, {ID: 3, Host: "c"},
-	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
+	}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1", 0)
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "failed to parse")
 }
@@ -453,6 +453,21 @@ var (
 	ErrorIAMMalformedPolicyDocument = "MalformedPolicyDocument"
 	ErrorAccessDenied               = "AccessDenied"

+	// ECR-specific error codes
+	ErrorRepositoryNotFound       = "RepositoryNotFoundException"
+	ErrorRepositoryPolicyNotFound = "RepositoryPolicyNotFoundException"
+	ErrorLifecyclePolicyNotFound  = "LifecyclePolicyNotFoundException"
+	ErrorRepositoryAlreadyExists  = "RepositoryAlreadyExistsException"
+	ErrorRepositoryNotEmpty       = "RepositoryNotEmptyException"
+	ErrorImageNotFound            = "ImageNotFoundException"
+	ErrorImageAlreadyExists       = "ImageAlreadyExistsException"
+	ErrorImageTagAlreadyExists    = "ImageTagAlreadyExistsException"
+	ErrorImageDigestDoesNotMatch  = "ImageDigestDoesNotMatchException"
+	ErrorLayersNotFound           = "LayersNotFoundException"
+	ErrorReferencedImagesNotFound = "ReferencedImagesNotFoundException"
+	ErrorTooManyTags              = "TooManyTagsException"
+	ErrorOperationNotSupported    = "OperationNotSupportedException"
+
 	// ELBv2-specific error codes
 	ErrorELBv2LoadBalancerNotFound         = "LoadBalancerNotFound"
 	ErrorELBv2TargetGroupNotFound          = "TargetGroupNotFound"
@@ -491,6 +506,14 @@ func ValidErrorCode(code string) string {
 	return ErrorServerInternal
 }

+// HasErrorCode reports whether s is exactly a registered AWS error code.
+// Use it to decide whether an error string may be surfaced to clients verbatim
+// (mapped to its real HTTP status) rather than wrapped/sanitized to 500.
+func HasErrorCode(s string) bool {
+	_, ok := ErrorLookup[s]
+	return ok
+}
+
 // IsErrorCode reports whether err carries the named AWS error code.
 // Uses substring matching because handlers return codes as plain strings, sometimes wrapped with %w.
 func IsErrorCode(err error, code string) bool {
@@ -949,6 +972,21 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorIAMMalformedPolicyDocument: {HTTPCode: 400, Message: "The policy document is malformed."},
 	ErrorAccessDenied:               {HTTPCode: 403, Message: "User is not authorized to perform this action."},

+	// ECR error codes
+	ErrorRepositoryNotFound:       {HTTPCode: 400, Message: "The repository could not be found. Check the spelling of the specified repository and ensure that you are performing operations on the correct registry."},
+	ErrorRepositoryPolicyNotFound: {HTTPCode: 400, Message: "The repository and registry do not have an associated repository policy."},
+	ErrorLifecyclePolicyNotFound:  {HTTPCode: 400, Message: "The lifecycle policy for the specified repository could not be found."},
+	ErrorRepositoryAlreadyExists:  {HTTPCode: 400, Message: "The repository already exists in the registry."},
+	ErrorRepositoryNotEmpty:       {HTTPCode: 400, Message: "The repository contains images. Either delete the images or use the force option to delete the repository."},
+	ErrorImageNotFound:            {HTTPCode: 400, Message: "The image requested does not exist in the specified repository."},
+	ErrorImageAlreadyExists:       {HTTPCode: 400, Message: "The specified image has already been pushed, and there were no changes to the manifest or image tag after the last push."},
+	ErrorImageTagAlreadyExists:    {HTTPCode: 400, Message: "The specified image is tagged with a tag that already exists. The repository is configured for tag immutability."},
+	ErrorImageDigestDoesNotMatch:  {HTTPCode: 400, Message: "The specified image digest does not match the digest of the supplied image manifest."},
+	ErrorLayersNotFound:           {HTTPCode: 400, Message: "The specified layers could not be found, or the specified layer is not valid for this repository."},
+	ErrorReferencedImagesNotFound: {HTTPCode: 400, Message: "The manifest list is referencing an image that does not exist."},
+	ErrorTooManyTags:              {HTTPCode: 400, Message: "The list of tags on the repository is over the limit. The maximum number of tags that can be applied to a repository is 50."},
+	ErrorOperationNotSupported:    {HTTPCode: 400, Message: "The specified operation is not supported in this registry."},
+
 	// ELBv2 error codes
 	ErrorELBv2LoadBalancerNotFound:         {HTTPCode: 400, Message: "One or more load balancers not found."},
 	ErrorELBv2TargetGroupNotFound:          {HTTPCode: 400, Message: "One or more target groups not found."},
@@ -70,6 +70,28 @@ func TestValidErrorCode(t *testing.T) {
 	}
 }

+func TestHasErrorCode(t *testing.T) {
+	tests := []struct {
+		name string
+		code string
+		want bool
+	}{
+		{"known code", ErrorInsufficientInstanceCapacity, true},
+		{"another known code", ErrorAuthFailure, true},
+		{"wrapped code is not exact", "launch workers: " + ErrorInsufficientInstanceCapacity, false},
+		{"unknown code", "CompletelyMadeUp", false},
+		{"empty string", "", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := HasErrorCode(tt.code); got != tt.want {
+				t.Errorf("HasErrorCode(%q) = %v, want %v", tt.code, got, tt.want)
+			}
+		})
+	}
+}
+
 // TestIsNotFound covers the destroy-orchestration tolerance helper: it matches
 // any AWS canonical InvalidX.NotFound (so teardown converges on already-gone
 // resources) and only those — a nil error or a non-NotFound failure is not
@@ -17,9 +17,24 @@ type ClusterConfig struct {
 	Version   string            `mapstructure:"version"`   // spinifex version
 	Network   NetworkConfig     `mapstructure:"network"`   // cluster-wide external network settings
 	Bootstrap BootstrapConfig   `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
+	AWS       AWSConfig         `mapstructure:"aws"`       // cluster-wide AWS-parity settings (region, endpoint suffix)
 	Nodes     map[string]Config `mapstructure:"nodes"`     // full config for every node
 }

+// Defaults for cluster-wide AWS-parity settings.
+const (
+	DefaultAWSRegion         = "us-east-1"
+	DefaultAWSInternalSuffix = "spinifex.internal"
+)
+
+// AWSConfig holds cluster-wide AWS-parity settings shared across services.
+// Region scopes the default AWS region; InternalSuffix is the internal DNS
+// suffix used to build service endpoints (e.g. ecr.{region}.{suffix}).
+type AWSConfig struct {
+	Region         string `mapstructure:"region"`
+	InternalSuffix string `mapstructure:"internal_suffix"`
+}
+
 // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
 type ExternalPool struct {
 	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
@@ -210,6 +225,10 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 	// Default ipsec_enabled to true; operators must explicitly set false to disable.
 	viper.SetDefault("network.ipsec_enabled", true)

+	// Cluster-wide AWS-parity defaults so existing deployments keep working.
+	viper.SetDefault("aws.region", DefaultAWSRegion)
+	viper.SetDefault("aws.internal_suffix", DefaultAWSInternalSuffix)
+
 	// Try to load config file if it exists
 	if configPath != "" {
 		// Check if file exists
@@ -232,6 +251,14 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 		return nil, fmt.Errorf("error unmarshaling config: %w", err)
 	}

+	// Backfill AWS-parity defaults when keys are unset so callers never see empties.
+	if config.AWS.Region == "" {
+		config.AWS.Region = DefaultAWSRegion
+	}
+	if config.AWS.InternalSuffix == "" {
+		config.AWS.InternalSuffix = DefaultAWSInternalSuffix
+	}
+
 	// Rewrite 0.0.0.0 in Predastore.Host to 127.0.0.1 for the local node only (not a valid connect address).
 	if local, ok := config.Nodes[config.Node]; ok {
 		if strings.HasPrefix(local.Predastore.Host, "0.0.0.0") {
@@ -98,6 +98,38 @@ func TestLoadConfig_EmptyConfigPath(t *testing.T) {
 	assert.Empty(t, cfg.Node)
 }

+func TestLoadConfig_AWSDefaults(t *testing.T) {
+	resetViper(t)
+	cfg, err := LoadConfig("")
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+	assert.Equal(t, DefaultAWSRegion, cfg.AWS.Region)
+	assert.Equal(t, DefaultAWSInternalSuffix, cfg.AWS.InternalSuffix)
+}
+
+func TestLoadConfig_AWSOverride(t *testing.T) {
+	resetViper(t)
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+
+	toml := `
+node = "n1"
+
+[aws]
+region = "ap-southeast-2"
+internal_suffix = "dev.local"
+
+[nodes.n1]
+region = "ap-southeast-2"
+`
+	require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
+
+	cfg, err := LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, "ap-southeast-2", cfg.AWS.Region)
+	assert.Equal(t, "dev.local", cfg.AWS.InternalSuffix)
+}
+
 func TestLoadConfig_NonexistentFile(t *testing.T) {
 	resetViper(t)
 	cfg, err := LoadConfig("/tmp/nonexistent-spinifex-config-test-12345.toml")
@@ -0,0 +1,183 @@
+package daemon
+
+import (
+	"errors"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// capacityReservation is an in-memory On-Demand Capacity Reservation pinned to
+// this node, holding a TotalInstanceCount × per-instance compute carve-out via
+// reservedCR*. Present in the map means active; cancel removes it. Lost on restart.
+type capacityReservation struct {
+	ID                    string
+	AccountID             string
+	InstanceType          string
+	AvailabilityZone      string
+	TotalInstanceCount    int
+	VCPUPerInstance       int
+	MemGBPerInstance      float64
+	InstanceMatchCriteria string
+	Tenancy               string
+	InstancePlatform      string
+	CreateDate            time.Time
+	ConsumedCount         int // instances launched into this reservation; slots moved reservedCR*->allocated*
+}
+
+// reservationCompute returns the full compute carve-out for a reservation:
+// TotalInstanceCount × per-instance (vCPU, memory). MemGBPerInstance already
+// folds in nbdkit overhead (set at create), so the reservedCR->allocated swap on
+// launch is net-zero.
+func (r *capacityReservation) reservationCompute() (vcpu int, memGB float64) {
+	return r.TotalInstanceCount * r.VCPUPerInstance, float64(r.TotalInstanceCount) * r.MemGBPerInstance
+}
+
+// CreateReservation re-checks fit under a single write-lock, then bumps reservedCR*,
+// stores the record, and refreshes subscriptions, so a concurrent allocate cannot
+// overcommit. Returns InsufficientInstanceCapacity if the carve-out no longer fits.
+func (rm *ResourceManager) CreateReservation(rec *capacityReservation) error {
+	wantVCPU, wantMem := rec.reservationCompute()
+
+	rm.mu.Lock()
+	remVCPU := rm.hostVCPU - rm.reservedVCPU - rm.reservedCRVCPU - rm.allocatedVCPU
+	remMem := rm.hostMemGB - rm.reservedMem - rm.reservedCRMem - rm.allocatedMem
+	if wantVCPU > remVCPU || wantMem > remMem {
+		rm.mu.Unlock()
+		return errors.New(awserrors.ErrorInsufficientInstanceCapacity)
+	}
+	rm.reservedCRVCPU += wantVCPU
+	rm.reservedCRMem += wantMem
+	rm.reservations[rec.ID] = rec
+	rm.mu.Unlock()
+
+	rm.updateInstanceSubscriptions()
+	return nil
+}
+
+// CancelReservation releases the unconsumed remainder of an account-owned
+// reservation's carve-out and removes it; consumed slots already moved to
+// allocated* and keep running. Returns the removed record and true on success,
+// or (nil, false) if no reservation with that id is owned by the account.
+func (rm *ResourceManager) CancelReservation(id, accountID string) (*capacityReservation, bool) {
+	rm.mu.Lock()
+	rec, ok := rm.reservations[id]
+	if !ok || rec.AccountID != accountID {
+		rm.mu.Unlock()
+		return nil, false
+	}
+	remaining := rec.TotalInstanceCount - rec.ConsumedCount
+	rm.reservedCRVCPU -= remaining * rec.VCPUPerInstance
+	rm.reservedCRMem -= float64(remaining) * rec.MemGBPerInstance
+	delete(rm.reservations, id)
+	rm.mu.Unlock()
+
+	rm.updateInstanceSubscriptions()
+	return rec, true
+}
+
+// AllocateFromReservation launches one instance into reservation crID via a
+// net-zero swap under rm.mu: the per-instance compute already held in reservedCR*
+// at create moves to allocated* and ConsumedCount bumps, so schedulable capacity
+// is unchanged by construction. Errors: unknown/foreign id ->
+// InvalidCapacityReservationId.NotFound; type mismatch -> InvalidParameterValue;
+// already full -> ReservationCapacityExceeded.
+func (rm *ResourceManager) AllocateFromReservation(crID, accountID string, it *ec2.InstanceTypeInfo) error {
+	rm.mu.Lock()
+	rec, ok := rm.reservations[crID]
+	if !ok || rec.AccountID != accountID {
+		rm.mu.Unlock()
+		return errors.New(awserrors.ErrorInvalidCapacityReservationIdNotFound)
+	}
+	if rec.InstanceType != aws.StringValue(it.InstanceType) {
+		rm.mu.Unlock()
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if rec.ConsumedCount >= rec.TotalInstanceCount {
+		rm.mu.Unlock()
+		return errors.New(awserrors.ErrorReservationCapacityExceeded)
+	}
+	rm.reservedCRVCPU -= rec.VCPUPerInstance
+	rm.reservedCRMem -= rec.MemGBPerInstance
+	rm.allocatedVCPU += rec.VCPUPerInstance
+	rm.allocatedMem += rec.MemGBPerInstance
+	rec.ConsumedCount++
+	rm.mu.Unlock()
+
+	rm.updateInstanceSubscriptions()
+	return nil
+}
+
+// ReleaseToReservation returns a launched instance's slot. If the reservation
+// still exists it is the inverse net-zero swap (allocated* -> reservedCR*,
+// ConsumedCount--); if the reservation is gone (cancelled or lost on restart) the
+// allocation is freed to the general pool using the catalog charge so it never
+// overcounts. Best-effort: a missing reservation is not an error.
+func (rm *ResourceManager) ReleaseToReservation(crID string, it *ec2.InstanceTypeInfo) {
+	rm.mu.Lock()
+	if rec, ok := rm.reservations[crID]; ok && rec.ConsumedCount > 0 {
+		rm.allocatedVCPU -= rec.VCPUPerInstance
+		rm.allocatedMem -= rec.MemGBPerInstance
+		rm.reservedCRVCPU += rec.VCPUPerInstance
+		rm.reservedCRMem += rec.MemGBPerInstance
+		rec.ConsumedCount--
+	} else {
+		rm.allocatedVCPU -= int(instanceTypeVCPUs(it))
+		rm.allocatedMem -= float64(rm.instanceMemChargeMiB(it)) / 1024.0
+	}
+	rm.mu.Unlock()
+
+	rm.updateInstanceSubscriptions()
+}
+
+// ReservationAvailable reports how many more instances can launch into crID
+// (Total - Consumed), or 0 when the reservation is unknown, owned by another
+// account, or for a different instance type. The daemon handler's up-front check
+// turns those zero cases into precise errors before the launch loop.
+func (rm *ResourceManager) ReservationAvailable(crID, accountID string, it *ec2.InstanceTypeInfo) int {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+	rec, ok := rm.reservations[crID]
+	if !ok || rec.AccountID != accountID || rec.InstanceType != aws.StringValue(it.InstanceType) {
+		return 0
+	}
+	return rec.TotalInstanceCount - rec.ConsumedCount
+}
+
+// ValidateReservationTarget is the owning daemon's up-front semantic check for a
+// targeted launch, run before the launch loop. It returns
+// InvalidCapacityReservationId.NotFound for an unknown or foreign id and
+// InvalidParameterValue on an instance-type mismatch. The "full" case is left to
+// the count gate in PrepareRunInstances, which caps the launch at the available
+// slots and returns ReservationCapacityExceeded when none remain.
+func (rm *ResourceManager) ValidateReservationTarget(crID, accountID string, it *ec2.InstanceTypeInfo) error {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+	rec, ok := rm.reservations[crID]
+	if !ok || rec.AccountID != accountID {
+		return errors.New(awserrors.ErrorInvalidCapacityReservationIdNotFound)
+	}
+	if rec.InstanceType != aws.StringValue(it.InstanceType) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return nil
+}
+
+// ListReservations returns value-copy snapshots of the account's reservations on
+// this node, taken under the read lock. Copies rather than the live pointers
+// because ConsumedCount is mutated under rm.mu by the launch/release paths, so a
+// caller reading the shared record lock-free would race.
+func (rm *ResourceManager) ListReservations(accountID string) []capacityReservation {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+
+	var out []capacityReservation
+	for _, rec := range rm.reservations {
+		if rec.AccountID == accountID {
+			out = append(out, *rec)
+		}
+	}
+	return out
+}
@@ -0,0 +1,355 @@
+package daemon
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// microType is a deterministic 2 vCPU / 1 GB instance type for carve-out math.
+// With nbdkit charges left at 0 the memory charge equals the catalog figure.
+func microType() *ec2.InstanceTypeInfo {
+	return &ec2.InstanceTypeInfo{
+		InstanceType: aws.String("t3.micro"),
+		VCpuInfo:     &ec2.VCpuInfo{DefaultVCpus: aws.Int64(2)},
+		MemoryInfo:   &ec2.MemoryInfo{SizeInMiB: aws.Int64(1024)},
+	}
+}
+
+// newReservationTestRM builds an 8 vCPU / 16 GB node with no host reserve, so a
+// fresh node fits exactly 4 t3.micro (CPU-bound: 8/2).
+func newReservationTestRM() *ResourceManager {
+	return &ResourceManager{
+		hostVCPU:  8,
+		hostMemGB: 16.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{
+			"t3.micro": microType(),
+		},
+		reservations: make(map[string]*capacityReservation),
+	}
+}
+
+func microReservation(id, accountID string, count int) *capacityReservation {
+	return &capacityReservation{
+		ID:                    id,
+		AccountID:             accountID,
+		InstanceType:          "t3.micro",
+		AvailabilityZone:      "ap-southeast-2a",
+		TotalInstanceCount:    count,
+		VCPUPerInstance:       2,
+		MemGBPerInstance:      1.0,
+		InstanceMatchCriteria: "open",
+		Tenancy:               "default",
+		InstancePlatform:      "Linux/UNIX",
+		CreateDate:            time.Now(),
+	}
+}
+
+// A reservation lowers both canAllocate and the GetResourceStats Available count
+// by its headline compute, then cancelling restores them.
+func TestCapacityReservation_CarveOutAndRestore(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+
+	require.Equal(t, 4, rm.canAllocate(mt, 100), "fresh node fits 4 t3.micro")
+
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 2)))
+
+	assert.Equal(t, 4, rm.reservedCRVCPU)
+	assert.Equal(t, 2.0, rm.reservedCRMem)
+	assert.Equal(t, 2, rm.canAllocate(mt, 100), "carve-out of 2 leaves room for 2 more")
+
+	// GetResourceStats folds the carve-out into the reported reserve and Available.
+	_, _, reservedVCPU, reservedMem, _, _, caps := rm.GetResourceStats()
+	assert.Equal(t, 4, reservedVCPU)
+	assert.Equal(t, 2.0, reservedMem)
+	var available int
+	for _, c := range caps {
+		if c.Name == "t3.micro" {
+			available = c.Available
+		}
+	}
+	assert.Equal(t, 2, available, "GetResourceStats Available reflects the carve-out")
+
+	rec, ok := rm.CancelReservation("cr-1", "acct-a")
+	require.True(t, ok)
+	assert.Equal(t, "cr-1", rec.ID)
+	assert.Equal(t, 0, rm.reservedCRVCPU)
+	assert.Equal(t, 0.0, rm.reservedCRMem)
+	assert.Equal(t, 4, rm.canAllocate(mt, 100), "cancel restores full capacity")
+}
+
+// A reservation larger than remaining schedulable capacity is rejected and
+// leaves the carve-out untouched.
+func TestCapacityReservation_OverReserveRejected(t *testing.T) {
+	rm := newReservationTestRM()
+
+	err := rm.CreateReservation(microReservation("cr-big", "acct-a", 5)) // 10 vCPU > 8
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error())
+	assert.Equal(t, 0, rm.reservedCRVCPU)
+	assert.Equal(t, 0.0, rm.reservedCRMem)
+	assert.Empty(t, rm.reservations)
+}
+
+// Cancel is account-scoped: a foreign account cannot release another's hold, and
+// an unknown id is reported missing.
+func TestCapacityReservation_CancelScoping(t *testing.T) {
+	rm := newReservationTestRM()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 1)))
+
+	_, ok := rm.CancelReservation("cr-1", "acct-b")
+	assert.False(t, ok, "another account cannot cancel the reservation")
+	assert.Equal(t, 2, rm.reservedCRVCPU, "carve-out unchanged after foreign cancel")
+
+	_, ok = rm.CancelReservation("cr-unknown", "acct-a")
+	assert.False(t, ok, "unknown id reports not found")
+
+	_, ok = rm.CancelReservation("cr-1", "acct-a")
+	assert.True(t, ok)
+}
+
+// ListReservations returns only the account's reservations.
+func TestCapacityReservation_ListScoping(t *testing.T) {
+	rm := newReservationTestRM()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 1)))
+	require.NoError(t, rm.CreateReservation(microReservation("cr-2", "acct-b", 1)))
+
+	a := rm.ListReservations("acct-a")
+	require.Len(t, a, 1)
+	assert.Equal(t, "cr-1", a[0].ID)
+
+	assert.Empty(t, rm.ListReservations("acct-c"))
+}
+
+// Concurrent CreateReservation and allocate share the same rm.mu re-check gate,
+// so the combined committed compute can never exceed the host's schedulable pool.
+func TestCapacityReservation_ConcurrentNoOvercommit(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+
+	const workers = 32
+	var wg sync.WaitGroup
+	wg.Add(workers)
+	for i := range workers {
+		go func(i int) {
+			defer wg.Done()
+			if i%2 == 0 {
+				_ = rm.CreateReservation(microReservation(fmt.Sprintf("cr-%d", i), "acct-a", 1))
+			} else {
+				_ = rm.allocate(mt)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+
+	committedVCPU := rm.reservedCRVCPU + rm.allocatedVCPU
+	committedMem := rm.reservedCRMem + rm.allocatedMem
+	assert.LessOrEqual(t, committedVCPU, rm.hostVCPU, "vCPU never overcommitted")
+	assert.LessOrEqual(t, committedMem, rm.hostMemGB, "memory never overcommitted")
+
+	// reservedCR* stays consistent with the surviving reservation records.
+	var sumVCPU int
+	var sumMem float64
+	for _, rec := range rm.reservations {
+		v, m := rec.reservationCompute()
+		sumVCPU += v
+		sumMem += m
+	}
+	assert.Equal(t, sumVCPU, rm.reservedCRVCPU)
+	assert.Equal(t, sumMem, rm.reservedCRMem)
+}
+
+// AllocateFromReservation moves a slot reservedCR*->allocated* without touching
+// schedulable capacity, bumps ConsumedCount, and drops AvailableInstanceCount.
+func TestAllocateFromReservation_NetZeroSwap(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	rec := microReservation("cr-1", "acct-a", 2)
+	require.NoError(t, rm.CreateReservation(rec))
+
+	before := rm.canAllocate(mt, 100)
+	require.Equal(t, 2, before, "carve-out of 2 leaves room for 2 general launches")
+
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt))
+
+	rm.mu.RLock()
+	assert.Equal(t, 2, rm.reservedCRVCPU, "one slot left reservedCR*")
+	assert.Equal(t, 1.0, rm.reservedCRMem)
+	assert.Equal(t, 2, rm.allocatedVCPU, "consumed slot now in allocated*")
+	assert.Equal(t, 1.0, rm.allocatedMem)
+	assert.Equal(t, 1, rec.ConsumedCount)
+	rm.mu.RUnlock()
+
+	assert.Equal(t, before, rm.canAllocate(mt, 100), "schedulable capacity unchanged by the swap")
+	assert.Equal(t, int64(1), aws.Int64Value(rec.toAWSCapacityReservation().AvailableInstanceCount))
+}
+
+// The semantic checks reject unknown/foreign ids, type mismatch, and over-consume
+// with their mapped AWS error codes, leaving accounting untouched.
+func TestAllocateFromReservation_Errors(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 1)))
+
+	err := rm.AllocateFromReservation("cr-missing", "acct-a", mt)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidCapacityReservationIdNotFound, err.Error())
+
+	err = rm.AllocateFromReservation("cr-1", "acct-b", mt)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidCapacityReservationIdNotFound, err.Error(), "foreign account is not found")
+
+	wrongType := &ec2.InstanceTypeInfo{
+		InstanceType: aws.String("t3.large"),
+		VCpuInfo:     &ec2.VCpuInfo{DefaultVCpus: aws.Int64(2)},
+		MemoryInfo:   &ec2.MemoryInfo{SizeInMiB: aws.Int64(8192)},
+	}
+	err = rm.AllocateFromReservation("cr-1", "acct-a", wrongType)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt), "first slot consumes")
+	err = rm.AllocateFromReservation("cr-1", "acct-a", mt)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorReservationCapacityExceeded, err.Error(), "second exceeds Total=1")
+}
+
+// ReleaseToReservation is the inverse swap when the reservation still exists:
+// allocated*->reservedCR*, ConsumedCount--, AvailableInstanceCount restored.
+func TestReleaseToReservation_RestoresSlot(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	rec := microReservation("cr-1", "acct-a", 2)
+	require.NoError(t, rm.CreateReservation(rec))
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt))
+
+	rm.ReleaseToReservation("cr-1", mt)
+
+	rm.mu.RLock()
+	assert.Equal(t, 4, rm.reservedCRVCPU, "slot returned to reservedCR*")
+	assert.Equal(t, 2.0, rm.reservedCRMem)
+	assert.Equal(t, 0, rm.allocatedVCPU)
+	assert.Equal(t, 0.0, rm.allocatedMem)
+	assert.Equal(t, 0, rec.ConsumedCount)
+	rm.mu.RUnlock()
+
+	assert.Equal(t, int64(2), aws.Int64Value(rec.toAWSCapacityReservation().AvailableInstanceCount))
+}
+
+// After the reservation is cancelled, a still-running consumer's release frees to
+// the general pool (catalog charge) instead of a now-gone reservedCR*, never
+// overcounting. Cancel itself frees only the unconsumed remainder.
+func TestReleaseToReservation_AfterCancelFreesToGeneralPool(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 2)))
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt))
+
+	_, ok := rm.CancelReservation("cr-1", "acct-a")
+	require.True(t, ok)
+
+	rm.mu.RLock()
+	assert.Equal(t, 0, rm.reservedCRVCPU, "cancel freed only the unconsumed slot")
+	assert.Equal(t, 0.0, rm.reservedCRMem)
+	assert.Equal(t, 2, rm.allocatedVCPU, "consumed slot keeps running on general capacity")
+	rm.mu.RUnlock()
+
+	rm.ReleaseToReservation("cr-1", mt)
+
+	rm.mu.RLock()
+	assert.Equal(t, 0, rm.allocatedVCPU, "release frees the dangling consumer to general pool")
+	assert.Equal(t, 0.0, rm.allocatedMem)
+	assert.Equal(t, 0, rm.reservedCRVCPU, "no phantom reservedCR* re-added")
+	rm.mu.RUnlock()
+
+	assert.Equal(t, 4, rm.canAllocate(mt, 100), "general capacity fully restored")
+}
+
+// ReservationAvailable reports Total-Consumed, and 0 for unknown/foreign-account
+// /type-mismatch so the count site degrades to the daemon's up-front error.
+func TestReservationAvailable(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 2)))
+
+	assert.Equal(t, 2, rm.ReservationAvailable("cr-1", "acct-a", mt))
+	assert.Equal(t, 0, rm.ReservationAvailable("cr-missing", "acct-a", mt))
+	assert.Equal(t, 0, rm.ReservationAvailable("cr-1", "acct-b", mt), "foreign account sees nothing")
+
+	wrongType := &ec2.InstanceTypeInfo{InstanceType: aws.String("t3.large")}
+	assert.Equal(t, 0, rm.ReservationAvailable("cr-1", "acct-a", wrongType), "type mismatch sees nothing")
+
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt))
+	assert.Equal(t, 1, rm.ReservationAvailable("cr-1", "acct-a", mt), "consume drops availability")
+}
+
+// ValidateReservationTarget is the up-front semantic check: unknown/foreign ids
+// map to NotFound, a type mismatch to InvalidParameterValue, and a matching
+// reservation passes (the full case is left to the launch-side count gate).
+func TestValidateReservationTarget(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	require.NoError(t, rm.CreateReservation(microReservation("cr-1", "acct-a", 1)))
+
+	require.NoError(t, rm.ValidateReservationTarget("cr-1", "acct-a", mt))
+
+	err := rm.ValidateReservationTarget("cr-missing", "acct-a", mt)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidCapacityReservationIdNotFound, err.Error())
+
+	err = rm.ValidateReservationTarget("cr-1", "acct-b", mt)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidCapacityReservationIdNotFound, err.Error(), "foreign account is not found")
+
+	wrongType := &ec2.InstanceTypeInfo{InstanceType: aws.String("t3.large")}
+	err = rm.ValidateReservationTarget("cr-1", "acct-a", wrongType)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
+
+	// A full reservation still validates — the count gate handles capacity.
+	require.NoError(t, rm.AllocateFromReservation("cr-1", "acct-a", mt))
+	require.NoError(t, rm.ValidateReservationTarget("cr-1", "acct-a", mt), "full reservation passes the semantic check")
+}
+
+// Concurrent general allocate and AllocateFromReservation share rm.mu; the swap is
+// net-zero on reservedCR*+allocated*, so committed compute never exceeds the host
+// and ConsumedCount never passes Total.
+func TestAllocateFromReservation_ConcurrentNoOvercommit(t *testing.T) {
+	rm := newReservationTestRM()
+	mt := microType()
+	rec := microReservation("cr-1", "acct-a", 2)
+	require.NoError(t, rm.CreateReservation(rec))
+
+	const workers = 32
+	var wg sync.WaitGroup
+	wg.Add(workers)
+	for i := range workers {
+		go func(i int) {
+			defer wg.Done()
+			if i%2 == 0 {
+				_ = rm.AllocateFromReservation("cr-1", "acct-a", mt)
+			} else {
+				_ = rm.allocate(mt)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+	assert.LessOrEqual(t, rm.reservedCRVCPU+rm.allocatedVCPU, rm.hostVCPU, "vCPU never overcommitted")
+	assert.LessOrEqual(t, rm.reservedCRMem+rm.allocatedMem, rm.hostMemGB, "memory never overcommitted")
+	assert.LessOrEqual(t, rec.ConsumedCount, rec.TotalInstanceCount, "ConsumedCount never passes Total")
+	assert.GreaterOrEqual(t, rec.ConsumedCount, 0)
+}
@@ -48,6 +48,7 @@ import (
 	handlers_ec2_tags "github.com/mulgadc/spinifex/spinifex/handlers/ec2/tags"
 	handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
+	handlers_ecr "github.com/mulgadc/spinifex/spinifex/handlers/ecr"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
 	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	handlers_imds "github.com/mulgadc/spinifex/spinifex/handlers/imds"
@@ -95,6 +96,14 @@ type ResourceManager struct {
 	reservedMem   float64
 	allocatedVCPU int
 	allocatedMem  float64
+	// reservedCRVCPU / reservedCRMem: compute held by capacity reservations
+	// pinned to this node. Subtracted from schedulable capacity exactly like
+	// the host reserve. In-memory only — lost on daemon restart.
+	reservedCRVCPU int
+	reservedCRMem  float64
+	// reservations: in-memory capacity reservations owned by this node, keyed
+	// by id. Mutated together with reservedCR* under mu.
+	reservations map[string]*capacityReservation
 	// nbdkitMainMiB / nbdkitAuxMiB: per-volume nbdkit memory charged at
 	// admission so nbdkit backing a guest's volumes is accounted explicitly.
 	nbdkitMainMiB int
@@ -142,6 +151,7 @@ type Daemon struct {
 	elbv2Service          *handlers_elbv2.ELBv2ServiceImpl
 	eksService            *handlers_eks.EKSServiceImpl
 	acmService            *handlers_acm.ACMServiceImpl
+	ecrMetaService        *handlers_ecr.MetaServiceImpl
 	routeTableService     *handlers_ec2_routetable.RouteTableServiceImpl
 	natGatewayService     *handlers_ec2_natgw.NatGatewayServiceImpl
 	externalIPAM          *handlers_ec2_vpc.ExternalIPAM
@@ -485,6 +495,7 @@ func NewResourceManager(gpuModels []instancetypes.GPUModel, migProfiles []instan
 		instanceTypes:      instanceTypes,
 		gpuManager:         gpuMgr,
 		readMemAvailableGB: memReader,
+		reservations:       make(map[string]*capacityReservation),
 	}, nil
 }

@@ -570,8 +581,8 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 		}

 		count := canAllocateCount(
-			rm.hostVCPU-rm.reservedVCPU, rm.allocatedVCPU,
-			rm.hostMemGB-rm.reservedMem, rm.allocatedMem,
+			rm.hostVCPU-rm.reservedVCPU-rm.reservedCRVCPU, rm.allocatedVCPU,
+			rm.hostMemGB-rm.reservedMem-rm.reservedCRMem, rm.allocatedMem,
 			vCPUs, rm.instanceMemChargeMiB(it),
 			1<<30, // effectively unlimited — let resources be the constraint
 			0, false,
@@ -589,6 +600,7 @@ func (rm *ResourceManager) GetAvailableInstanceTypeInfos(showCapacity bool) []*e
 	slog.Info("GetAvailableInstanceTypeInfos", "total_types", len(rm.instanceTypes), "total_available_slots", len(infos),
 		"hostVCPU", rm.hostVCPU, "hostMem", rm.hostMemGB,
 		"reservedVCPU", rm.reservedVCPU, "reservedMem", rm.reservedMem,
+		"reservedCRVCPU", rm.reservedCRVCPU, "reservedCRMem", rm.reservedCRMem,
 		"showCapacity", showCapacity)

 	return infos
@@ -624,17 +636,19 @@ func (rm *ResourceManager) GetResourceStats() (totalVCPU int, totalMemGB float64

 	totalVCPU = rm.hostVCPU
 	totalMemGB = rm.hostMemGB
-	reservedVCPU = rm.reservedVCPU
-	reservedMemGB = rm.reservedMem
+	// Fold the capacity-reservation carve-out into the reported reserve figures;
+	// Phase 1 has no separate status field for it.
+	reservedVCPU = rm.reservedVCPU + rm.reservedCRVCPU
+	reservedMemGB = rm.reservedMem + rm.reservedCRMem
 	allocVCPU = rm.allocatedVCPU
 	allocMemGB = rm.allocatedMem

-	remainingVCPU := rm.hostVCPU - rm.reservedVCPU - rm.allocatedVCPU
-	remainingMem := rm.hostMemGB - rm.reservedMem - rm.allocatedMem
+	remainingVCPU := rm.hostVCPU - reservedVCPU - rm.allocatedVCPU
+	remainingMem := rm.hostMemGB - reservedMemGB - rm.allocatedMem
 	if remainingVCPU < 0 || remainingMem < 0 {
 		slog.Error("schedulable capacity negative — reserve misconfigured or allocation drift",
-			"hostVCPU", rm.hostVCPU, "reservedVCPU", rm.reservedVCPU, "allocatedVCPU", rm.allocatedVCPU,
-			"hostMemGB", rm.hostMemGB, "reservedMem", rm.reservedMem, "allocatedMem", rm.allocatedMem,
+			"hostVCPU", rm.hostVCPU, "reservedVCPU", reservedVCPU, "allocatedVCPU", rm.allocatedVCPU,
+			"hostMemGB", rm.hostMemGB, "reservedMem", reservedMemGB, "allocatedMem", rm.allocatedMem,
 			"remainingVCPU", remainingVCPU, "remainingMem", remainingMem)
 	}

@@ -771,6 +785,11 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.RemoveInstanceFromPlacementGroup", d.handleEC2RemoveInstanceFromPlacementGroup, "spinifex-workers"},
 		{"ec2.ReserveClusterNode", d.handleEC2ReserveClusterNode, "spinifex-workers"},
 		{"ec2.FinalizeClusterInstances", d.handleEC2FinalizeClusterInstances, "spinifex-workers"},
+		// Capacity reservations: Create is node-targeted (gateway pins one node);
+		// Describe fans out and Cancel broadcasts, so both use plain Subscribe.
+		{fmt.Sprintf("ec2.CreateCapacityReservation.%s", d.node), d.handleEC2CreateCapacityReservation, ""},
+		{"ec2.DescribeCapacityReservations", d.handleEC2DescribeCapacityReservations, ""},
+		{"ec2.CancelCapacityReservation", d.handleEC2CancelCapacityReservation, ""},
 		{"ec2.CreateNatGateway", d.handleEC2CreateNatGateway, "spinifex-workers"},
 		{"ec2.DeleteNatGateway", d.handleEC2DeleteNatGateway, "spinifex-workers"},
 		{"ec2.DescribeNatGateways", d.handleEC2DescribeNatGateways, "spinifex-workers"},
@@ -936,6 +955,38 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"acm.DescribeCertificate", d.handleACMDescribeCertificate, "spinifex-workers"},
 			natsSub{"acm.ListCertificates", d.handleACMListCertificates, "spinifex-workers"},
 			natsSub{"acm.DeleteCertificate", d.handleACMDeleteCertificate, "spinifex-workers"},
+			natsSub{"acm.ListTagsForCertificate", d.handleACMListTagsForCertificate, "spinifex-workers"},
+			natsSub{"acm.AddTagsToCertificate", d.handleACMAddTagsToCertificate, "spinifex-workers"},
+			natsSub{"acm.RemoveTagsFromCertificate", d.handleACMRemoveTagsFromCertificate, "spinifex-workers"},
+		)
+	}
+
+	// ECR gateway → daemon subscriptions. The daemon owns the per-account
+	// JetStream KV metadata; blob/manifest bytes never traverse these subjects.
+	if d.ecrMetaService != nil {
+		subs = append(subs,
+			natsSub{handlers_ecr.SubjectRepoCreate, d.handleECRRepoCreate, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectRepoDescribe, d.handleECRRepoDescribe, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectRepoList, d.handleECRRepoList, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectRepoDelete, d.handleECRRepoDelete, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectPolicyPut, d.handleECRPolicyPut, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectPolicyGet, d.handleECRPolicyGet, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectPolicyDelete, d.handleECRPolicyDelete, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectLifecyclePut, d.handleECRLifecyclePut, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectLifecycleGet, d.handleECRLifecycleGet, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectLifecycleDelete, d.handleECRLifecycleDelete, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectTagPut, d.handleECRTagPut, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectTagGet, d.handleECRTagGet, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectTagList, d.handleECRTagList, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectTagDelete, d.handleECRTagDelete, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectManifestPut, d.handleECRManifestPut, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectManifestDescribe, d.handleECRManifestDescribe, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectManifestList, d.handleECRManifestList, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectManifestDelete, d.handleECRManifestDelete, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectUploadCreate, d.handleECRUploadCreate, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectUploadGet, d.handleECRUploadGet, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectUploadUpdate, d.handleECRUploadUpdate, "spinifex-workers"},
+			natsSub{handlers_ecr.SubjectUploadDelete, d.handleECRUploadDelete, "spinifex-workers"},
 		)
 	}

@@ -1343,6 +1394,15 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("failed to initialize ACM service: %w", err)
 	}

+	// ECR metadata service: owns per-account JetStream KV for repos, tags,
+	// manifest records and upload-state CAS. Disabled (gateway returns NATS
+	// timeouts) when JetStream is unavailable.
+	if js, jsErr := d.natsConn.JetStream(); jsErr != nil {
+		slog.Warn("ECR metadata service disabled: JetStream unavailable", "err", jsErr)
+	} else {
+		d.ecrMetaService = handlers_ecr.NewKVMetaService(js)
+	}
+
 	if err := d.eksService.SpawnRegisteredReconcilers(); err != nil {
 		slog.Warn("EKS: SpawnRegisteredReconcilers failed", "err", err)
 	}
@@ -2023,8 +2083,8 @@ func (rm *ResourceManager) canAllocateLocked(instanceType *ec2.InstanceTypeInfo,
 	}

 	n := canAllocateCount(
-		rm.hostVCPU-rm.reservedVCPU, rm.allocatedVCPU,
-		rm.hostMemGB-rm.reservedMem, rm.allocatedMem,
+		rm.hostVCPU-rm.reservedVCPU-rm.reservedCRVCPU, rm.allocatedVCPU,
+		rm.hostMemGB-rm.reservedMem-rm.reservedCRMem, rm.allocatedMem,
 		instanceTypeVCPUs(instanceType),
 		rm.instanceMemChargeMiB(instanceType),
 		count,
@@ -2044,7 +2104,7 @@ func (rm *ResourceManager) liveMemGate(n int, instanceType *ec2.InstanceTypeInfo
 		return n
 	}
 	memGB := float64(rm.instanceMemChargeMiB(instanceType)) / 1024.0
-	return liveMemCount(n, availGB, rm.reservedMem, memGB)
+	return liveMemCount(n, availGB, rm.reservedMem+rm.reservedCRMem, memGB)
 }

 // allocate reserves resources for one instance and updates NATS subscriptions.
@@ -2084,6 +2144,8 @@ func (rm *ResourceManager) deallocate(instanceType *ec2.InstanceTypeInfo) {
 	rm.updateInstanceSubscriptions()
 }

+var _ handlers_ec2_instance.InstanceTypeAllocator = (*ResourceManager)(nil)
+
 // Allocate, Deallocate, CanAllocate satisfy handlers_ec2_instance.InstanceTypeAllocator.
 func (rm *ResourceManager) Allocate(it *ec2.InstanceTypeInfo) error { return rm.allocate(it) }
 func (rm *ResourceManager) Deallocate(it *ec2.InstanceTypeInfo)     { rm.deallocate(it) }
@@ -17,3 +17,15 @@ func (d *Daemon) handleACMListCertificates(msg *nats.Msg) {
 func (d *Daemon) handleACMDeleteCertificate(msg *nats.Msg) {
 	handleNATSRequest(msg, d.acmService.DeleteCertificate)
 }
+
+func (d *Daemon) handleACMListTagsForCertificate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.acmService.ListTagsForCertificate)
+}
+
+func (d *Daemon) handleACMAddTagsToCertificate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.acmService.AddTagsToCertificate)
+}
+
+func (d *Daemon) handleACMRemoveTagsFromCertificate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.acmService.RemoveTagsFromCertificate)
+}
@@ -0,0 +1,183 @@
+package daemon
+
+import (
+	"log/slog"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/instancetypes"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+)
+
+// handleEC2CreateCapacityReservation is the node-targeted Create handler: resolve
+// the per-instance carve-out from the local catalog, generate the id, and commit
+// under the fit re-check. A lost race returns InsufficientInstanceCapacity.
+func (d *Daemon) handleEC2CreateCapacityReservation(msg *nats.Msg) {
+	accountID := utils.AccountIDFromMsg(msg)
+	input := new(ec2.CreateCapacityReservationInput)
+	if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil {
+		if err := msg.Respond(errResp); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
+		return
+	}
+
+	// GPU capacity reservations are out of scope; an unknown type is equally
+	// unreservable. Both reject as InvalidInstanceType.
+	instanceType := aws.StringValue(input.InstanceType)
+	it := d.resourceMgr.instanceTypes[instanceType]
+	if it == nil || instancetypes.IsGPUType(it) {
+		respondWithError(msg, awserrors.ErrorInvalidInstanceType)
+		return
+	}
+
+	matchCriteria := aws.StringValue(input.InstanceMatchCriteria)
+	if matchCriteria == "" {
+		matchCriteria = ec2.InstanceMatchCriteriaOpen
+	}
+	tenancy := aws.StringValue(input.Tenancy)
+	if tenancy == "" {
+		tenancy = ec2.CapacityReservationTenancyDefault
+	}
+
+	rec := &capacityReservation{
+		ID:                    utils.GenerateResourceID("cr"),
+		AccountID:             accountID,
+		InstanceType:          instanceType,
+		AvailabilityZone:      aws.StringValue(input.AvailabilityZone),
+		TotalInstanceCount:    int(aws.Int64Value(input.InstanceCount)),
+		VCPUPerInstance:       int(instanceTypeVCPUs(it)),
+		MemGBPerInstance:      float64(d.resourceMgr.instanceMemChargeMiB(it)) / 1024.0,
+		InstanceMatchCriteria: matchCriteria,
+		Tenancy:               tenancy,
+		InstancePlatform:      aws.StringValue(input.InstancePlatform),
+		CreateDate:            time.Now().UTC(),
+	}
+
+	if err := d.resourceMgr.CreateReservation(rec); err != nil {
+		respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
+		return
+	}
+
+	// Subscribe the per-reservation launch subject before responding: the SUB and
+	// this reply share d.natsConn, so the server registers the responder before
+	// the gateway sees the new id and can route a targeted launch to it. A failed
+	// subscribe would leave the reservation holding capacity no targeted launch
+	// could ever reach, so roll it back rather than return a dead id.
+	if err := d.subscribeReservationLaunch(rec.ID); err != nil {
+		slog.Error("Failed to subscribe reservation launch subject, rolling back reservation",
+			"crId", rec.ID, "err", err)
+		d.resourceMgr.CancelReservation(rec.ID, accountID)
+		respondWithError(msg, awserrors.ErrorServerInternal)
+		return
+	}
+
+	respondWithJSON(msg, rec.toAWSCapacityReservation())
+}
+
+// handleEC2DescribeCapacityReservations is the fan-out Describe handler. Each
+// node returns its own in-memory reservations for the account (possibly empty);
+// the gateway aggregates and applies id/filter scoping.
+func (d *Daemon) handleEC2DescribeCapacityReservations(msg *nats.Msg) {
+	accountID := utils.AccountIDFromMsg(msg)
+
+	out := &ec2.DescribeCapacityReservationsOutput{}
+	for _, rec := range d.resourceMgr.ListReservations(accountID) {
+		out.CapacityReservations = append(out.CapacityReservations, rec.toAWSCapacityReservation())
+	}
+	respondWithJSON(msg, out)
+}
+
+// handleEC2CancelCapacityReservation is the broadcast Cancel handler. Only the
+// node owning the reservation releases it; every node acks with Return set so
+// the gateway can tell "cancelled" from "no node owns this id".
+func (d *Daemon) handleEC2CancelCapacityReservation(msg *nats.Msg) {
+	accountID := utils.AccountIDFromMsg(msg)
+	input := new(ec2.CancelCapacityReservationInput)
+	if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil {
+		if err := msg.Respond(errResp); err != nil {
+			slog.Error("Failed to respond to NATS request", "err", err)
+		}
+		return
+	}
+
+	_, found := d.resourceMgr.CancelReservation(aws.StringValue(input.CapacityReservationId), accountID)
+	if found {
+		// Drop the launch subject so a targeted launch against the cancelled id
+		// hits no responder → InvalidCapacityReservationId.NotFound at the gateway.
+		d.unsubscribeReservationLaunch(aws.StringValue(input.CapacityReservationId))
+	}
+	respondWithJSON(msg, &ec2.CancelCapacityReservationOutput{Return: aws.Bool(found)})
+}
+
+// reservationLaunchSubject is the per-reservation subject the owning daemon
+// subscribes so the gateway routes a targeted launch straight to it, with no
+// cr→node lookup. ErrNoResponders on this subject therefore means the
+// reservation is gone (cancelled or lost to a restart).
+func reservationLaunchSubject(crID string) string {
+	return "ec2.RunInstances.cr." + crID
+}
+
+// subscribeReservationLaunch wires the cr launch subject to the standard
+// RunInstances handler — the target id rides in the input, so handleEC2RunInstances
+// serves it unchanged. Tracked in natsSubscriptions (keyed by cr id) so cancel
+// and shutdown can drop it. Idempotent; returns the subscribe error so the
+// caller can roll the reservation back rather than leave it unlaunchable.
+func (d *Daemon) subscribeReservationLaunch(crID string) error {
+	if d.natsConn == nil {
+		return nil
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if _, exists := d.natsSubscriptions[crID]; exists {
+		return nil
+	}
+	sub, err := d.natsConn.Subscribe(reservationLaunchSubject(crID), d.handleEC2RunInstances)
+	if err != nil {
+		return err
+	}
+	d.natsSubscriptions[crID] = sub
+	return nil
+}
+
+// unsubscribeReservationLaunch drops the cr launch subscription on cancel.
+func (d *Daemon) unsubscribeReservationLaunch(crID string) {
+	d.mu.Lock()
+	sub, ok := d.natsSubscriptions[crID]
+	if ok {
+		delete(d.natsSubscriptions, crID)
+	}
+	d.mu.Unlock()
+	if ok {
+		if err := sub.Unsubscribe(); err != nil {
+			slog.Warn("Failed to unsubscribe reservation launch subject", "crId", crID, "err", err)
+		}
+	}
+}
+
+// toAWSCapacityReservation renders the in-memory record as the AWS API type.
+// Reservations are always active with AvailableInstanceCount = Total − Consumed,
+// and never expire (EndDateType=unlimited).
+func (r *capacityReservation) toAWSCapacityReservation() *ec2.CapacityReservation {
+	return &ec2.CapacityReservation{
+		CapacityReservationId:  aws.String(r.ID),
+		OwnerId:                aws.String(r.AccountID),
+		InstanceType:           aws.String(r.InstanceType),
+		InstancePlatform:       aws.String(r.InstancePlatform),
+		AvailabilityZone:       aws.String(r.AvailabilityZone),
+		TotalInstanceCount:     aws.Int64(int64(r.TotalInstanceCount)),
+		AvailableInstanceCount: aws.Int64(int64(r.TotalInstanceCount - r.ConsumedCount)),
+		State:                  aws.String(ec2.CapacityReservationStateActive),
+		StartDate:              aws.Time(r.CreateDate),
+		CreateDate:             aws.Time(r.CreateDate),
+		InstanceMatchCriteria:  aws.String(r.InstanceMatchCriteria),
+		Tenancy:                aws.String(r.Tenancy),
+		EndDateType:            aws.String(ec2.EndDateTypeUnlimited),
+		EbsOptimized:           aws.Bool(false),
+		EphemeralStorage:       aws.Bool(false),
+		ReservationType:        aws.String(ec2.CapacityReservationTypeDefault),
+	}
+}
@@ -0,0 +1,309 @@
+package daemon
+
+import (
+	"encoding/json"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// crIDPattern matches the cr-<17 hex> id minted by GenerateResourceID("cr").
+var crIDPattern = regexp.MustCompile(`^cr-[0-9a-f]{17}$`)
+
+// natsRequestAs is natsRequest with an explicit account, for exercising the
+// handlers' account scoping (natsRequest always uses testAccountID).
+func natsRequestAs(nc *nats.Conn, subject string, data []byte, account string, timeout time.Duration) (*nats.Msg, error) {
+	msg := nats.NewMsg(subject)
+	msg.Data = data
+	msg.Header.Set("X-Account-ID", account)
+	return nc.RequestMsg(msg, timeout)
+}
+
+// replyErrCode returns the awserror Code on an error reply, or "" if the reply
+// is a normal (non-error) payload.
+func replyErrCode(t *testing.T, data []byte) string {
+	t.Helper()
+	var re struct {
+		Code string `json:"Code"`
+	}
+	require.NoError(t, json.Unmarshal(data, &re))
+	return re.Code
+}
+
+// seedReservation inserts a reservation directly into the node's map, bypassing
+// the fit gate so scoping tests can hold several records (the schedulable pool
+// only fits one t3.micro under the pinned 4-vCPU test host).
+func seedReservation(rm *ResourceManager, id, account, instanceType string) {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+	rm.reservations[id] = &capacityReservation{
+		ID:                    id,
+		AccountID:             account,
+		InstanceType:          instanceType,
+		AvailabilityZone:      "ap-southeast-2a",
+		TotalInstanceCount:    1,
+		VCPUPerInstance:       2,
+		MemGBPerInstance:      1.0,
+		InstanceMatchCriteria: ec2.InstanceMatchCriteriaOpen,
+		Tenancy:               ec2.CapacityReservationTenancyDefault,
+		InstancePlatform:      "Linux/UNIX",
+		CreateDate:            time.Now().UTC(),
+	}
+}
+
+// The node-targeted Create handler resolves per-instance compute from the local
+// catalog, mints a cr-id, commits the carve-out, and renders the AWS view.
+func TestHandleEC2CreateCapacityReservation_RoundTrip(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	instType := getTestInstanceType(t)
+	it := daemon.resourceMgr.instanceTypes[instType]
+	require.NotNil(t, it, "test instance type missing from catalog")
+	wantVCPU := int(instanceTypeVCPUs(it))
+	require.GreaterOrEqual(t, daemon.resourceMgr.canAllocate(it, 1000), 1, "host must fit one for the round-trip")
+
+	subject := "ec2.CreateCapacityReservation." + daemon.node
+	sub, err := daemon.natsConn.Subscribe(subject, daemon.handleEC2CreateCapacityReservation)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	reqData, _ := json.Marshal(&ec2.CreateCapacityReservationInput{
+		InstanceType:     aws.String(instType),
+		InstanceCount:    aws.Int64(1),
+		AvailabilityZone: aws.String("ap-southeast-2a"),
+		InstancePlatform: aws.String("Linux/UNIX"),
+	})
+	reply, err := natsRequest(daemon.natsConn, subject, reqData, 5*time.Second)
+	require.NoError(t, err)
+	require.Empty(t, replyErrCode(t, reply.Data), "create should not error: %s", reply.Data)
+
+	var cr ec2.CapacityReservation
+	require.NoError(t, json.Unmarshal(reply.Data, &cr))
+	assert.Regexp(t, crIDPattern, aws.StringValue(cr.CapacityReservationId))
+	assert.Equal(t, testAccountID, aws.StringValue(cr.OwnerId))
+	assert.Equal(t, instType, aws.StringValue(cr.InstanceType))
+	assert.Equal(t, "ap-southeast-2a", aws.StringValue(cr.AvailabilityZone))
+	assert.Equal(t, "Linux/UNIX", aws.StringValue(cr.InstancePlatform))
+	assert.Equal(t, int64(1), aws.Int64Value(cr.TotalInstanceCount))
+	assert.Equal(t, int64(1), aws.Int64Value(cr.AvailableInstanceCount))
+	assert.Equal(t, ec2.CapacityReservationStateActive, aws.StringValue(cr.State))
+	assert.Equal(t, ec2.InstanceMatchCriteriaOpen, aws.StringValue(cr.InstanceMatchCriteria), "match criteria defaults to open")
+	assert.Equal(t, ec2.CapacityReservationTenancyDefault, aws.StringValue(cr.Tenancy), "tenancy defaults to default")
+	assert.Equal(t, ec2.EndDateTypeUnlimited, aws.StringValue(cr.EndDateType))
+
+	daemon.resourceMgr.mu.RLock()
+	gotVCPU := daemon.resourceMgr.reservedCRVCPU
+	daemon.resourceMgr.mu.RUnlock()
+	assert.Equal(t, wantVCPU, gotVCPU, "carve-out committed to the resource manager")
+}
+
+// Create rejects an unknown type, a GPU type, and a request larger than the
+// node's remaining schedulable pool, leaving no carve-out behind in any case.
+func TestHandleEC2CreateCapacityReservation_Rejections(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	// Inject a synthetic GPU type; the catalog otherwise has none.
+	daemon.resourceMgr.instanceTypes["g4dn.xlarge"] = &ec2.InstanceTypeInfo{
+		InstanceType: aws.String("g4dn.xlarge"),
+		VCpuInfo:     &ec2.VCpuInfo{DefaultVCpus: aws.Int64(4)},
+		MemoryInfo:   &ec2.MemoryInfo{SizeInMiB: aws.Int64(16384)},
+		GpuInfo:      &ec2.GpuInfo{Gpus: []*ec2.GpuDeviceInfo{{Name: aws.String("T4"), Count: aws.Int64(1)}}},
+	}
+
+	instType := getTestInstanceType(t)
+	it := daemon.resourceMgr.instanceTypes[instType]
+	require.NotNil(t, it)
+	overCount := int64(daemon.resourceMgr.canAllocate(it, 1000) + 1)
+
+	subject := "ec2.CreateCapacityReservation." + daemon.node
+	sub, err := daemon.natsConn.Subscribe(subject, daemon.handleEC2CreateCapacityReservation)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	tests := []struct {
+		name    string
+		input   *ec2.CreateCapacityReservationInput
+		wantErr string
+	}{
+		{
+			name: "unknown type",
+			input: &ec2.CreateCapacityReservationInput{
+				InstanceType: aws.String("nonexistent.type"), InstanceCount: aws.Int64(1),
+				AvailabilityZone: aws.String("ap-southeast-2a"),
+			},
+			wantErr: awserrors.ErrorInvalidInstanceType,
+		},
+		{
+			name: "gpu type",
+			input: &ec2.CreateCapacityReservationInput{
+				InstanceType: aws.String("g4dn.xlarge"), InstanceCount: aws.Int64(1),
+				AvailabilityZone: aws.String("ap-southeast-2a"),
+			},
+			wantErr: awserrors.ErrorInvalidInstanceType,
+		},
+		{
+			name: "exceeds schedulable",
+			input: &ec2.CreateCapacityReservationInput{
+				InstanceType: aws.String(instType), InstanceCount: aws.Int64(overCount),
+				AvailabilityZone: aws.String("ap-southeast-2a"),
+			},
+			wantErr: awserrors.ErrorInsufficientInstanceCapacity,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reqData, _ := json.Marshal(tt.input)
+			reply, err := natsRequest(daemon.natsConn, subject, reqData, 5*time.Second)
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantErr, replyErrCode(t, reply.Data))
+		})
+	}
+
+	daemon.resourceMgr.mu.RLock()
+	defer daemon.resourceMgr.mu.RUnlock()
+	assert.Equal(t, 0, daemon.resourceMgr.reservedCRVCPU, "no carve-out after rejected creates")
+	assert.Empty(t, daemon.resourceMgr.reservations)
+}
+
+// The fan-out Describe handler returns only the caller's reservations.
+func TestHandleEC2DescribeCapacityReservations_Scoping(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	seedReservation(daemon.resourceMgr, "cr-aaaaaaaaaaaaaaaaa", testAccountID, getTestInstanceType(t))
+	seedReservation(daemon.resourceMgr, "cr-bbbbbbbbbbbbbbbbb", "999999999999", getTestInstanceType(t))
+
+	sub, err := daemon.natsConn.Subscribe("ec2.DescribeCapacityReservations", daemon.handleEC2DescribeCapacityReservations)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	reqData, _ := json.Marshal(&ec2.DescribeCapacityReservationsInput{})
+
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeCapacityReservations", reqData, 5*time.Second)
+	require.NoError(t, err)
+	var out ec2.DescribeCapacityReservationsOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &out))
+	require.Len(t, out.CapacityReservations, 1, "only the caller's reservation is returned")
+	assert.Equal(t, "cr-aaaaaaaaaaaaaaaaa", aws.StringValue(out.CapacityReservations[0].CapacityReservationId))
+
+	reply, err = natsRequestAs(daemon.natsConn, "ec2.DescribeCapacityReservations", reqData, "111111111111", 5*time.Second)
+	require.NoError(t, err)
+	var empty ec2.DescribeCapacityReservationsOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &empty))
+	assert.Empty(t, empty.CapacityReservations, "an account with no reservations sees none")
+}
+
+// The broadcast Cancel handler releases the carve-out only for the owning
+// account, and every node acks found/not-found so the gateway can disambiguate.
+func TestHandleEC2CancelCapacityReservation_ScopingAndAck(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	instType := getTestInstanceType(t)
+	it := daemon.resourceMgr.instanceTypes[instType]
+	require.NotNil(t, it)
+	require.GreaterOrEqual(t, daemon.resourceMgr.canAllocate(it, 1000), 1)
+	rec := &capacityReservation{
+		ID:                    "cr-ccccccccccccccccc",
+		AccountID:             testAccountID,
+		InstanceType:          instType,
+		AvailabilityZone:      "ap-southeast-2a",
+		TotalInstanceCount:    1,
+		VCPUPerInstance:       int(instanceTypeVCPUs(it)),
+		MemGBPerInstance:      float64(instanceTypeMemoryMiB(it)) / 1024.0,
+		InstanceMatchCriteria: ec2.InstanceMatchCriteriaOpen,
+		Tenancy:               ec2.CapacityReservationTenancyDefault,
+		InstancePlatform:      "Linux/UNIX",
+		CreateDate:            time.Now().UTC(),
+	}
+	require.NoError(t, daemon.resourceMgr.CreateReservation(rec))
+
+	sub, err := daemon.natsConn.Subscribe("ec2.CancelCapacityReservation", daemon.handleEC2CancelCapacityReservation)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	cancel := func(account string) bool {
+		reqData, _ := json.Marshal(&ec2.CancelCapacityReservationInput{CapacityReservationId: aws.String(rec.ID)})
+		reply, rerr := natsRequestAs(daemon.natsConn, "ec2.CancelCapacityReservation", reqData, account, 5*time.Second)
+		require.NoError(t, rerr)
+		var out ec2.CancelCapacityReservationOutput
+		require.NoError(t, json.Unmarshal(reply.Data, &out))
+		return aws.BoolValue(out.Return)
+	}
+
+	assert.False(t, cancel("999999999999"), "a foreign account cannot cancel")
+	daemon.resourceMgr.mu.RLock()
+	heldVCPU := daemon.resourceMgr.reservedCRVCPU
+	daemon.resourceMgr.mu.RUnlock()
+	assert.Equal(t, rec.VCPUPerInstance, heldVCPU, "carve-out untouched after foreign cancel")
+
+	assert.True(t, cancel(testAccountID), "the owner cancels successfully")
+	daemon.resourceMgr.mu.RLock()
+	freedVCPU := daemon.resourceMgr.reservedCRVCPU
+	daemon.resourceMgr.mu.RUnlock()
+	assert.Equal(t, 0, freedVCPU, "carve-out released on cancel")
+
+	assert.False(t, cancel(testAccountID), "a second cancel reports not-found")
+}
+
+// Create subscribes the per-reservation launch subject (so a targeted launch has
+// a responder) and Cancel drops it (so a launch against the cancelled id returns
+// ErrNoResponders → InvalidCapacityReservationId.NotFound at the gateway).
+func TestHandleEC2CapacityReservation_LaunchSubjectLifecycle(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	instType := getTestInstanceType(t)
+	it := daemon.resourceMgr.instanceTypes[instType]
+	require.NotNil(t, it)
+	require.GreaterOrEqual(t, daemon.resourceMgr.canAllocate(it, 1000), 1)
+
+	createSubject := "ec2.CreateCapacityReservation." + daemon.node
+	createSub, err := daemon.natsConn.Subscribe(createSubject, daemon.handleEC2CreateCapacityReservation)
+	require.NoError(t, err)
+	defer func() { _ = createSub.Unsubscribe() }()
+	cancelSub, err := daemon.natsConn.Subscribe("ec2.CancelCapacityReservation", daemon.handleEC2CancelCapacityReservation)
+	require.NoError(t, err)
+	defer func() { _ = cancelSub.Unsubscribe() }()
+
+	reqData, _ := json.Marshal(&ec2.CreateCapacityReservationInput{
+		InstanceType: aws.String(instType), InstanceCount: aws.Int64(1),
+		AvailabilityZone: aws.String("ap-southeast-2a"), InstancePlatform: aws.String("Linux/UNIX"),
+	})
+	reply, err := natsRequest(daemon.natsConn, createSubject, reqData, 5*time.Second)
+	require.NoError(t, err)
+	require.Empty(t, replyErrCode(t, reply.Data), "create should not error: %s", reply.Data)
+	var cr ec2.CapacityReservation
+	require.NoError(t, json.Unmarshal(reply.Data, &cr))
+	crID := aws.StringValue(cr.CapacityReservationId)
+
+	daemon.mu.Lock()
+	_, tracked := daemon.natsSubscriptions[crID]
+	daemon.mu.Unlock()
+	assert.True(t, tracked, "create tracks the cr launch subscription")
+
+	// An empty payload makes handleEC2RunInstances reject on MissingParameter
+	// before any allocation; the point is only that a responder exists (no
+	// ErrNoResponders) — the SUB raced ahead of the create reply on the same conn.
+	launchSubject := reservationLaunchSubject(crID)
+	_, err = natsRequest(daemon.natsConn, launchSubject, []byte("{}"), 2*time.Second)
+	require.NotErrorIs(t, err, nats.ErrNoResponders, "cr launch subject has a responder after create")
+
+	cancelData, _ := json.Marshal(&ec2.CancelCapacityReservationInput{CapacityReservationId: aws.String(crID)})
+	creply, err := natsRequest(daemon.natsConn, "ec2.CancelCapacityReservation", cancelData, 5*time.Second)
+	require.NoError(t, err)
+	var cout ec2.CancelCapacityReservationOutput
+	require.NoError(t, json.Unmarshal(creply.Data, &cout))
+	require.True(t, aws.BoolValue(cout.Return), "owner cancel succeeds")
+
+	daemon.mu.Lock()
+	_, tracked = daemon.natsSubscriptions[crID]
+	daemon.mu.Unlock()
+	assert.False(t, tracked, "cancel drops the cr launch subscription")
+
+	_, err = natsRequest(daemon.natsConn, launchSubject, []byte("{}"), 1*time.Second)
+	assert.ErrorIs(t, err, nats.ErrNoResponders, "cr launch subject has no responder after cancel")
+}
@@ -0,0 +1,91 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleECRRepoCreate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.RepoCreate)
+}
+
+func (d *Daemon) handleECRRepoDescribe(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.RepoDescribe)
+}
+
+func (d *Daemon) handleECRRepoList(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.RepoList)
+}
+
+func (d *Daemon) handleECRRepoDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.RepoDelete)
+}
+
+func (d *Daemon) handleECRPolicyPut(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.PolicyPut)
+}
+
+func (d *Daemon) handleECRPolicyGet(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.PolicyGet)
+}
+
+func (d *Daemon) handleECRPolicyDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.PolicyDelete)
+}
+
+func (d *Daemon) handleECRLifecyclePut(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.LifecyclePut)
+}
+
+func (d *Daemon) handleECRLifecycleGet(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.LifecycleGet)
+}
+
+func (d *Daemon) handleECRLifecycleDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.LifecycleDelete)
+}
+
+func (d *Daemon) handleECRTagPut(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.TagPut)
+}
+
+func (d *Daemon) handleECRTagGet(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.TagGet)
+}
+
+func (d *Daemon) handleECRTagList(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.TagList)
+}
+
+func (d *Daemon) handleECRTagDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.TagDelete)
+}
+
+func (d *Daemon) handleECRManifestPut(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.ManifestPut)
+}
+
+func (d *Daemon) handleECRManifestDescribe(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.ManifestDescribe)
+}
+
+func (d *Daemon) handleECRManifestList(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.ManifestList)
+}
+
+func (d *Daemon) handleECRManifestDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.ManifestDelete)
+}
+
+func (d *Daemon) handleECRUploadCreate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.UploadCreate)
+}
+
+func (d *Daemon) handleECRUploadGet(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.UploadGet)
+}
+
+func (d *Daemon) handleECRUploadUpdate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.UploadUpdate)
+}
+
+func (d *Daemon) handleECRUploadDelete(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecrMetaService.UploadDelete)
+}
@@ -50,7 +50,21 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 		return
 	}

-	reservation, instances, instanceType, err := d.instanceService.PrepareRunInstances(input, accountID)
+	// Targeted launch: the gateway routes only when an explicit reservation id is
+	// present, but the id rides in the input either way. Validate semantics
+	// up-front (the per-instance atomic re-check under rm.mu covers the race);
+	// the count gate in PrepareRunInstances handles a full reservation.
+	reservationID := capacityReservationTargetID(input)
+	if reservationID != "" {
+		if it, ok := d.resourceMgr.instanceTypes[aws.StringValue(input.InstanceType)]; ok {
+			if vErr := d.resourceMgr.ValidateReservationTarget(reservationID, accountID, it); vErr != nil {
+				respondWithError(msg, awserrors.ValidErrorCode(vErr.Error()))
+				return
+			}
+		}
+	}
+
+	reservation, instances, instanceType, err := d.instanceService.PrepareRunInstances(input, accountID, reservationID)
 	if err != nil {
 		respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
 		return
@@ -68,7 +82,11 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 		slog.Error("handleEC2RunInstances failed to marshal reservation", "err", err)
 		respondWithError(msg, awserrors.ErrorServerInternal)
 		for range instances {
-			d.resourceMgr.deallocate(instanceType)
+			if reservationID == "" {
+				d.resourceMgr.deallocate(instanceType)
+			} else {
+				d.resourceMgr.ReleaseToReservation(reservationID, instanceType)
+			}
 		}
 		return
 	}
@@ -100,6 +118,16 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 	d.instanceService.LaunchRunInstances(instances, input, instanceType)
 }

+// capacityReservationTargetID returns the explicit targeted-launch reservation id
+// from the input, or "" when the launch is untargeted (general path).
+func capacityReservationTargetID(input *ec2.RunInstancesInput) string {
+	spec := input.CapacityReservationSpecification
+	if spec == nil || spec.CapacityReservationTarget == nil {
+		return ""
+	}
+	return aws.StringValue(spec.CapacityReservationTarget.CapacityReservationId)
+}
+
 // describeInstancesValidFilters defines the set of filter names accepted by DescribeInstances.
 var describeInstancesValidFilters = map[string]bool{
 	"instance-state-name": true,
@@ -307,6 +335,19 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
 					}
 				}

+				// Echo the consumed capacity reservation so targeted-launch
+				// Terraform converges — without it the instance reports no
+				// reservation and the plan never settles.
+				if instance.CapacityReservationId != "" {
+					instanceCopy.CapacityReservationId = aws.String(instance.CapacityReservationId)
+					instanceCopy.CapacityReservationSpecification = &ec2.CapacityReservationSpecificationResponse{
+						CapacityReservationPreference: aws.String(ec2.CapacityReservationPreferenceOpen),
+						CapacityReservationTarget: &ec2.CapacityReservationTargetResponse{
+							CapacityReservationId: aws.String(instance.CapacityReservationId),
+						},
+					}
+				}
+
 				// Apply filters against the fully-built instance copy
 				if len(parsedFilters) > 0 && !instanceMatchesFilters(instance, &instanceCopy, parsedFilters) {
 					continue
@@ -595,3 +595,82 @@ func TestHandleEC2DescribeStoppedInstances_ReservationGrouping(t *testing.T) {
 	assert.Equal(t, "r-shared", *output.Reservations[0].ReservationId)
 	assert.Len(t, output.Reservations[0].Instances, 2)
 }
+
+// DescribeInstances echoes the consumed capacity reservation so a targeted
+// launch reports a CapacityReservationId (without it, Terraform never converges).
+func TestHandleEC2DescribeInstances_CapacityReservationEcho(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	reservation := &ec2.Reservation{}
+	reservation.SetReservationId("r-cr-echo")
+	reservation.SetOwnerId(testAccountID)
+	instance := &ec2.Instance{}
+	instance.SetInstanceId("i-cr-echo")
+	instance.SetInstanceType("t3.micro")
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:                    "i-cr-echo",
+		Status:                vm.StateRunning,
+		AccountID:             testAccountID,
+		CapacityReservationId: "cr-0123456789abcdef0",
+		Reservation:           reservation,
+		Instance:              instance,
+	})
+
+	sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstances", daemon.handleEC2DescribeInstances)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	reqData, _ := json.Marshal(&ec2.DescribeInstancesInput{})
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstances", reqData, 5*time.Second)
+	require.NoError(t, err)
+	var out ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &out))
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+
+	got := out.Reservations[0].Instances[0]
+	assert.Equal(t, "cr-0123456789abcdef0", aws.StringValue(got.CapacityReservationId))
+	require.NotNil(t, got.CapacityReservationSpecification)
+	assert.Equal(t, ec2.CapacityReservationPreferenceOpen,
+		aws.StringValue(got.CapacityReservationSpecification.CapacityReservationPreference))
+	require.NotNil(t, got.CapacityReservationSpecification.CapacityReservationTarget)
+	assert.Equal(t, "cr-0123456789abcdef0",
+		aws.StringValue(got.CapacityReservationSpecification.CapacityReservationTarget.CapacityReservationId))
+}
+
+// An instance with no reservation reports no capacity-reservation fields.
+func TestHandleEC2DescribeInstances_NoCapacityReservationEcho(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	reservation := &ec2.Reservation{}
+	reservation.SetReservationId("r-plain")
+	reservation.SetOwnerId(testAccountID)
+	instance := &ec2.Instance{}
+	instance.SetInstanceId("i-plain")
+	instance.SetInstanceType("t3.micro")
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:          "i-plain",
+		Status:      vm.StateRunning,
+		AccountID:   testAccountID,
+		Reservation: reservation,
+		Instance:    instance,
+	})
+
+	sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstances", daemon.handleEC2DescribeInstances)
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	reqData, _ := json.Marshal(&ec2.DescribeInstancesInput{})
+	reply, err := natsRequest(daemon.natsConn, "ec2.DescribeInstances", reqData, 5*time.Second)
+	require.NoError(t, err)
+	var out ec2.DescribeInstancesOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &out))
+	require.Len(t, out.Reservations, 1)
+	require.Len(t, out.Reservations[0].Instances, 1)
+
+	got := out.Reservations[0].Instances[0]
+	assert.Empty(t, aws.StringValue(got.CapacityReservationId))
+	assert.Nil(t, got.CapacityReservationSpecification)
+}
@@ -1,15 +1,59 @@
 package daemon

-import "github.com/nats-io/nats.go"
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/nats-io/nats.go"
+)

 func (d *Daemon) handleEC2CreateTags(msg *nats.Msg) {
-	handleNATSRequest(msg, d.tagsService.CreateTags)
+	handleNATSRequest(msg, d.createTags)
 }

 func (d *Daemon) handleEC2DeleteTags(msg *nats.Msg) {
-	handleNATSRequest(msg, d.tagsService.DeleteTags)
+	handleNATSRequest(msg, d.deleteTags)
 }

 func (d *Daemon) handleEC2DescribeTags(msg *nats.Msg) {
 	handleNATSRequest(msg, d.tagsService.DescribeTags)
 }
+
+// createTags writes the generic tag store, then mirrors tags into owning records
+// so tag-filtered describes observe them: subnet/vpc (e.g. LBC subnet
+// auto-discovery via DescribeSubnets) and AMIs (DescribeImages).
+func (d *Daemon) createTags(input *ec2.CreateTagsInput, accountID string) (*ec2.CreateTagsOutput, error) {
+	out, err := d.tagsService.CreateTags(input, accountID)
+	if err != nil {
+		return nil, err
+	}
+	if d.vpcService != nil {
+		if err := d.vpcService.ApplyRecordTags(input, accountID); err != nil {
+			return nil, err
+		}
+	}
+	if d.imageService != nil {
+		if err := d.imageService.ApplyRecordTags(input, accountID); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+// deleteTags removes from the generic tag store, then mirrors the removal into
+// the owning subnet/vpc records and AMI configs.
+func (d *Daemon) deleteTags(input *ec2.DeleteTagsInput, accountID string) (*ec2.DeleteTagsOutput, error) {
+	out, err := d.tagsService.DeleteTags(input, accountID)
+	if err != nil {
+		return nil, err
+	}
+	if d.vpcService != nil {
+		if err := d.vpcService.RemoveRecordTags(input, accountID); err != nil {
+			return nil, err
+		}
+	}
+	if d.imageService != nil {
+		if err := d.imageService.RemoveRecordTags(input, accountID); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
@@ -16,7 +16,6 @@ import (
 	awss3 "github.com/aws/aws-sdk-go/service/s3"

 	"github.com/mulgadc/spinifex/spinifex/awserrors"
-	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account"
 	handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw"
 	handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
@@ -1125,21 +1124,6 @@ func TestHandleEC2CreateImage_MalformedJSON(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationError, errResp["Code"])
 }

-// --- SetConfigPath test ---
-
-func TestSetConfigPath(t *testing.T) {
-	clusterCfg := &config.ClusterConfig{
-		Node:  "node-1",
-		Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
-	}
-	daemon, err := NewDaemon(clusterCfg)
-	require.NoError(t, err)
-
-	assert.Empty(t, daemon.configPath)
-	daemon.SetConfigPath("/etc/spinifex/config.toml")
-	assert.Equal(t, "/etc/spinifex/config.toml", daemon.configPath)
-}
-
 // --- handleEC2StartStoppedInstance tests ---

 func TestHandleEC2StartStoppedInstance_MissingInstance(t *testing.T) {
@@ -427,7 +427,7 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput)
 		}}
 	}

-	_, instances, instanceType, err := d.instanceService.PrepareRunInstances(runInput, input.AccountID)
+	_, instances, instanceType, err := d.instanceService.PrepareRunInstances(runInput, input.AccountID, "")
 	if err != nil {
 		return nil, err
 	}
@@ -507,10 +507,31 @@ func (d *Daemon) attachSystemMgmtNIC(inst *vm.VM) error {
 // stop qemu and free the ENI — otherwise the local-only path returns NotFound
 // and the teardown deletes a still-attached ENI (InvalidNetworkInterface.InUse).
 func (d *Daemon) TerminateSystemInstance(instanceID string) error {
+	var termErr error
 	if _, exists := d.vmMgr.Get(instanceID); exists {
-		return d.terminateSystemInstanceLocal(instanceID)
+		termErr = d.terminateSystemInstanceLocal(instanceID)
+	} else {
+		termErr = d.terminateSystemInstanceRemote(instanceID)
+	}
+
+	// Backstop for the no-owner case: terminateSystemInstanceLocal runs the
+	// reclaim on the owning node, but when no node owns the VM (already gone) the
+	// remote route returns NotFound and never reaches it, so an internet-facing
+	// ALB's EIP would orphan. Idempotent — a no-op once the record is released.
+	d.reclaimSystemInstanceEIP(instanceID)
+	return termErr
+}
+
+// reclaimSystemInstanceEIP releases any EIP still recorded against instanceID via
+// the authoritative EIP KV, independent of the cached fields on the VM record.
+// Idempotent and nil-safe; logs on failure.
+func (d *Daemon) reclaimSystemInstanceEIP(instanceID string) {
+	if d.eipService == nil {
+		return
+	}
+	if err := d.eipService.ReleaseAddressByInstanceID(instanceID); err != nil {
+		slog.Warn("reclaimSystemInstanceEIP: backstop release failed", "instanceId", instanceID, "err", err)
 	}
-	return d.terminateSystemInstanceRemote(instanceID)
 }

 // terminateSystemInstanceLocal stops a VM owned by this node.
@@ -521,8 +542,12 @@ func (d *Daemon) terminateSystemInstanceLocal(instanceID string) error {
 	}

 	// Release EIP through the EIP service for system VMs whose public IP was
-	// allocated via AllocateAddress (internet-facing ALBs).
+	// allocated via AllocateAddress (internet-facing ALBs). releaseSystemInstanceEIP
+	// uses the VM's cached allocation; reclaimSystemInstanceEIP then reconciles
+	// against the EIP KV in case the cache was empty (the authoritative path — this
+	// runs on the owning node, including the NATS terminate handler).
 	d.releaseSystemInstanceEIP(instance)
+	d.reclaimSystemInstanceEIP(instanceID)

 	if err := d.vmMgr.Terminate(instanceID); err != nil {
 		slog.Error("TerminateSystemInstance: vmMgr.Terminate failed", "instanceId", instanceID, "err", err)
@@ -484,6 +484,18 @@ func TestHandleEC2DescribeInstanceTypes(t *testing.T) {
 		initialCount := len(initialOutput.InstanceTypes)
 		t.Logf("Initial supported instance types: %d", initialCount)

+		// Pin host capacity so the allocate below is independent of the
+		// runner's live headroom. Without this, readMemAvailableGB reads
+		// live /proc MemAvailable, which a box already running VMs can drive
+		// below a 2 vCPU type's footprint, failing allocate spuriously.
+		daemon.resourceMgr.mu.Lock()
+		daemon.resourceMgr.hostVCPU = 16
+		daemon.resourceMgr.hostMemGB = 32.0
+		daemon.resourceMgr.reservedVCPU = 0
+		daemon.resourceMgr.reservedMem = 0
+		daemon.resourceMgr.readMemAvailableGB = nil
+		daemon.resourceMgr.mu.Unlock()
+
 		schedulableMem := daemon.resourceMgr.hostMemGB - daemon.resourceMgr.reservedMem
 		var instanceType2CPU *ec2.InstanceTypeInfo
 		for _, it := range initialOutput.InstanceTypes {
@@ -759,40 +771,6 @@ func TestDaemon_BootAllocation(t *testing.T) {
 	assert.Equal(t, expectedMem, daemon.resourceMgr.allocatedMem)
 }

-// TestStopInstance_Deallocation verifies that stopping an instance deallocates resources
-func TestStopInstance_Deallocation(t *testing.T) {
-	clusterCfg := &config.ClusterConfig{
-		Node:  "node-1",
-		Nodes: map[string]config.Config{"node-1": {BaseDir: "/tmp"}},
-	}
-	daemon, err := NewDaemon(clusterCfg)
-	require.NoError(t, err)
-
-	// Setup a running instance with allocated resources
-	instanceId := "i-test-stop"
-	instanceTypeStr := getTestInstanceType(t)
-	instanceType := daemon.resourceMgr.instanceTypes[instanceTypeStr]
-	daemon.vmMgr.Insert(&vm.VM{
-		ID:           instanceId,
-		InstanceType: instanceTypeStr,
-		Status:       vm.StateRunning,
-		AccountID:    testAccountID,
-	})
-
-	err = daemon.resourceMgr.allocate(instanceType)
-	require.NoError(t, err)
-	assert.Greater(t, daemon.resourceMgr.allocatedVCPU, 0)
-
-	// Call stopInstance (we can't easily wait for QMP/PID here, so we just want to see deallocate call)
-	// Actually stopInstance runs in goroutines and waits for PID removal.
-	// This might be tricky to test without heavy mocking.
-
-	// Let's test the ResourceManager deallocate directly since we've already verified
-	// that stopInstance calls it in the code.
-	daemon.resourceMgr.deallocate(instanceType)
-	assert.Equal(t, 0, daemon.resourceMgr.allocatedVCPU)
-}
-
 // TestCanAllocate_CountEdgeCases tests edge cases for canAllocate with count parameter
 func TestCanAllocate_CountEdgeCases(t *testing.T) {
 	t.Run("MinCount_equals_MaxCount", func(t *testing.T) {
@@ -904,6 +882,18 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) {
 		require.NotNil(t, microType)
 		require.NotNil(t, mediumType)

+		// Pin host capacity so the test is independent of the runner's
+		// schedulable headroom. A constrained box (e.g. 4 vCPU, 2 schedulable
+		// after reserve, more when EKS VMs hold capacity) cannot fit a medium
+		// and rm.allocate would fail with an unexpected insufficient-capacity error.
+		rm.mu.Lock()
+		rm.hostVCPU = 16
+		rm.hostMemGB = 32.0
+		rm.reservedVCPU = 0
+		rm.reservedMem = 0
+		rm.readMemAvailableGB = nil // accounting test: pin to synthetic host, not live /proc
+		rm.mu.Unlock()
+
 		initialMicro := rm.canAllocate(microType, 100)
 		initialMedium := rm.canAllocate(mediumType, 100)

@@ -1627,78 +1617,6 @@ func TestResourceManager_ConcurrentAccess(t *testing.T) {
 	assert.Equal(t, float64(0), rm.allocatedMem, "All memory should be deallocated")
 }

-// TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping verifies that
-// the deleteOnTermination flag from RunInstancesInput.BlockDeviceMappings is
-// propagated to the EBSRequest on the instance's volume list.
-func TestGenerateVolumes_DeleteOnTermination_FromBlockDeviceMapping(t *testing.T) {
-	tests := []struct {
-		name                    string
-		deleteOnTerminationFlag *bool
-		expectedFlag            bool
-	}{
-		{
-			name:                    "DeleteOnTermination=true",
-			deleteOnTerminationFlag: aws.Bool(true),
-			expectedFlag:            true,
-		},
-		{
-			name:                    "DeleteOnTermination=false",
-			deleteOnTerminationFlag: aws.Bool(false),
-			expectedFlag:            false,
-		},
-		{
-			name:                    "DeleteOnTermination=nil (defaults to true)",
-			deleteOnTerminationFlag: nil,
-			expectedFlag:            true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Build a RunInstancesInput with BlockDeviceMappings
-			input := &ec2.RunInstancesInput{
-				ImageId:      aws.String("vol-existing-volume"),
-				InstanceType: aws.String("t3.micro"),
-				MinCount:     aws.Int64(1),
-				MaxCount:     aws.Int64(1),
-			}
-
-			if tt.deleteOnTerminationFlag != nil {
-				input.BlockDeviceMappings = []*ec2.BlockDeviceMapping{
-					{
-						DeviceName: aws.String("/dev/vda"),
-						Ebs: &ec2.EbsBlockDevice{
-							VolumeSize:          aws.Int64(8),
-							DeleteOnTermination: tt.deleteOnTerminationFlag,
-						},
-					},
-				}
-			}
-
-			// Exercise the parsing logic that GenerateVolumes uses
-			// Default is true (matches AWS RunInstances behavior for root volumes)
-			deleteOnTermination := true
-			if len(input.BlockDeviceMappings) > 0 {
-				bdm := input.BlockDeviceMappings[0]
-				if bdm.Ebs != nil && bdm.Ebs.DeleteOnTermination != nil {
-					deleteOnTermination = *bdm.Ebs.DeleteOnTermination
-				}
-			}
-
-			assert.Equal(t, tt.expectedFlag, deleteOnTermination,
-				"deleteOnTermination should match expected value")
-
-			// Verify the flag is correctly assigned to an EBSRequest
-			ebsReq := types.EBSRequest{
-				Name:                "vol-test",
-				Boot:                true,
-				DeleteOnTermination: deleteOnTermination,
-			}
-			assert.Equal(t, tt.expectedFlag, ebsReq.DeleteOnTermination)
-		})
-	}
-}
-
 // TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination tests that
 // the instanceCleanerAdapter correctly handles DeleteOnTermination for each
 // volume type: internal volumes (EFI, cloud-init) always go through
@@ -3279,7 +3197,8 @@ func TestVolumeMounterAdapter_UnmountOne_Success(t *testing.T) {
 }

 // TestVolumeMounterAdapter_UnmountOne_UnmountError verifies that UnmountOne
-// tolerates an error response from ebs.unmount (logs only, no propagation).
+// propagates an error response from ebs.unmount. DetachVolume gates the volume's
+// available transition on this error, so it must not be swallowed.
 func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
 	natsURL := sharedNATSURL

@@ -3294,11 +3213,12 @@ func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()

-	adapter.UnmountOne(types.EBSRequest{Name: "vol-rollback-err"})
+	require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-rollback-err"}),
+		"an ebs.unmount error response must propagate so DetachVolume can keep the volume attached")
 }

 // TestVolumeMounterAdapter_UnmountOne_StillMounted verifies that UnmountOne
-// tolerates an unmount response that reports the volume still mounted.
+// returns an error when the unmount response reports the volume still mounted.
 func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
 	natsURL := sharedNATSURL

@@ -3313,18 +3233,103 @@ func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
 	require.NoError(t, err)
 	defer sub.Unsubscribe()

-	adapter.UnmountOne(types.EBSRequest{Name: "vol-still-mounted"})
+	require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-still-mounted"}),
+		"a still-mounted response must propagate as an error")
 }

-// TestVolumeMounterAdapter_UnmountOne_NATSTimeout verifies that UnmountOne
-// tolerates NATS request timeout (no subscriber on ebs.unmount).
-func TestVolumeMounterAdapter_UnmountOne_NATSTimeout(t *testing.T) {
-	natsURL := sharedNATSURL
+// TestVolumeMounterAdapter_UnmountOne_RequestFailure verifies that UnmountOne
+// surfaces a failed ebs.unmount NATS request as an error. A closed connection
+// fails immediately, rather than blocking on the full unmountSealTimeout (120s)
+// that a no-subscriber timeout would incur in every preflight run.
+func TestVolumeMounterAdapter_UnmountOne_RequestFailure(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	nc.Close()

-	daemon := createTestDaemon(t, natsURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	adapter := newVolumeMounterAdapter(nc, "node-1", nil)
+	require.Error(t, adapter.UnmountOne(types.EBSRequest{Name: "vol-timeout"}),
+		"a failed ebs.unmount request must propagate as an error")
+}

-	adapter.UnmountOne(types.EBSRequest{Name: "vol-timeout"})
+// recordingVolState records UpdateVolumeState calls so the bulk-unmount test can
+// assert which volumes transitioned to available.
+type recordingVolState struct {
+	mu    sync.Mutex
+	calls []string
+}
+
+func (r *recordingVolState) UpdateVolumeState(volumeID, state, instanceID, attachmentDevice string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.calls = append(r.calls, volumeID+":"+state)
+	return nil
+}
+
+func (r *recordingVolState) snapshot() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.calls...)
+}
+
+var _ vm.VolumeStateUpdater = (*recordingVolState)(nil)
+
+// TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable verifies the
+// teardown-path durability gate: a per-volume seal failure during the bulk
+// Unmount (stop/terminate) must NOT transition that volume to available — a
+// reattach on a WAL-less node would find no checkpoint (bad superblock) — while
+// other volumes still seal and the loop completes (terminate stays idempotent).
+func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	volState := &recordingVolState{}
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
+
+	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+		var req types.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		resp := types.EBSUnMountResponse{Volume: req.Name, Mounted: false}
+		if req.Name == "vol-fail" {
+			resp.Error = "seal volume: predastore unreachable"
+		}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	inst := &vm.VM{
+		ID: "i-1",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{{Name: "vol-fail"}, {Name: "vol-ok"}},
+		},
+	}
+	require.NoError(t, adapter.Unmount(inst),
+		"bulk Unmount must tolerate a per-volume seal failure so terminate stays idempotent")
+
+	calls := volState.snapshot()
+	assert.NotContains(t, calls, "vol-fail:available",
+		"a failed seal must not transition the volume to available")
+	assert.Contains(t, calls, "vol-ok:available",
+		"a successful seal must still transition the volume to available")
+}
+
+// TestUnmountResponseError covers the shared seal-result parser used by both the
+// gated DetachVolume path (unmountOne) and the tolerated teardown path (Unmount).
+func TestUnmountResponseError(t *testing.T) {
+	t.Run("success (not mounted, no error)", func(t *testing.T) {
+		data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Mounted: false})
+		require.NoError(t, unmountResponseError(data))
+	})
+	t.Run("seal error propagates", func(t *testing.T) {
+		data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Error: "seal volume: boom"})
+		require.Error(t, unmountResponseError(data))
+	})
+	t.Run("still mounted propagates", func(t *testing.T) {
+		data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", Mounted: true})
+		require.Error(t, unmountResponseError(data))
+	})
+	t.Run("malformed payload propagates", func(t *testing.T) {
+		require.Error(t, unmountResponseError([]byte("not json")))
+	})
 }

 // TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
@@ -25,6 +25,17 @@ func (a *daemonEKSSubnetResolver) GetSubnetVPC(accountID, subnetID string) (stri
 	return rec.VpcId, nil
 }

+func (a *daemonEKSSubnetResolver) GetSubnetAZ(accountID, subnetID string) (string, error) {
+	if a.d == nil || a.d.vpcService == nil {
+		return "", errors.New("eks: VPC service not initialized")
+	}
+	rec, err := a.d.vpcService.GetSubnet(accountID, subnetID)
+	if err != nil {
+		return "", err
+	}
+	return rec.AvailabilityZone, nil
+}
+
 func (a *daemonEKSSubnetResolver) GetVPCCIDR(accountID, vpcID string) (string, error) {
 	if a.d == nil || a.d.vpcService == nil {
 		return "", errors.New("eks: VPC service not initialized")
@@ -21,6 +21,11 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 		masterKey = nil
 	}

+	internalSuffix := ""
+	if d.clusterConfig != nil {
+		internalSuffix = d.clusterConfig.AWS.InternalSuffix
+	}
+
 	gatewayCA := ""
 	if d.config.NATS.CACert != "" {
 		if caBytes, readErr := os.ReadFile(d.config.NATS.CACert); readErr == nil {
@@ -31,13 +36,14 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 		}
 	}

-	return handlers_eks.EKSServiceDeps{
+	deps := handlers_eks.EKSServiceDeps{
 		Config:           d.config,
 		NATSConn:         d.natsConn,
 		MasterKey:        masterKey,
 		GatewayBaseURL:   d.resolveGatewayBaseURL(),
 		Region:           d.config.Region,
 		HolderID:         d.node,
+		InternalSuffix:   internalSuffix,
 		SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
 		SystemAccessKey:  d.config.Predastore.AccessKey,
 		SystemSecretKey:  d.config.Predastore.SecretKey,
@@ -57,6 +63,25 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 		PlacementGroup:   handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
 		Scheduler:        handlers_eks.NewNATSHostScheduler(d.natsConn),
 	}
+
+	// A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS
+	// find-or-create the node-role instance profile workers need for ECR pulls.
+	// Only wired when the master key is present; a typed-nil would defeat the
+	// service's own nil guard.
+	if masterKey != nil {
+		clusterSize := 1
+		if d.clusterConfig != nil {
+			clusterSize = len(d.clusterConfig.Nodes)
+		}
+		if iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.natsConn, masterKey, clusterSize); iamErr != nil {
+			slog.Warn("EKS: IAM service init failed; nodegroup workers launch without an instance profile, internal ECR pulls will fail",
+				"err", iamErr)
+		} else {
+			deps.IAM = iamSvc
+		}
+	}
+
+	return deps
 }

 // resolveGatewayHost returns the single AWSGW host used by LB VMs, EKS OIDC
@@ -2,14 +2,19 @@ package daemon

 import (
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
+	"time"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
 	spxtypes "github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
 )

 // Compile-time check that Daemon satisfies the EKS worker-launch surface.
@@ -30,27 +35,25 @@ func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID strin
 	return d.instanceService.RunInstances(input, accountID)
 }

-// TerminateWorkerInstances terminates nodegroup workers. Already-gone instances
-// are treated as success so retried DeleteNodegroup calls are idempotent.
+// TerminateWorkerInstances terminates nodegroup workers by routing a terminate
+// command to whichever node owns each VM (per-instance ec2.cmd.<id>), mirroring
+// the gateway TerminateInstances path. The owning daemon runs the full teardown
+// — detach + force-delete the worker's primary ENI and clear its
+// spinifex-vpc-enis record — so a dangling in-use ENI cannot pin the customer
+// VPC/subnet undeletable (mulga-siv-408). A local-only vmMgr.Get would skip any
+// worker placed on another node, stranding both the VM and its ENI. An instance
+// with no owner (no responder, not found) is already gone, so a retried
+// DeleteNodegroup stays idempotent.
 func (d *Daemon) TerminateWorkerInstances(instanceIDs []string, accountID string) error {
-	if d.instanceService == nil {
-		return errors.New("eks worker: instance service not initialized")
+	if d.natsConn == nil {
+		return errors.New("eks worker: NATS connection not initialized")
 	}
 	var errs []error
 	for _, id := range instanceIDs {
 		if id == "" {
 			continue
 		}
-		instance, ok := d.vmMgr.Get(id)
-		if !ok {
-			slog.Debug("TerminateWorkerInstances: instance already gone", "instanceId", id)
-			continue
-		}
-		cmd := spxtypes.EC2InstanceCommand{
-			ID:         id,
-			Attributes: spxtypes.EC2CommandAttributes{TerminateInstance: true},
-		}
-		if err := d.instanceService.StopOrTerminateInstance(instance, cmd); err != nil {
+		if err := d.terminateWorkerInstance(id, accountID); err != nil {
 			errs = append(errs, fmt.Errorf("terminate worker %s: %w", id, err))
 		}
 	}
@@ -59,3 +62,52 @@ func (d *Daemon) TerminateWorkerInstances(instanceIDs []string, accountID string
 	}
 	return nil
 }
+
+// terminateWorkerInstance routes a single worker terminate to its owning node.
+// StopInstance is set alongside TerminateInstance so the owner does not restart
+// it. A NoResponders reply means no daemon owns the instance: it is already
+// gone, which a retried teardown treats as success. A NotFound error payload
+// from the owner is likewise idempotent.
+func (d *Daemon) terminateWorkerInstance(instanceID, accountID string) error {
+	cmd := spxtypes.EC2InstanceCommand{
+		ID: instanceID,
+		Attributes: spxtypes.EC2CommandAttributes{
+			StopInstance:      true,
+			TerminateInstance: true,
+		},
+	}
+	data, err := json.Marshal(cmd)
+	if err != nil {
+		return fmt.Errorf("marshal terminate command: %w", err)
+	}
+
+	subject := fmt.Sprintf("ec2.cmd.%s", instanceID)
+	var msg *nats.Msg
+	for attempt := range 3 {
+		reqMsg := nats.NewMsg(subject)
+		reqMsg.Data = data
+		reqMsg.Header.Set(utils.AccountIDHeader, accountID)
+		msg, err = d.natsConn.RequestMsg(reqMsg, 5*time.Second)
+		if err == nil || !errors.Is(err, nats.ErrNoResponders) {
+			break
+		}
+		if attempt < 2 {
+			time.Sleep(time.Duration(attempt+1) * time.Second)
+		}
+	}
+	if errors.Is(err, nats.ErrNoResponders) {
+		slog.Debug("TerminateWorkerInstances: no owner for instance, already gone", "instanceId", instanceID)
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if errPayload, parseErr := utils.ValidateErrorPayload(msg.Data); parseErr != nil {
+		if *errPayload.Code == awserrors.ErrorInvalidInstanceIDNotFound {
+			slog.Debug("TerminateWorkerInstances: owner reports instance gone, idempotent", "instanceId", instanceID)
+			return nil
+		}
+		return errors.New(*errPayload.Code)
+	}
+	return nil
+}
@@ -1,23 +1,28 @@
 package daemon

 import (
+	"encoding/json"
 	"testing"
+	"time"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	spxtypes "github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

-// A worker whose ID is not tracked by the local vmMgr is already-gone, so
+// A worker with no owner (no responder on ec2.cmd.<id>) is already-gone, so
 // terminating it is an idempotent no-op success (a retried DeleteNodegroup /
 // scale-down must not wedge on drained instances).
 func TestTerminateWorkerInstances_NotFoundIsIdempotent(t *testing.T) {
-	d := newDaemonWithVMs()
-	// Non-nil service passes the init guard; no VM matches the IDs so
-	// StopOrTerminateInstance is never reached — the not-found branch returns
-	// success.
-	d.instanceService = &handlers_ec2_instance.InstanceServiceImpl{}
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	d := &Daemon{natsConn: nc}

 	require.NoError(t, d.TerminateWorkerInstances([]string{"i-gone1", "i-gone2"}, "111122223333"))
 	// Empty / blank IDs are skipped.
@@ -25,8 +30,75 @@ func TestTerminateWorkerInstances_NotFoundIsIdempotent(t *testing.T) {
 	require.NoError(t, d.TerminateWorkerInstances(nil, "111122223333"))
 }

-// Both worker methods must guard against an uninitialized instance service
-// rather than nil-panic.
+// Worker terminate must route to whichever node owns the VM via ec2.cmd.<id>,
+// not a local-only vmMgr lookup; the owner runs the full teardown (incl. ENI
+// detach+delete) so no dangling ENI pins the VPC undeletable (mulga-siv-408).
+func TestTerminateWorkerInstances_RoutesToOwner(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	d := &Daemon{natsConn: nc}
+
+	gotCmd := make(chan spxtypes.EC2InstanceCommand, 1)
+	sub, err := nc.Subscribe("ec2.cmd.i-owned", func(msg *nats.Msg) {
+		var cmd spxtypes.EC2InstanceCommand
+		_ = json.Unmarshal(msg.Data, &cmd)
+		gotCmd <- cmd
+		_ = msg.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	require.NoError(t, d.TerminateWorkerInstances([]string{"i-owned"}, "111122223333"))
+
+	select {
+	case cmd := <-gotCmd:
+		assert.Equal(t, "i-owned", cmd.ID)
+		assert.True(t, cmd.Attributes.TerminateInstance, "owner must receive a terminate command")
+		assert.True(t, cmd.Attributes.StopInstance, "StopInstance set so the owner does not restart it")
+	case <-time.After(2 * time.Second):
+		t.Fatal("expected terminate command routed to the owning node")
+	}
+}
+
+// A NotFound error payload from the owner (instance already drained between
+// enumerate and terminate) is idempotent success, not a teardown failure.
+func TestTerminateWorkerInstances_OwnerNotFoundPayloadIdempotent(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	d := &Daemon{natsConn: nc}
+
+	sub, err := nc.Subscribe("ec2.cmd.i-raced", func(msg *nats.Msg) {
+		_ = msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	require.NoError(t, d.TerminateWorkerInstances([]string{"i-raced"}, "111122223333"))
+}
+
+// A non-NotFound error payload from the owner must surface so the teardown
+// backstop retries rather than silently leaving the worker (and its ENI) alive.
+func TestTerminateWorkerInstances_OwnerErrorSurfaces(t *testing.T) {
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	d := &Daemon{natsConn: nc}
+
+	sub, err := nc.Subscribe("ec2.cmd.i-protected", func(msg *nats.Msg) {
+		_ = msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorOperationNotPermitted))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	err = d.TerminateWorkerInstances([]string{"i-protected"}, "111122223333")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorOperationNotPermitted)
+}
+
+// Both worker methods must guard against an uninitialized daemon rather than
+// nil-panic.
 func TestWorkerLauncher_NilInstanceService(t *testing.T) {
 	d := &Daemon{}

@@ -77,6 +77,11 @@ type volumeMounterAdapter struct {

 var _ vm.VolumeMounter = (*volumeMounterAdapter)(nil)

+// unmountSealTimeout bounds the ebs.unmount NATS request. The handler now drives
+// a synchronous block-map seal to predastore (S3 I/O), so it matches
+// viperblockd's 120s KillProcess wait rather than a bare RPC budget.
+const unmountSealTimeout = 120 * time.Second
+
 func newVolumeMounterAdapter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
 	return &volumeMounterAdapter{nc: nc, node: node, volState: volState}
 }
@@ -152,16 +157,27 @@ func (a *volumeMounterAdapter) Unmount(instance *vm.VM) error {
 			continue
 		}

-		msg, err := a.nc.Request(a.topic("unmount"), ebsUnMountRequest, 30*time.Second)
+		// ebs.unmount seals the block map to predastore. Teardown tolerates a
+		// failed seal (log + continue) so terminate stays idempotent, but the
+		// volume must NOT then go available: a reattach on a node without the
+		// local WAL would find no checkpoint (bad superblock). On terminate the
+		// volume is deleted regardless; on stop it stays attached/retryable.
+		sealed := true
+		msg, err := a.nc.Request(a.topic("unmount"), ebsUnMountRequest, unmountSealTimeout)
 		if err != nil {
 			slog.Error("Failed to unmount volume",
 				"name", ebsRequest.Name, "instance", instance.ID, "err", err)
+			sealed = false
+		} else if sealErr := unmountResponseError(msg.Data); sealErr != nil {
+			slog.Error("Volume unmount seal failed, leaving volume non-available",
+				"instance", instance.ID, "volume", ebsRequest.Name, "err", sealErr)
+			sealed = false
 		} else {
 			slog.Info("Unmounted volume",
 				"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
 		}

-		if !ebsRequest.EFI && !ebsRequest.CloudInit && a.volState != nil {
+		if sealed && !ebsRequest.EFI && !ebsRequest.CloudInit && a.volState != nil {
 			if err := a.volState.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil {
 				slog.Error("Failed to update volume state to available after unmount",
 					"volumeId", ebsRequest.Name, "err", err)
@@ -200,13 +216,16 @@ func (a *volumeMounterAdapter) MountOne(req *types.EBSRequest) error {
 	return nil
 }

-// UnmountOne sends ebs.unmount; errors are logged and swallowed.
-func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) {
+// UnmountOne sends ebs.unmount and returns any error. The handler seals the
+// volume's block map to predastore, so the caller decides whether a failure
+// blocks the volume's available transition.
+func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) error {
 	if err := a.unmountOne(req); err != nil {
 		slog.Error("UnmountOne failed", "volume", req.Name, "err", err)
-		return
+		return err
 	}
 	slog.Info("UnmountOne: volume unmounted successfully", "volume", req.Name)
+	return nil
 }

 // unmountOne sends ebs.unmount and returns any error.
@@ -215,12 +234,19 @@ func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error {
 	if err != nil {
 		return fmt.Errorf("marshal unmount request: %w", err)
 	}
-	msg, err := a.nc.Request(a.topic("unmount"), payload, 10*time.Second)
+	msg, err := a.nc.Request(a.topic("unmount"), payload, unmountSealTimeout)
 	if err != nil {
 		return fmt.Errorf("ebs.unmount NATS request: %w", err)
 	}
+	return unmountResponseError(msg.Data)
+}
+
+// unmountResponseError reports a seal/unmount failure from an ebs.unmount
+// response payload: a non-empty Error, or a volume still reported mounted.
+// Returns nil when the unmount and its block-map seal succeeded.
+func unmountResponseError(data []byte) error {
 	var resp types.EBSUnMountResponse
-	if err := json.Unmarshal(msg.Data, &resp); err != nil {
+	if err := json.Unmarshal(data, &resp); err != nil {
 		return fmt.Errorf("unmarshal unmount response: %w", err)
 	}
 	if resp.Error != "" {
@@ -287,6 +313,14 @@ func (a *resourceControllerAdapter) Deallocate(instanceType string) {
 	a.rm.deallocate(it)
 }

+func (a *resourceControllerAdapter) ReleaseToReservation(reservationID, instanceType string) {
+	it := a.rm.instanceTypes[instanceType]
+	if it == nil {
+		return
+	}
+	a.rm.ReleaseToReservation(reservationID, it)
+}
+
 func (a *resourceControllerAdapter) CanAllocate(instanceType string, count int) int {
 	it := a.rm.instanceTypes[instanceType]
 	if it == nil {
@@ -28,6 +28,15 @@ var acmActions = map[string]acmHandler{
 	"DeleteCertificate": func(gw *GatewayConfig, acct string, b []byte) (any, error) {
 		return gateway_acm.DeleteCertificate(gw.NATSConn, acct, b)
 	},
+	"ListTagsForCertificate": func(gw *GatewayConfig, acct string, b []byte) (any, error) {
+		return gateway_acm.ListTagsForCertificate(gw.NATSConn, acct, b)
+	},
+	"AddTagsToCertificate": func(gw *GatewayConfig, acct string, b []byte) (any, error) {
+		return gateway_acm.AddTagsToCertificate(gw.NATSConn, acct, b)
+	},
+	"RemoveTagsFromCertificate": func(gw *GatewayConfig, acct string, b []byte) (any, error) {
+		return gateway_acm.RemoveTagsFromCertificate(gw.NATSConn, acct, b)
+	},
 }

 // acmActionFromTarget extracts the action suffix from an X-Amz-Target header.
@@ -44,3 +44,30 @@ func DeleteCertificate(natsConn *nats.Conn, accountID string, body []byte) (*acm
 	}
 	return handlers_acm.NewNATSACMService(natsConn).DeleteCertificate(input, accountID)
 }
+
+// ListTagsForCertificate — CertificateManager.ListTagsForCertificate
+func ListTagsForCertificate(natsConn *nats.Conn, accountID string, body []byte) (*acm.ListTagsForCertificateOutput, error) {
+	input := new(acm.ListTagsForCertificateInput)
+	if err := unmarshalIfBody(body, input); err != nil {
+		return nil, errors.New(awserrors.ErrorInvalidParameter)
+	}
+	return handlers_acm.NewNATSACMService(natsConn).ListTagsForCertificate(input, accountID)
+}
+
+// AddTagsToCertificate — CertificateManager.AddTagsToCertificate
+func AddTagsToCertificate(natsConn *nats.Conn, accountID string, body []byte) (*acm.AddTagsToCertificateOutput, error) {
+	input := new(acm.AddTagsToCertificateInput)
+	if err := unmarshalIfBody(body, input); err != nil {
+		return nil, errors.New(awserrors.ErrorInvalidParameter)
+	}
+	return handlers_acm.NewNATSACMService(natsConn).AddTagsToCertificate(input, accountID)
+}
+
+// RemoveTagsFromCertificate — CertificateManager.RemoveTagsFromCertificate
+func RemoveTagsFromCertificate(natsConn *nats.Conn, accountID string, body []byte) (*acm.RemoveTagsFromCertificateOutput, error) {
+	input := new(acm.RemoveTagsFromCertificateInput)
+	if err := unmarshalIfBody(body, input); err != nil {
+		return nil, errors.New(awserrors.ErrorInvalidParameter)
+	}
+	return handlers_acm.NewNATSACMService(natsConn).RemoveTagsFromCertificate(input, accountID)
+}
--- a/Show More
+++ b/Show More